From a40df206668454efe8c9e60de25084ba297f6153 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 12:07:28 +0300 Subject: [PATCH 01/69] =?UTF-8?q?docs:=20ADR-0033=20=E2=80=94=20registry?= =?UTF-8?q?=20evidence=20replaces=20the=20name-distinctiveness=20tier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records the decision before any code: a link publishes only against a checkable Trade Register fact (#279). If review rejects the grounds, the whole approach falls — which is why this ADR comes first rather than last. Seven decisions, each separately contestable: the evidence ladder (six outcomes, first match wins); a heuristic that grounds an assertion — the explicit exception to ADR-0007 decisions 2 and 3, together with what justifies it (corroboration against the official's own filing, never a standalone name match) and the filters that can only withhold; the joint-stock bar as a union of three signals, where an unknown legal-form code withholds; reconciliation of a terminated stake against the live deed, reversing ADR-0021 E11 (the "и към днешна дата" labels defer to phase 2); the deed cache, extending ADR-0010 decision 6 with a 35-day retention; monotonicity as a gate rather than a store — §8 of the issue is false as written; and the launch gates. Supersedes ADR-0009, ADR-0015, ADR-0017 (headers and index synchronised). Amends ADR-0007, ADR-0010, ADR-0021 (E11 only), ADR-0028 — accepted ADRs are not rewritten, so the amendment is recorded here. Cross-checked against issue #279 and the earlier spike; carries the three facts the issue does not: the sustained 429 at ~50 requests, the open question of the ЕАД legal-form code, and the deliberate non-use of the beneficial-ownership fields (C-37/20). --- ...name-uniqueness-guard-and-publish-tiers.md | 3 +- docs/adr/0015-tr-name-uniqueness-census.md | 3 +- docs/adr/0017-name-collision-tier-gate.md | 3 +- ...-evidence-replaces-name-distinctiveness.md | 309 ++++++++++++++++++ docs/adr/README.md | 7 +- 5 files changed, 319 insertions(+), 6 deletions(-) create mode 100644 docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md diff --git a/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md b/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md index 7803e901..8783b3ac 100644 --- a/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md +++ b/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md @@ -1,8 +1,9 @@ # ADR-0009: Name-uniqueness is not absolute → single-ЕИК guard + publish tiers -- Status: Accepted +- Status: Superseded by [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Date: 2026-07-05 - Deciders: lb, Claude +- Superseded by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Related: spec §5; refines [ADR-0008](0008-deterministic-name-to-eik-resolution.md) ## Context diff --git a/docs/adr/0015-tr-name-uniqueness-census.md b/docs/adr/0015-tr-name-uniqueness-census.md index b3fb2789..c86d8eec 100644 --- a/docs/adr/0015-tr-name-uniqueness-census.md +++ b/docs/adr/0015-tr-name-uniqueness-census.md @@ -1,8 +1,9 @@ # ADR-0015: TR name-uniqueness census (promoting tier-C generic-name matches) -- Status: Accepted (design; implemented as a Phase-1 pipeline step) +- Status: Superseded by [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Date: 2026-07-05 - Deciders: lb, Claude +- Superseded by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Related: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md); spec §5 ## Context diff --git a/docs/adr/0017-name-collision-tier-gate.md b/docs/adr/0017-name-collision-tier-gate.md index 8d4a234b..38ed31b5 100644 --- a/docs/adr/0017-name-collision-tier-gate.md +++ b/docs/adr/0017-name-collision-tier-gate.md @@ -1,8 +1,9 @@ # ADR-0017: A globally non-unique name cannot ride the name-distinctive tier — even with a certain ЕИК -- Status: Accepted +- Status: Superseded by [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Date: 2026-07-05 - Deciders: lb, Claude +- Superseded by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — the name-distinctive tier it gates leaves the publishing path; the `nameGloballyUnique` precondition survives, narrowed to the weakest evidence rung - Related: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md), [ADR-0016](0016-free-text-entity-resolution.md); `scripts/cacbg/load.mjs`, `scripts/cacbg/audit.mjs` ## Context diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md new file mode 100644 index 00000000..e4d8ff2d --- /dev/null +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -0,0 +1,309 @@ +# ADR-0033: A link publishes only against a Trade Register fact — the evidence ladder replaces the name-distinctiveness tier + +- Status: Accepted (design; implemented as the #279 pipeline change) +- Date: 2026-08-05 +- Deciders: Todor (maintainer), Claude +- Supersedes: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md), [ADR-0015](0015-tr-name-uniqueness-census.md), [ADR-0017](0017-name-collision-tier-gate.md) +- Amends: [ADR-0007](0007-scope-and-certainty-bar.md) (decisions 2 and 3), [ADR-0010](0010-pii-posture.md) (decisions 3 and 6), [ADR-0021](0021-methodology-page-and-temporal-freshness.md) (E11 only — E10 stands and is strengthened), [ADR-0028](0028-declared-eik-is-a-determining-identifier.md) +- Related: [ADR-0008](0008-deterministic-name-to-eik-resolution.md), [ADR-0011](0011-host-scoped-tls-pinning.md), [ADR-0026](0026-person-grain-name-institution.md), [ADR-0027](0027-overmerge-gate-is-telemetry-not-a-gate.md), [ADR-0031](0031-suppressions-version-controlled-fingerprinted.md), [ADR-0032](0032-family-ownership-published-under-public-interest.md); spec §3.3/§5/§8/§9, [related-persons-lia.md](../spec/related-persons-lia.md); `scripts/tr/`, `scripts/cacbg/{load,classify}.mjs`, `packages/db/src/queries/related-persons.ts`, `apps/web/app/routes/conflict*.tsx` + +## Context + +Today the identity of the company behind a declared name is decided by a heuristic. `B_distinctive` +(ADR-0009) publishes a link when the name is *structurally* distinctive — a digit, a Latin token, or ≥3 +content words. It fails in both directions: it publishes on a bare name coincidence, and it withholds +everything else. Of the 101 links published today, **36 rest on name coincidence alone**. + +The Trade Register answers the identity question directly, over a public endpoint with no authentication: +`GET https://portal.registryagency.bg/CR/api/Deeds/{ЕИК}`. Either the declarant is named in the company's +live deed, or facts they declared match the ones registered against that ЕИК. + +Four things must be on the table before the decision, because each of them changes what is being decided. + +**1. This is an expansion, not a tightening.** The measured surface grows from 101 links to **329 links / +311 people / 239 companies**. „From held / published / withdrawn = 163 / 65 / 101" means **264 links we +deliberately withhold today become named public claims**. At 329 links a 1% error rate is three falsely +named people. The precision bar has to go **up**, not stay flat — which is why the acceptance gate below is +a hand-labelled sample and not the control totals. + +**2. The second rung is a heuristic used to assert — stated precisely.** What the registry evidence +establishes is **the identity of the company**, not that the official owns it: the ownership claim comes +from the official's own filed declaration and is not a heuristic at all. Rung 2 („Документ") confirms that +*the company this declared name refers to is the same legal entity as the winner we matched*, by finding +the declarant's full name inside that company's live fields. A homonym failure therefore does not invent an +ownership claim — it **attaches the official to the wrong company's ЕИК, contracts and money**. That is a +different error from the one the adversarial reading suggests, and it is still a false public claim about a +named person. + +A three-name subset match against free text, in a register that carries no ЕГН, is **not deterministic**. +ADR-0007's decision list rules it out twice: item 2 (published only when the official↔company↔ЕИК +resolution is *deterministic*) and item 3 (heuristics may *withhold* or *triage*, **never assert**). #279 +replaces a weaker heuristic with a stronger one and promotes it to grounds for assertion. That is the +decision this ADR exists to make explicit; it cannot be smuggled in as an implementation detail. + +**3. The shape of the access is the permitted one.** Spec §3.3 permits „bounded per-ЕИК … thousands, not +~900k" and forbids bulk scraping; ADR-0007 decision 1 already anticipated the „per-ЕИК lookup only, later +leg". The crawl is one lookup per candidate ЕИК — on the order of hundreds — with a closed candidate set: +the crawler never follows a link out of a deed. This is that later leg, not the bulk reuse. + +**4. The endpoint rate-limits, and the limiter is a stated preference.** An earlier spike against this same +API triggered **HTTP 429 at roughly 50 cumulative requests ending in a burst**, and the block was then +*sustained* — every subsequent request returned 429, including simple `/Deeds/{eik}` calls that had worked +seconds earlier. There is no `Retry-After` and no `X-RateLimit-*` header, so a client can only avoid +tripping the limit, never pace against a published one; 25 spaced requests were fine, so pacing matters more +than total volume. #279 §3's „~400 companies, about 20 minutes" is therefore feasible only at its stated +1 request / 3 seconds with no burst, and only if the crawl is resumable. The limiter is the operator's only +available way to express a rate preference, and tuning around it empirically is precisely the posture spec +§3.3's „NEVER bulk-scrape" exists to forbid. This is the evidence behind decision 7's crawl hygiene. + +**5. Parts of the source description in #279 were wrong, and were corrected by live probing** (9 sequential +requests, 2026-08-04 — well inside the pacing above, so no 429 was encountered): + +- `?entryDate=` **is** honoured — a bogus-parameter control returned byte-identical output to the + plain request, so the difference is the parameter, not noise. `…/Fields/{ident}/History` **does** return + JSON, not application HTML. #279 §3 states the opposite for both. +- A wrong `subUIC` yields **HTTP 500 `GL_ERROR_L`**, not 404 — so a 500 can never be read as „no history" + or „outside the register". +- `legalForm` 4 = ООД and 10 = ЕООД (as #279 says); the nomenclature endpoint does **not** settle the enum + — `legalForm=4` and `legalForm=10` return a byte-identical catalogue. +- The envelope's `fullName` carries the legal form (`"ПИМК" ООД`) while `CR_F_2_L` is bare. The joint-stock + bar can therefore be derived from the ЗТРРЮЛНЦ-mandated suffix, at zero extra requests. +- Erasure is **structural, not textual**: an erased entity is a `record-container` block carrying the class + `field-text--erased`. Entities inside one field are separated by + `
` blocks delimited by `
`. +- Erased versions are **live in the current deed**: the first company sampled carries a fully-erased + `CR_F_23_L` dated 2013-07-16, which read naïvely becomes „latest ownership entry: 2013-07-16" and feeds + the refutation rule. +- Seats move. The same company's `CR_F_5_L` shows entry dates 2010-11-04 and 2014-01-23. + +The as-of capability that `?entryDate=` unlocks would fix the two weakest rungs, but it **re-baselines every +control number in #279 §10**. It is therefore deliberately out of scope here and becomes a separate change +under a new rules version — which decision 6 makes the only sanctioned way the surface may move. + +## Decision + +### 1. The evidence ladder replaces the publish tiers + +For every resolved link (person × ЕИК) with a fetched deed, **the first matching rung wins**: + +| # | Outcome (data label) | Condition | Effect | +|---|---|---|---| +| 1 | **„Бар: акционерна форма"** — joint-stock bar | the company is a joint-stock form (АД / ЕАД / КДА) | never published, whatever follows | +| 2 | **„Документ"** — documented | the declarant's full name appears in a live `CR_F_7_L` / `CR_F_18_L` / `CR_F_19_L` / `CR_F_23_L` | published; the role (owner vs manager only) is kept for the label | +| 3 | **„Потвърдено"** — confirmed | the normalized declared seat equals the registered seat for that ЕИК, **or** the declarant wrote the ЕИК in the declaration | published | +| 4 | **„Оборена"** — refuted | *own* stake only: the person appears in no live field, **and** the latest entry date across the live ownership fields is strictly before the first declared year | link withdrawn | +| 5 | **„Неизвестна"** — unknown | anything else | held | +| 6 | **„Извън ТР"** — outside the register | the ЕИК is not in the register (ДЗЗД, БУЛСТАТ associations) | held | + +The Bulgarian labels are the persisted vocabulary — they reach the data, the audit and the methodology +page — so they are fixed here rather than translated at each layer. + +Erased versions are skipped everywhere the live state is read — otherwise the date on an erased record can +„certify" a state that is not in force. + +`publish_tier` carries the evidence kind. **`B_distinctive` leaves the publishing path entirely**; +name distinctiveness survives only as an ordering signal for the review queue. Rung 3's seat leg takes the +declared seat **only from declarations by the same person for the same company**: 4.9% of company-name keys +carry more than one distinct declared seat, so a company-only key would let one person's seat confirm +another person's link. + +This supersedes ADR-0009 (the tier ladder) and ADR-0015 (the name census, whose only job was to unblock +`C_hold`; `tr-census.mjs` and its `promote()` are deleted). ADR-0028's holding survives — a declared ЕИК +*is* the identity — but it is now rung 3 rather than a tier of its own, it is subject to rung 1, and its +census-exemption clause is moot. + +### 2. A heuristic may ground an assertion here — the argued exception to ADR-0007 + +Rung 2 asserts on a name match. We accept it, on these grounds and no wider: + +- **It is a full-subset match, patronymic included.** Partial (2-of-3) matching is refused outright: of 301 + matches measured, 46 were two-token only — precisely the homonym risk. A declarant name with fewer than + three tokens can never earn „Документ". +- **It is scoped to a single registry entity.** Tokens are matched only within one `record-container` + block, after erased blocks are dropped and after HTML entities are decoded. Matching against a whole + field's text would combine the given name of one person with the surname of another — the defect that + ships a libel. +- **The company identity behind it passed preventive control.** The candidate ЕИК comes from the + deterministic exact-name resolution of ADR-0008/#226, not from fuzzy matching. +- **It is corroborated, and that is the whole difference.** A bare TR name is spec §4's *weakest* join — + no ЕГН, no birthdate, not even ADR-0026's `(name, ведомство)` grain. The registry-graph spike that + explored this API concluded, for its own use, that no person-derived edge may reach a user-visible + surface at all, and it named the one cheap corroborator that would change that: *an officer who also + appears as a declarant with a declared stake in the same ЕИК, confirmed by an independent source*. That + is exactly what rung 2 is. The register supplies the name-in-this-company fact; the official's own filing + supplies the stake. Neither alone would publish. **The corroboration is what licenses the assertion, so + the rule must never be extended to a name-only join** — matching a person across two companies, or + treating a registry name as an identity in its own right, stays forbidden. +- **The filters that can only withhold are retained, not removed.** `nameGloballyUnique` and + `nameDistinctiveness` stay in the pipeline as an **AND-gate on the weakest rung only** („Потвърдено"). + They cost near-zero recall and preserve ADR-0017's outcome even though its subject — the name-distinctive + tier — is gone. +- **Homoglyphs are not folded.** `company-name-key.ts` deliberately does not fold Cyrillic↔Latin; person + names take the same posture. A Latin letter inside a name means no „Документ", and the occurrence is + counted rather than silently dropped. + +**The residual collision rate is estimated, not measured, and we say so.** Bulgarian names are three-part +by statute (ЗГР чл. 9) and the API renders full triples consistently. A triple of common components — +Георги · Иванов · Петров — is on the order of 10⁻⁵ of ~3.2M men, so roughly 29 people nationally share it; +but the population that can produce a false link is company officers, a ~1% slice, which squares the +probability to ~10⁻⁴ per colliding pair. That is an argument for plausibility, not a measurement, and the +residual concentrates on exactly the common names. It is therefore **not** the basis on which this +publishes — the hand-labelled sample of decision 7 is. Hyphenated surnames count as one token; Latin-script +names are counted the same way but flagged, since a three-token foreign name is not a patronymic triple. + +The tie-breaker for every rung is the repo's own sentence, from +`packages/shared/src/company-name-key.ts`: **„When in doubt the key stays MORE specific (a recall miss is +safe; an over-merge is not)."** + +This **amends ADR-0007 decisions 2 and 3**: a published claim may now rest on a disclosed, bounded, +entity-scoped full-name match against a public register, in addition to deterministic facts. Everything +else in ADR-0007 stands — in particular that every heuristic is disclosed on the methodology page and +labelled in the data, which decision 7 makes a launch condition rather than a follow-up. + +### 3. The joint-stock bar is a union of three independent signals + +The bar exists because the shareholder book is not public (so the claim is unverifiable) and a parcel of +listed shares is not a material ownership conflict (the „11 Trace shares → €88M" trap). It fires if **any** +of the following says joint-stock: + +1. `closelyHeldForm` on the **declared** name (`load.mjs`) — a different, earlier stage; it **stays**, and + is not redundant with the two below; +2. the фирма suffix taken from the deed envelope's `fullName`, through that same tested predicate; +3. the numeric `legalForm` code from the deed. + +**An unknown `legalForm` code withholds and is reported** — it never falls through to publication. The enum +is known only from four observed values and the nomenclature endpoint does not settle it (Context 5), so +fail-open here would be a bar that silently stops barring. `КДА` — which rung 1 explicitly requires barring +— is absent from both `JOINT_STOCK` and `FORM_TOKENS` in `classify.mjs` today and is added. + +**The named open question is ЕАД.** #279 §3 lists 5 = АД, and the spike's catalogue-derived table also +reads 5 as АД/ЕАД jointly — but ООД and ЕООД turned out **not** to share a code (4 and 10), so a separate +ЕАД code is the likelier reading and no observation settles it. A single-owner joint-stock company is +precisely the shape a declarant is most likely to hold and rung 1 most needs to bar, so this is the one +place where a fail-open enum would do real damage. Signal 2 (the ЗТРРЮЛНЦ suffix on `fullName`) covers it +independently of the code, which is why the bar is a union and not a lookup. + +### 4. Termination is reconciled against the live deed — reversing ADR-0021 E11 for own stakes + +ADR-0021 E11 marks an ownership link `withdrawn` when the company is absent from the person's latest +ownership filing. That is an inference from silence, and its commonest cause is a finished mandate rather +than a sale. Before the withdrawal takes effect, every terminated **own** stake is reconciled against the +live deed: a person still named in a live ownership field (`CR_F_18/19/23_L`) has not divested, and the +link surfaces. Family stakes are never reconciled — the relative's name is neither stored nor checked +(ADR-0010 decision 4, ADR-0032 decision 2), so declared termination applies to them directly. + +**Phase 1 uses the reconciliation only to avoid withdrawing the link.** The „и към днешна дата" labels of +#279 §7, and the post-period contracts they license, are **deferred to phase 2 behind an LIA addendum**: +those labels assert a present tense about a named person, on evidence whose freshness is bounded by the +cache refresh cycle, and they change the claim's shape enough to need the balancing assessment updated +first. The derived live status is therefore recomputed on every run and **never sealed**. + +ADR-0021 **E10 is untouched and strengthened** — see decision 7. + +### 5. The deed cache — amends ADR-0010 + +A deed contains third-party personal data: addresses of natural persons, and the names of people who are +not office holders. + +- **Extraction stays inside ADR-0010 decision 3.** `CR_F_5_L` yields only the „Населено място:" segment; + no parser function returns an address. Decision 3 is re-affirmed, not overridden. +- **Storage is what changes, and that is ADR-0010 decision 6.** Raw deeds are cached under git-ignored + `scratch/tr/`, behind the same refuse-to-run guard as the declaration cache, with a **35-day retention** + (one refresh cycle plus slack) and a purge step in the same job. Decision 6's scope extends from „raw + declaration XML … deleted post-spike" to a second source with a stated TTL. +- **The cache index holds no name at all** — only ЕИК, dates, codes and verdicts, plus a body hash instead + of any content excerpt. Names exist solely in the raw JSON files, are read only to produce a boolean, and + never enter a public table, a response, or a log. ЕГН was absent from every payload examined, and the + index additionally refuses any ten-digit run in a text column — sound because an ЕИК is 9 or 13 digits, + never 10. +- **The deed's beneficial-ownership fields are deliberately not used.** A deed also carries `CR_F_550_L` + (действителни собственици, чл. 63 ЗМИП) and the control fields `CR_F_537/538_L`. Using them would make a + beneficial-ownership claim, which ADR-0007 decision 1 parked after CJEU C-37/20. Rung 2 reads the four + fields #279 names and no others; widening it is a new decision, not an improvement. +- **The sealed `matched_fact` is a closed vocabulary** (`seat:`, `role:owner:`, `eik`), + enforced by a pattern check in the audit. Without that, the matched *name* eventually gets stored there + as a convenience, which is exactly what #279 §9 forbids. + +### 6. Monotonicity is a gate, not a store — and #279 §8 is corrected + +#279 §8 requires the evidence seal to be kept „forever" and recomputation to be strictly additive. As +written that is false, and the implementation must not pretend otherwise: + +- §7's labels flip. A person who leaves the register must lose the label *and* the post-period contracts it + licensed. (Phase 1 avoids this by not shipping the labels at all — decision 4.) +- A permanent seal and a live, expiring cache contradict each other. +- A **court-annulled entry** (чл. 29 ЗТРРЮЛНЦ) invalidates the evidence without any rules change. It is + wired to the correction path of ADR-0031 and named as a ground in the suppression runbook. + +Therefore: seals are **re-derived deterministically** on every run — a seal is written for *every* link, +including held ones, so the review queue explains itself — and monotonicity is enforced as a **gate**. The +audit compares against the pre-wipe export and raises a hard finding when a previously published link +disappears under an **unchanged** `rules_version`; under a changed version it degrades to a printed diff. +Removal remains an intentional event: a rules-version bump, or a correction of wrong input. + +### 7. Launch gates + +None of these are follow-ups. + +- **Precision is proven by a hand-labelled sample, not by the control totals.** A human verifies against the + portal: every reconciled link, every refuted link, and random samples of „Документ" and „Потвърдено". + **Pass mark: zero wrong-company and zero wrong-person errors.** The control numbers of #279 §10 are a + reproducibility check — and they are usable as one only after the §5/§10 discrepancy is resolved with the + author (§5's „102 по седалище, 3 по ЕИК" leaves those three ЕИК confirmations in no bucket of the §10 + row, which sums to exactly 568 without them), and re-measured on top of the §12 phantom-row fix, which + changed the corpus — that fix has landed as #281 (87 phantom rows across 15 sets; 256,286 announced − + 87 = 256,199, covered exactly by 255,582 fetched + 617 missing at source). +- **Every anti-false-zero control is a positive control.** „Sofia seat confirmations: 0" is indistinguishable + from a broken normalizer without one; so is a joint-stock bar with no marginal effect over + `closelyHeldForm`, and a matcher that always returns false (ADR-0027's lesson). +- **A partial cache must fail closed.** A missing *or* incomplete registry cache makes the loader throw. An + 80%-restored cache would otherwise yield roughly 80 published links — above the ship floor of 50 — and so + would ship a decimated surface and wipe the rest from production. The ship floor rises accordingly and is + passed explicitly rather than defaulted. +- **Crawl hygiene.** „Outside the register" is permanent only from a documented positive response; 429, + 5xx and timeouts are transient and are never cached as a negative. A 429 stops the run rather than + marking anything, and is never retried — „5 retries with growing backoff" and „429 stops the run" are + consistent only if retries exclude 429. Given Context 4, the crawl is sequential, paced, resumable, and + ends the run on the first 429 rather than backing off into it. **The deed's returned UIC must equal the + requested ЕИК**, or the deed is refused: an ЕИК is TEXT everywhere because public bodies' identifiers are + exactly the `000…` shape that loses its leading zeros to a numeric round-trip, and the failure mode of + losing them is fetching, and then publishing against, a different company's deed. +- **The methodology page carries the rule verbatim** (ADR-0021 E10): the ladder, the ≥3-token requirement, + the entity-boundary rule, the disclosure that identity rests on a name match **without ЕГН**, the homonym + and seat caveats, the lookup date, the refresh cadence and the retention. E10's existing promise that + held links „се показват едва след като регистърът стане достъпен" is finally kept by this change. +- **The PII rails are asserted, not assumed** — over the shipped dump and over the loader's own output. + +## Consequences + +- The surface roughly triples, and 264 links that are withheld today become named public claims. The + failure mode of the whole change is a **wrong-company** attribution, not a wrong publication decision. +- `tr-census.mjs` and the open-data census pipeline of ADR-0015 are removed. Its premise — that a name + proven nationally unique identifies the company — is the premise this ADR abandons. +- The pipeline gains a hard dependency on an external registry. Every path that lacked one before now has a + fail-closed branch: no cache, partial cache, unknown legal form, rate limit. +- We accept a slower, more fragile refresh in exchange for evidence. Decisions stay a pure, zero-network + function of declarations, cached deeds and contracts, but they no longer run on the 6-hourly ETL cycle + (#279 §9 assumes they do; they do not — the loader needs `node:sqlite`, the declaration corpus and the + full winner set, none of which exist inside a Worker). Cadence becomes two scheduled workflows: decisions + daily, registry lookups monthly. +- Two capabilities are deliberately left on the table and become separate changes under new rules versions: + as-of evidence via `?entryDate=`, and the §7 „и към днешна дата" labels with their post-period contracts. +- The seat rung is structurally capped and we accept the cap: a declared seat is captured **only** from the + ООД/ЕООД holdings table of an *asset* declaration (`parse.mjs`), so it can never rescue a link declared + solely in an interests declaration. This is probably consistent with the measured 102 — rung 1 bars the + joint-stock cases anyway — but it is confirmed against the corpus before launch, not assumed. + +### Measured outcome — recorded on completion + +This ADR is accepted on the design. Exactly one amendment is permitted afterwards: this subsection is +filled in with the measured result, and nothing else in the file is rewritten (repo convention — an +accepted ADR is superseded, not edited). To be recorded: + +- the realized ladder split against the #279 §10 control row, with each number naming the SQL that produced + it, and the 54-link gap between identified (281 + 102) and surfaced (329) attributed bucket by bucket; +- the true candidate-ЕИК count (#279's „~400" is an assumption; the figure is `COUNT(DISTINCT eik)` over + all links, not just published ones); +- the hand-labelled sample result; +- the marginal effect of the registry joint-stock bar over `closelyHeldForm`, and the count of unknown + `legalForm` codes encountered. diff --git a/docs/adr/README.md b/docs/adr/README.md index 0bd84ab6..ecaa107f 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -16,15 +16,15 @@ | [0006](0006-eop-wins-dedup.md) | Dedup на два източника: EOP печели по `contract_number` | Прието | | [0007](0007-scope-and-certainty-bar.md) | Свързани лица: обхват и праг на сигурност (само 100% детерминистични съвпадения) | Прието | | [0008](0008-deterministic-name-to-eik-resolution.md) | Детерминистично разрешаване име→ЕИК (авто-публикуване; ръчна опашка само при двусмислие) | Прието | -| [0009](0009-name-uniqueness-guard-and-publish-tiers.md) | Пазач за уникалност на името + нива на публикуване (A_seat / B_distinctive / C_hold) | Прието | +| [0009](0009-name-uniqueness-guard-and-publish-tiers.md) | Пазач за уникалност на името + нива на публикуване (A_seat / B_distinctive / C_hold) | Заменено от [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) | | [0010](0010-pii-posture.md) | Позиция за лични данни: без ЕГН/адреси; третите лица — само за вътрешна проверка | Прието | | [0011](0011-host-scoped-tls-pinning.md) | TLS pinning само за хоста (счупена верига на register.cacbg.bg), не глобален байпас | Прието | | [0012](0012-crawler-and-persistence-architecture.md) | Архитектура на обхождането и съхранението (resumable; кеш по xml_file + ControlHash) | Прието | | [0013](0013-two-declaration-templates.md) | Два шаблона декларации (имущество + интереси) в един парсер | Прието | | [0014](0014-match-output-layers-and-interpretation.md) | Слоеве на съвпадението и тълкуване (собственост/контрол, времеви, собствена институция) | Прието | -| [0015](0015-tr-name-uniqueness-census.md) | Преброяване за уникалност на имена от ТР — промотира глобално уникални tier-C връзки | Прието | +| [0015](0015-tr-name-uniqueness-census.md) | Преброяване за уникалност на имена от ТР — промотира глобално уникални tier-C връзки | Заменено от [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) | | [0016](0016-free-text-entity-resolution.md) | Разрешаване на субекти от свободен текст (деклариран ЕИК + извличане от проза) | Прието | -| [0017](0017-name-collision-tier-gate.md) | Гейт срещу колизия на имена извън отличаващото ниво | Прието | +| [0017](0017-name-collision-tier-gate.md) | Гейт срещу колизия на имена извън отличаващото ниво | Заменено от [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) | | [0018](0018-folder-discovery-and-republication-dedup.md) | Откриване на папки от индекса + dedup на препубликувани декларации (ControlHash) | Прието | | [0019](0019-private-interest-vs-ex-officio-classification.md) | Разделяне на частен финансов интерес от служебни борд-роли (multi-declarant tell) | Прието | | [0020](0020-conflict-explorer-surface-posture.md) | Повърхност на експлорера — само interest_links, noindex до одобрение, произход на всеки ред | Прието | @@ -40,5 +40,6 @@ | [0030](0030-family-ownership-withheld-nameless-aggregate.md) | Дял на свързано лице не влиза в поименната повърхност (v1): събира се и се одитира, но се отчита само като безименен сбор (GDPR C-37/20; заменя ADR-0023) | Заменено от [ADR-0032](0032-family-ownership-published-under-public-interest.md) | | [0031](0031-suppressions-version-controlled-fingerprinted.md) | Свалянията на връзки са версиониран списък с HMAC-отпечатък (salt = CI secret), прилаган при зареждане — не служебна таблица; не изтича сигнала „кой е свален" към прод | Прието | | [0032](0032-family-ownership-published-under-public-interest.md) | Дял на свързано лице се публикува на поименната повърхност наравно със собствения — на основание надделяващ обществен интерес (ЗДОИ чл. 41и; C-184/20 забранява само поименните данни); близкият никога не се назовава, връзката не се твърди; заменя ADR-0030 | Прието | +| [0033](0033-registry-evidence-replaces-name-distinctiveness.md) | Връзка се публикува само срещу проверим факт от Търговския регистър — доказателственият ред заменя нивата по отличителност на името; евристиката вече обосновава твърдение (изрично изключение от ADR-0007), барът за акционерни форми е обединение от три сигнала, монотонността е гейт, не хранилище; заменя ADR-0009/0015/0017 | Прието (дизайн) | Свързан проектен документ: [spec/related-persons-foundation.md](../spec/related-persons-foundation.md). From 4fde3778d08d9f4c268243a876a05b3751170db4 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 12:20:20 +0300 Subject: [PATCH 02/69] docs: ADR-0023 carries its superseded status The index has marked it superseded by ADR-0030 since PR #226, but the file header still read "Accepted". A header/index divergence, spotted while synchronising the index for ADR-0033. --- docs/adr/0023-anonymized-family-ownership-surface.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/adr/0023-anonymized-family-ownership-surface.md b/docs/adr/0023-anonymized-family-ownership-surface.md index 3ead3824..d5dcea50 100644 --- a/docs/adr/0023-anonymized-family-ownership-surface.md +++ b/docs/adr/0023-anonymized-family-ownership-surface.md @@ -1,8 +1,9 @@ # ADR-0023: Anonymized close-relative (family) ownership surface -- Status: Accepted +- Status: Superseded by [ADR-0030](0030-family-ownership-withheld-nameless-aggregate.md) - Date: 2026-07-07 - Deciders: lb, Claude +- Superseded by: [ADR-0030](0030-family-ownership-withheld-nameless-aggregate.md) - Related: [ADR-0007](0007-scope-and-certainty-bar.md), [ADR-0010](0010-pii-posture.md), [ADR-0019](0019-private-interest-vs-ex-officio-classification.md), [ADR-0022](0022-public-surface-private-ownership-only.md); `scripts/cacbg/parse.mjs`, `scripts/cacbg/load.mjs`, `packages/db/src/queries/related-persons.ts`, `apps/web/app/routes/conflict*.tsx` ## Context From ea00781d3622fa8889f658cc87bf2ca597c4e527 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 16:54:28 +0300 Subject: [PATCH 03/69] test(seed): conflict-of-interest smoke fixture for local development MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /conflicts rendered an empty surface locally — seed.sql carried only authorities, tenders and bidders, so the свързани-лица routes had nothing to show and the feature could not be verified in a browser at all. Adds two invented officials, three invented companies and the rows the read gate actually requires (packages/db/src/queries/related-persons.ts): published status with a surfaced ownership class, a live contract for the ЕИК, and signed_at inside the declared window so the „в периода" split is exercised. One link per outcome — a self stake, a family stake, and a joint-stock link left held — so a change to the publishing rule shows up as a before/after rather than as an empty page either way. Every person and company is invented; no real declarant, no real winner. The ЕИК pass the real 9-digit checksum (normalize-raw.sql's eik_valid), so the fixture cannot teach a wrong ЕИК shape. --- scripts/seed.sql | 79 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/scripts/seed.sql b/scripts/seed.sql index 2ff25add..9da64854 100644 --- a/scripts/seed.sql +++ b/scripts/seed.sql @@ -16,3 +16,82 @@ INSERT OR IGNORE INTO bidders (id, name, bulstat) VALUES ('bidder-a', 'Алфа ЕООД', '111111111'), ('bidder-b', 'Бета АД', '222222222'), ('bidder-c', 'Гама ООД', '333333333'); + +-- ── свързани лица (conflict-of-interest) smoke fixture ──────────────────────────────────────────── +-- Without these the /conflicts routes render an empty surface locally, so the whole feature is +-- unverifiable in a browser. Every person and company below is INVENTED — no real declarant, no real +-- winner. The ЕИК pass the real 9-digit checksum (scripts/normalize-raw.sql's eik_valid), so the +-- fixture cannot teach a wrong ЕИК shape to anyone reading it. +-- +-- The rows are shaped by what the read gate actually requires (packages/db/src/queries/related-persons.ts): +-- • status='published' AND interest_class IN (private_ownership, family_ownership) — SURFACED_OWNERSHIP +-- • a LIVE contract for that ЕИК — the read-time zero-contract gate +-- • signed_at inside [first_declared_year, last_declared_year] — IN_WINDOW ⇒ the „в периода" chip +-- • no published SELF stake on the same (official, ЕИК) for a family row — NOT_REDUNDANT_FAMILY +-- One link per outcome, including the non-surfaced ones, so a change to the publishing rule is +-- visible as a before/after rather than as an empty page either way. + +INSERT OR IGNORE INTO authorities (id, name, bulstat, region) VALUES + ('auth-plovdiv', 'Община Пример', '131223340', 'Пловдив'); + +INSERT OR IGNORE INTO tenders + (id, source_id, title, authority_id, cpv_code, estimated_value, currency, procedure_type, status, published_at, deadline_at) +VALUES + ('t-conflict-01', 'AOP-2022-0101', 'Строителен надзор на общински обекти', 'auth-plovdiv', '71520000', 900000, 'BGN', 'открита процедура', 'closed', '2022-01-10', '2022-02-10'), + ('t-conflict-02', 'AOP-2022-0102', 'Консултантски услуги по проект', 'auth-sofia', '79400000', 400000, 'BGN', 'открита процедура', 'closed', '2022-04-05', '2022-05-05'), + ('t-conflict-03', 'AOP-2022-0103', 'Доставка на офис оборудване', 'auth-mrrb', '30190000', 250000, 'BGN', 'открита процедура', 'closed', '2022-06-01', '2022-07-01'); + +-- eik_valid=1 so these behave like resolved winners; settlement feeds the seat-confirmation rung. +INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, legal_form, settlement) VALUES + ('eik:201122335', 'АЛФА СТРОЙ ООД', '201122335', '201122335', 1, 'ООД', 'Пловдив'), + ('eik:203445566', 'БЕТА КОНСУЛТ ЕООД', '203445566', '203445566', 1, 'ЕООД', 'София'), + ('eik:204556676', 'ГАМА ИНВЕСТ АД', '204556676', '204556676', 1, 'АД', 'Варна'); + +INSERT OR IGNORE INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES + ('c-conflict-01', 't-conflict-01', 'eik:201122335', 840000, 'BGN', '2022-03-14', 'Д-2022-101', 429000), + ('c-conflict-02', 't-conflict-02', 'eik:203445566', 372000, 'BGN', '2022-06-20', 'Д-2022-102', 190000), + ('c-conflict-03', 't-conflict-03', 'eik:204556676', 236000, 'BGN', '2022-08-02', 'Д-2022-103', 120000); + +-- person id = 'person:' || key(name) || '|' || key(institution) — (name, institution), never a bare +-- name (ADR-0026), so two namesakes at different bodies stay distinct. +INSERT OR IGNORE INTO persons (id, name) VALUES + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'Иван Петров Тестов'), + ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ', 'Мария Георгиева Образцова'); + +INSERT OR IGNORE INTO declarations + (id, person_id, xml_file, control_hash, folder_year, declared_year, template, category, institution, position, source_url) +VALUES + ('decl:2023:demo-ivan.xml', 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'demo-ivan.xml', 'demo-hash-ivan', '2023', '2023', 'assets', 'Местна власт', 'Община Пример', 'Кмет', 'https://register.cacbg.bg/2023/demo-ivan.xml'), + ('decl:2023:demo-maria.xml', 'person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ', 'demo-maria.xml', 'demo-hash-maria', '2023', '2023', 'assets', 'Местна власт', 'Община София', 'Директор дирекция', 'https://register.cacbg.bg/2023/demo-maria.xml'); + +INSERT OR IGNORE INTO declared_interests (id, declaration_id, entity_raw, entity_key, kind, detail, timing, seat) VALUES + ('di:decl:2023:demo-ivan.xml:1', 'decl:2023:demo-ivan.xml', 'АЛФА СТРОЙ ООД', 'АЛФА СТРОЙ ООД', 'shares', '50%', 'annual', 'Пловдив'), + ('di:decl:2023:demo-ivan.xml:2', 'decl:2023:demo-ivan.xml', 'ГАМА ИНВЕСТ АД', 'ГАМА ИНВЕСТ АД', 'shares', '0,5%', 'annual', 'Варна'), + ('di:decl:2023:demo-maria.xml:1', 'decl:2023:demo-maria.xml', 'БЕТА КОНСУЛТ ЕООД', 'БЕТА КОНСУЛТ ЕООД', 'shares', '100%', 'annual', 'София'); + +-- Three links, three outcomes: +-- published private_ownership — the official's own stake, own_institution='exact' (the strongest signal) +-- published family_ownership — a relative's declared stake; the relative is never named (ADR-0032) +-- held (АД, C_hold) — a joint-stock parcel: collected, never surfaced (ADR-0022 materiality) +INSERT OR IGNORE INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, + relation, interest_class, contemporaneous, own_institution, evidence_count, + first_declared_year, last_declared_year, contract_count, contract_value_eur, + first_contract_year, last_contract_year, status) +VALUES + ('il:person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', + 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'eik:201122335', '201122335', 'АЛФА СТРОЙ ООД', + 'exact_name_key', 'seed-demo', 'B_distinctive', 'owns', 'private_ownership', 1, 'exact', 1, + '2021', '2023', 1, 429000, '2022', '2022', 'published'), + ('il:person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', + 'person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ', 'eik:203445566', '203445566', 'БЕТА КОНСУЛТ ЕООД', + 'exact_name_key', 'seed-demo', 'A_seat', 'related', 'family_ownership', 1, 'exact', 1, + '2021', '2023', 1, 190000, '2022', '2022', 'published'), + ('il:person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|204556676', 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|204556676', + 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'eik:204556676', '204556676', 'ГАМА ИНВЕСТ АД', + 'exact_name_key', 'seed-demo', 'C_hold', 'owns', 'management_role', 0, 'none', 1, + '2021', '2023', 1, 120000, '2022', '2022', 'held'); + +INSERT OR IGNORE INTO interest_link_authorities (link_key, authority_id, authority_name, contract_count, value_eur, own) VALUES + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'auth-plovdiv', 'Община Пример', 1, 429000, 'exact'), + ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'auth-sofia', 'Община София', 1, 190000, 'exact'); From b36403177c6542af9286cbf888dae998283f76fc Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 17:21:09 +0300 Subject: [PATCH 04/69] =?UTF-8?q?feat(tr):=20registry=20client,=20deed=20c?= =?UTF-8?q?ache=20and=20the=20=D0=95=D0=98=D0=9A=20checksum=20as=20a=20Nod?= =?UTF-8?q?e=20capability?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundations for the Trade Register evidence leg (#279, ADR-0033). No crawling and no publishing decision yet — this is the transport, the cache and the identifier rule, each with the refusals that make the later steps safe. scripts/tr/eik.mjs — the ЕИК checksum, which until now was callable only from SQL (normalize-raw.sql's eik_valid). The crawler has to decide in Node whether a code is worth a lookup. A test lifts the CASE expression straight out of the .sql file and runs both implementations over ~120 values through node:sqlite, so the twin cannot drift from the rule it mirrors. scripts/tr/client.mjs — 429 is never retried. The register's block is sustained (an earlier spike saw 429 at ~50 requests and then 429 to everything, with no Retry-After and no quota header), so it is an instruction to stop, not a transient. 5xx and network faults retry with growing backoff; a 429 arriving mid-retry aborts rather than letting a later 200 mask the block. A foreign host is refused before any packet is sent. Two transport decisions worth stating: this leg does NOT reuse cacbg/tls.mjs — that module pins register.cacbg.bg's leaf because that host serves a broken chain (ADR-0011), whereas this host verifies against system roots, which is stronger. And it uses node:https, not fetch: measured live, the identical request returns 500/empty via undici and 200 with the full 34,398-byte deed via node:https. scripts/tr/cache.mjs — resumability plus the PII rail. The index holds no name at all: ЕИК, dates, codes, verdicts and a body hash, never an excerpt. Any value carrying a ten-digit run is refused outright — the ЕГН shape, and sound as a check precisely because an ЕИК is 9 or 13 digits, never 10. scripts/tr/*.test.mjs matched NO CI job before this commit; the workflow globbed only scripts/ and scripts/cacbg/. Fixed here rather than later, or every test above would have been decorative. cacbg/guard.mjs's assertScratchIgnored takes the subdirectory instead of being copied — a duplicated safety rail drifts from its original. Existing callers are unchanged. --- .github/workflows/scripts-test.yml | 13 +- scripts/cacbg/guard.mjs | 11 +- scripts/tr/cache.mjs | 187 +++++++++++++++++++++++++++++ scripts/tr/cache.test.mjs | 129 ++++++++++++++++++++ scripts/tr/client.mjs | 129 ++++++++++++++++++++ scripts/tr/client.test.mjs | 152 +++++++++++++++++++++++ scripts/tr/eik.mjs | 61 ++++++++++ scripts/tr/eik.test.mjs | 135 +++++++++++++++++++++ scripts/tr/paths.mjs | 54 +++++++++ scripts/tr/paths.test.mjs | 64 ++++++++++ 10 files changed, 928 insertions(+), 7 deletions(-) create mode 100644 scripts/tr/cache.mjs create mode 100644 scripts/tr/cache.test.mjs create mode 100644 scripts/tr/client.mjs create mode 100644 scripts/tr/client.test.mjs create mode 100644 scripts/tr/eik.mjs create mode 100644 scripts/tr/eik.test.mjs create mode 100644 scripts/tr/paths.mjs create mode 100644 scripts/tr/paths.test.mjs diff --git a/.github/workflows/scripts-test.yml b/.github/workflows/scripts-test.yml index 7ee3ef4c..d3112c48 100644 --- a/.github/workflows/scripts-test.yml +++ b/.github/workflows/scripts-test.yml @@ -28,10 +28,11 @@ jobs: # so no `pnpm install` is required. - run: node --test scripts/*.test.mjs - # свързани-лица CACBG pipeline tests (parse/classify/load/audit/tr-census/extract). These use - # node:sqlite (DatabaseSync) and import the shared companyNameKey .ts via the register-ts resolve - # hook, so they need Node 24 (node:sqlite + native TS type-stripping, no experimental flag) rather - # than the Node 22 above. The libel-critical resolution logic gates merges here. + # свързани-лица pipeline tests — the CACBG leg (parse/classify/load/audit/tr-census/extract) and the + # Търговски регистър leg (scripts/tr: ЕИК checksum, HTTP client, deed cache). These use node:sqlite + # (DatabaseSync) and import the shared companyNameKey .ts via the register-ts resolve hook, so they + # need Node 24 (node:sqlite + native TS type-stripping, no experimental flag) rather than the Node 22 + # above. The libel-critical resolution logic gates merges here. cacbg: runs-on: ubuntu-latest timeout-minutes: 10 @@ -47,4 +48,6 @@ jobs: # parse.mjs imports fast-xml-parser (a workspace dep), so this job needs node_modules — unlike # the plain-node scripts above. Install before the scraper tests run. - run: pnpm install --frozen-lockfile - - run: node --import ./scripts/cacbg/register-ts.mjs --test scripts/cacbg/*.test.mjs + # Both legs, or the glob silently excludes a whole module: scripts/tr/*.test.mjs matched NO job + # until this line existed, so every test in it would have been decorative. + - run: node --import ./scripts/cacbg/register-ts.mjs --test scripts/cacbg/*.test.mjs scripts/tr/*.test.mjs diff --git a/scripts/cacbg/guard.mjs b/scripts/cacbg/guard.mjs index 97a326bd..9502ac96 100644 --- a/scripts/cacbg/guard.mjs +++ b/scripts/cacbg/guard.mjs @@ -9,8 +9,15 @@ import { fileURLToPath } from 'node:url'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); export const SCRATCH = path.join(ROOT, 'scratch', 'cacbg'); -export function assertScratchIgnored() { - const probe = path.join('scratch', 'cacbg', '.probe'); +/** + * Assert that a `scratch/` tree is git-ignored, before anything writes PII into it. + * Parameterised rather than copied: the Trade Register leg needs the identical rail for its deed + * cache (owner names, company addresses — ADR-0033 decision 5), and a second copy of a safety rail + * drifts from the original. Existing no-argument callers are unaffected. + * @param {string} [subdir] directory under scratch/ to probe + */ +export function assertScratchIgnored(subdir = 'cacbg') { + const probe = path.join('scratch', subdir, '.probe'); try { execFileSync('git', ['check-ignore', '-q', probe], { cwd: ROOT }); } catch { diff --git a/scripts/tr/cache.mjs b/scripts/tr/cache.mjs new file mode 100644 index 00000000..673c6b47 --- /dev/null +++ b/scripts/tr/cache.mjs @@ -0,0 +1,187 @@ +// The deed cache — resumability, and the PII rail (issue #279, ADR-0033 decision 5). +// +// A registry deed contains third-party personal data: the names of owners and managers who hold no +// public office, and the company's street address. Two rails follow from that, and both live here: +// +// 1. The INDEX stores no name at all — ЕИК, dates, codes, verdicts, and a hash of the body. Names +// exist only in the raw JSON under git-ignored scratch/, are read only to produce a boolean, and +// never enter a public table, a response or a log. A hash rather than an excerpt, because an +// excerpt of a deed IS third-party personal data. +// 2. Nothing may carry a ten-digit run. That is the ЕГН shape, and the check is sound precisely +// because an ЕИК is 9 or 13 digits — never 10 — so it cannot reject a legitimate identifier. +// ЕГН was absent from every payload examined; this is the rail for the day one leaks. +// +// Resumability is the other job: the register throttles hard and a 429 ends the run (client.mjs), so a +// crawl must be able to pick up exactly where it stopped without re-requesting what it already has. + +import { DatabaseSync } from 'node:sqlite'; +import fs from 'node:fs'; +import path from 'node:path'; +import { safeEik } from './paths.mjs'; + +const SCHEMA = ` +CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, + status TEXT NOT NULL, -- fetched | outside_tr + http_status INTEGER, + fetched_at TEXT NOT NULL, + raw_path TEXT, -- relative to the raw dir; the ONLY place names live + body_sha256 TEXT, -- integrity + change detection, never an excerpt + legal_form_code INTEGER, + legal_form_verdict TEXT, -- closely_held | joint_stock | unknown (unknown WITHHOLDS) + seat_normalized TEXT, -- settlement only; never a street address (ADR-0010 item 3) + seat_entry_date TEXT, + latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, + outside_reason TEXT +); +CREATE INDEX IF NOT EXISTS idx_deeds_status ON deeds(status); +CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); +`; + +/** Open (creating if absent) the cache at `file`. Idempotent — never wipes an existing cache. */ +export function openCache(file) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const db = new DatabaseSync(file); + db.exec('PRAGMA journal_mode = WAL'); + db.exec(SCHEMA); + return db; +} + +// ── the ЕГН rail ────────────────────────────────────────────────────────────── +const TEN_DIGITS = /\d{10}/; +/** + * Refuse any text destined for the index that carries a ten-digit run. + * An ЕИК is 9 or 13 digits, so a 10-digit run is never a legitimate identifier here — it is an ЕГН + * shape, and storing one would breach ADR-0010 decision 2 outright. + */ +function assertNoEgnShape(value, field) { + if (value != null && TEN_DIGITS.test(String(value))) { + throw new Error( + `REFUSE TO STORE: ${field} carries a ten-digit run (ЕГН shape) — the index holds no personal data`, + ); + } +} + +const TEXT_FIELDS = ['rawPath', 'legalFormVerdict', 'seatNormalized', 'outsideReason']; + +/** + * Record a fetched deed. Replaces on re-fetch so a refresh never duplicates a row. + * @param {import('node:sqlite').DatabaseSync} db + * @param {object} d + */ +export function upsertDeed(db, d) { + const eik = safeEik(d.eik); + for (const f of TEXT_FIELDS) assertNoEgnShape(d[f], f); + db.prepare( + `INSERT INTO deeds (eik, status, http_status, fetched_at, raw_path, body_sha256, + legal_form_code, legal_form_verdict, seat_normalized, seat_entry_date, + latest_own_entry_date, attempts, outside_reason) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(eik) DO UPDATE SET + status=excluded.status, http_status=excluded.http_status, fetched_at=excluded.fetched_at, + raw_path=excluded.raw_path, body_sha256=excluded.body_sha256, + legal_form_code=excluded.legal_form_code, legal_form_verdict=excluded.legal_form_verdict, + seat_normalized=excluded.seat_normalized, seat_entry_date=excluded.seat_entry_date, + latest_own_entry_date=excluded.latest_own_entry_date, + attempts=deeds.attempts + 1, outside_reason=excluded.outside_reason`, + ).run( + eik, + d.status ?? 'fetched', + d.httpStatus ?? null, + d.fetchedAt, + d.rawPath ?? null, + d.bodySha256 ?? null, + d.legalFormCode ?? null, + d.legalFormVerdict ?? null, + d.seatNormalized ?? null, + d.seatEntryDate ?? null, + d.latestOwnEntryDate ?? null, + d.attempts ?? 1, + d.outsideReason ?? null, + ); +} + +/** + * Mark an ЕИК as not present in the Trade Register (ДЗЗД, БУЛСТАT associations). + * + * PERMANENT BY INTENT, so the caller must only reach here on a DOCUMENTED positive response. A 429, + * a 5xx or a timeout is transient and must never be cached as a negative — that is how a temporary + * wall becomes permanent data (R6). + */ +export function markOutsideTr(db, eik, reason, now = new Date()) { + assertNoEgnShape(reason, 'outsideReason'); + upsertDeed(db, { + eik, + status: 'outside_tr', + fetchedAt: now.toISOString(), + outsideReason: reason, + }); +} + +/** One row by ЕИК, camelCased, or null. */ +export function readDeed(db, eik) { + const r = db.prepare('SELECT * FROM deeds WHERE eik = ?').get(safeEik(eik)); + if (!r) return null; + return { + eik: r.eik, + status: r.status, + httpStatus: r.http_status, + fetchedAt: r.fetched_at, + rawPath: r.raw_path, + bodySha256: r.body_sha256, + legalFormCode: r.legal_form_code, + legalFormVerdict: r.legal_form_verdict, + seatNormalized: r.seat_normalized, + seatEntryDate: r.seat_entry_date, + latestOwnEntryDate: r.latest_own_entry_date, + attempts: r.attempts, + outsideReason: r.outside_reason, + }; +} + +/** + * Which of `wanted` still need a request — the resumability primitive. + * A cached row past `maxAgeDays` becomes pending again (the 35-day retention of ADR-0033 decision 5); + * pass no `maxAgeDays` to treat any cached row as current. + */ +export function pendingEiks(db, wanted, { maxAgeDays = null, now = new Date() } = {}) { + const cutoff = maxAgeDays == null ? null : now.getTime() - maxAgeDays * 86_400_000; + const out = []; + const stmt = db.prepare('SELECT fetched_at FROM deeds WHERE eik = ?'); + for (const raw of wanted) { + const row = stmt.get(safeEik(raw)); + if (!row) { + out.push(String(raw)); + continue; + } + if (cutoff != null && Date.parse(row.fetched_at) < cutoff) out.push(String(raw)); + } + return out; +} + +/** + * How much of `wanted` the cache actually covers — the input to the fail-closed load gate. + * `outside_tr` counts as COVERED: it is a known, resolved outcome, not a gap. A partial cache must + * make the loader throw rather than publish a decimated surface (ADR-0033 decision 7). + */ +export function coverage(db, wanted) { + let fetched = 0, + outsideTr = 0; + const stmt = db.prepare('SELECT status FROM deeds WHERE eik = ?'); + for (const raw of wanted) { + const row = stmt.get(safeEik(raw)); + if (!row) continue; + if (row.status === 'outside_tr') outsideTr++; + else fetched++; + } + const covered = fetched + outsideTr; + return { + wanted: wanted.length, + fetched, + outsideTr, + covered, + missing: wanted.length - covered, + ratio: wanted.length === 0 ? 1 : covered / wanted.length, + }; +} diff --git a/scripts/tr/cache.test.mjs b/scripts/tr/cache.test.mjs new file mode 100644 index 00000000..e757593d --- /dev/null +++ b/scripts/tr/cache.test.mjs @@ -0,0 +1,129 @@ +// node:test — the deed cache. Its job is to make the crawl resumable and to hold the PII rail. +// +// The rail (ADR-0033 decision 5): the INDEX stores no name at all — only ЕИК, dates, codes, verdicts +// and a body hash. Names exist solely in the raw JSON under git-ignored scratch/, are read only to +// produce a boolean, and never enter a public table, a response or a log. The ten-digit refusal below +// is the ЕГН guard, and it is sound precisely because an ЕИК is 9 or 13 digits — never 10. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { openCache, upsertDeed, markOutsideTr, pendingEiks, readDeed, coverage } from './cache.mjs'; + +function tmpDb() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-cache-')); + return { dir, file: path.join(dir, 'tr-cache.sqlite') }; +} +const withCache = (fn) => { + const { dir, file } = tmpDb(); + const db = openCache(file); + try { + return fn(db); + } finally { + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } +}; + +const deed = (over = {}) => ({ + eik: '115536179', + httpStatus: 200, + fetchedAt: '2026-08-05T10:00:00Z', + rawPath: 'deeds/115536179.json', + bodySha256: 'a'.repeat(64), + legalFormCode: 4, + legalFormVerdict: 'closely_held', + seatNormalized: 'ПЛОВДИВ', + seatEntryDate: '2014-01-23', + latestOwnEntryDate: '2013-07-16', + ...over, +}); + +test('openCache is idempotent — re-opening an existing cache preserves rows', () => { + const { dir, file } = tmpDb(); + let db = openCache(file); + upsertDeed(db, deed()); + db.close(); + db = openCache(file); // must not wipe + assert.equal(readDeed(db, '115536179')?.eik, '115536179'); + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); +}); + +test('upsertDeed replaces on re-fetch rather than duplicating', () => + withCache((db) => { + upsertDeed(db, deed()); + upsertDeed(db, deed({ seatNormalized: 'СОФИЯ', fetchedAt: '2026-09-01T10:00:00Z' })); + assert.equal(coverage(db, ['115536179']).fetched, 1); + assert.equal(readDeed(db, '115536179').seatNormalized, 'СОФИЯ'); + })); + +test('pendingEiks returns only what is not yet cached — this is what makes a run resumable', () => + withCache((db) => { + upsertDeed(db, deed({ eik: '115536179' })); + markOutsideTr(db, '204556676', 'BULSTAT association'); + const want = ['115536179', '204556676', '201122335', '203445566']; + assert.deepEqual(pendingEiks(db, want).sort(), ['201122335', '203445566']); + })); + +test('a stale deed becomes pending again past the TTL, a fresh one does not', () => + withCache((db) => { + upsertDeed(db, deed({ fetchedAt: '2026-01-01T00:00:00Z' })); // long past + upsertDeed(db, deed({ eik: '201122335', fetchedAt: '2026-08-05T00:00:00Z' })); + const now = new Date('2026-08-05T12:00:00Z'); + assert.deepEqual(pendingEiks(db, ['115536179', '201122335'], { maxAgeDays: 35, now }), [ + '115536179', + ]); + })); + +test('coverage reports the fraction cached — the input to the fail-closed load gate', () => + withCache((db) => { + upsertDeed(db, deed({ eik: '115536179' })); + upsertDeed(db, deed({ eik: '201122335' })); + markOutsideTr(db, '204556676', 'ДЗЗД'); + const c = coverage(db, ['115536179', '201122335', '204556676', '203445566']); + assert.equal(c.wanted, 4); + assert.equal(c.fetched, 2); + assert.equal(c.outsideTr, 1); + assert.equal(c.missing, 1); + // „outside ТР" is a RESOLVED outcome, not a gap: it is known and permanent, so it counts as covered. + assert.equal(c.covered, 3); + })); + +// ── the PII rail ────────────────────────────────────────────────────────────── +test('the index REFUSES a value carrying a ten-digit run (the ЕГН guard)', () => + withCache((db) => { + // Sound because an ЕИК is 9 or 13 digits, never 10 — so this can never reject a legitimate code. + assert.throws( + () => upsertDeed(db, deed({ seatNormalized: 'СОФИЯ 8001014567' })), + /ten-digit|ЕГН/i, + ); + assert.throws(() => markOutsideTr(db, '204556676', 'подадено от 8001014567'), /ten-digit|ЕГН/i); + })); + +test('valid 9- and 13-digit codes are NOT caught by the ЕГН guard', () => + withCache((db) => { + assert.doesNotThrow(() => upsertDeed(db, deed({ outsideReason: null }))); + assert.doesNotThrow(() => markOutsideTr(db, '1155361790001', 'клон')); + })); + +test('the schema exposes no column that could hold a person name', () => + withCache((db) => { + const cols = db + .prepare(`SELECT name FROM pragma_table_info('deeds')`) + .all() + .map((r) => r.name); + for (const forbidden of ['name', 'person', 'owner', 'manager', 'holder', 'full_name']) + assert.ok(!cols.includes(forbidden), `deeds.${forbidden} must not exist (PII rail)`); + // A hash, never an excerpt — an excerpt of a deed is third-party personal data. + assert.ok(cols.includes('body_sha256')); + })); + +test('markOutsideTr is permanent-by-intent and records WHY', () => + withCache((db) => { + markOutsideTr(db, '204556676', 'ДЗЗД — BULSTAT, not TR'); + const row = readDeed(db, '204556676'); + assert.equal(row.status, 'outside_tr'); + assert.match(row.outsideReason, /ДЗЗД/); + })); diff --git a/scripts/tr/client.mjs b/scripts/tr/client.mjs new file mode 100644 index 00000000..23d3769e --- /dev/null +++ b/scripts/tr/client.mjs @@ -0,0 +1,129 @@ +// HTTP client for the Търговски регистър public API (issue #279, ADR-0033). +// +// TLS: PLAIN HTTPS, verified against the system roots at Node's secure default. Do NOT reach for +// scripts/cacbg/tls.mjs here. That module exists because register.cacbg.bg serves an incomplete chain, +// so we pin ITS leaf key (ADR-0011) — `getPinned` hard-refuses every other host by design. This host +// chains correctly, so ordinary verification is available and is strictly stronger than a pin we would +// have to hand-maintain. Copying the pinning across "for consistency" would weaken this leg. +// +// Rate limiting: the register throttles, and when it does the block is SUSTAINED. An earlier spike saw +// HTTP 429 at roughly 50 cumulative requests ending in a burst, and thereafter 429 to every subsequent +// request — including simple /Deeds/{eik} calls that had worked seconds before. There is no +// `Retry-After` and no `X-RateLimit-*` header, so a client cannot pace against a published budget; it +// can only avoid tripping one. That makes a 429 an instruction to STOP, not a transient to retry +// through: `politeTrGet` retries 5xx and network faults with growing backoff, and never retries a 429. +// (#279 §3's "5 retries with growing backoff" and the decision that a 429 ends the run are consistent +// only if the retry set excludes 429.) The limiter is the operator's only way to express a rate +// preference, and tuning around it empirically is what spec §3.3's "NEVER bulk-scrape" forbids. + +import https from 'node:https'; +import { safeEik } from './paths.mjs'; + +export const TR_HOST = 'portal.registryagency.bg'; + +// Identify the crawler honestly. The operator's only lever on us is the rate limiter, so at minimum +// they should be able to see who is calling and where to complain. +export const TR_USER_AGENT = + 'sigma-bot/1.0 (+https://github.com/midt-bg/sigma) contact: via repo issues'; + +/** Thrown on HTTP 429. Distinguishable so the crawler can end the run instead of marking anything. */ +export class RateLimitError extends Error { + constructor(url) { + super(`REFUSE TO CONTINUE: ${TR_HOST} returned 429 for ${url} — the block is sustained; stop`); + this.name = 'RateLimitError'; + this.url = url; + } +} + +/** Refuse any URL that is not an https request to the register. Call before touching the network. */ +export function assertTrHost(url) { + let u; + try { + u = new URL(String(url)); + } catch { + throw new Error(`bad URL: ${url}`); + } + if (u.protocol !== 'https:' || u.hostname !== TR_HOST) { + throw new Error(`refusing non-${TR_HOST} host: ${url}`); + } + return url; +} + +/** The public deed endpoint for one ЕИК. Public JSON, no authentication. */ +export function deedUrl(eik) { + return `https://${TR_HOST}/CR/api/Deeds/${safeEik(eik)}`; +} + +const wait = (ms) => new Promise((r) => setTimeout(r, ms)); + +/** + * Default transport: `node:https`, system roots, secure defaults. + * + * NOT `fetch`. Measured against the live endpoint on 2026-08-05, the identical request returns + * **HTTP 500 with an empty body via undici's fetch and HTTP 200 with the full 34,398-byte deed via + * node:https** — same URL, same accept header, same user-agent. The server rejects something undici + * adds to the wire (encoding negotiation / connection handling); it is not an auth or rate-limit + * problem, and retrying only multiplies the failure. `scripts/cacbg/tls.mjs` is on node:https too, so + * both crawl legs now share one transport primitive. + * + * `rejectUnauthorized` is left at its secure default and named here on purpose: the sibling CACBG + * module deliberately does NOT verify against system roots (it pins a leaf instead, ADR-0011), so a + * reader comparing the two needs to see that this leg takes the ordinary, stronger path. + */ +export function httpsGet(url, { timeoutMs = 20_000 } = {}) { + return new Promise((resolve, reject) => { + const req = https.get( + url, + { headers: { accept: 'application/json', 'user-agent': TR_USER_AGENT } }, + (res) => { + const chunks = []; + res.on('data', (c) => chunks.push(c)); + res.on('end', () => + resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) }), + ); + res.on('error', reject); + }, + ); + req.setTimeout(timeoutMs, () => req.destroy(new Error(`timeout after ${timeoutMs}ms: ${url}`))); + req.on('error', reject); + }); +} + +/** + * GET with polite retries. 5xx and network faults retry with growing backoff; 2xx/4xx return as-is; + * **429 throws `RateLimitError` immediately and is never retried**. + * @param {string} url + * @param {{httpGet?:Function, sleep?:Function, tries?:number, backoffMs?:number}} [opts] + * `httpGet` and `sleep` are the injection seam — the whole retry policy is testable offline. + */ +export async function politeTrGet(url, opts = {}) { + const { httpGet = httpsGet, sleep = wait, tries = 5, backoffMs = 1000 } = opts; + assertTrHost(url); // before any request — a foreign host must cost zero packets + let backoff = backoffMs; + for (let attempt = 1; ; attempt++) { + let res; + try { + res = await httpGet(url); + } catch (err) { + if (attempt >= tries) throw err; + await sleep(backoff); + backoff *= 2; + continue; + } + // Checked before the retry branch: a 429 arriving mid-retry must abort the whole call, not be + // folded into the 5xx budget and not let a later 200 mask the block. + if (res.status === 429) throw new RateLimitError(url); + if (res.status >= 500) { + if (attempt >= tries) return res; // give the caller the last response to record + await sleep(backoff); + backoff *= 2; + continue; + } + return res; + } +} + +/** One deed by ЕИК. Thin wrapper so callers never build the URL themselves. */ +export function trGet(eik, opts) { + return politeTrGet(deedUrl(eik), opts); +} diff --git a/scripts/tr/client.test.mjs b/scripts/tr/client.test.mjs new file mode 100644 index 00000000..6694076f --- /dev/null +++ b/scripts/tr/client.test.mjs @@ -0,0 +1,152 @@ +// node:test — the Trade Register HTTP client. Offline: every test drives an injected getter. +// +// The load-bearing behaviour here is what the client REFUSES to do. The register rate-limits, and +// when it does the block is sustained (an earlier spike saw HTTP 429 at ~50 cumulative requests, then +// 429 to every subsequent call including ones that had just worked — no Retry-After, no quota header). +// So a 429 is an instruction to stop, not a transient to retry through. ADR-0033 decision 7. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + TR_HOST, + TR_USER_AGENT, + deedUrl, + RateLimitError, + politeTrGet, + assertTrHost, + httpsGet, +} from './client.mjs'; + +const okRes = (body = '{}') => ({ status: 200, headers: {}, body: Buffer.from(body) }); + +test('deedUrl builds the documented endpoint and refuses a non-ЕИК', () => { + assert.equal(deedUrl('115536179'), `https://${TR_HOST}/CR/api/Deeds/115536179`); + assert.equal(deedUrl('000696327'), `https://${TR_HOST}/CR/api/Deeds/000696327`); + for (const bad of ['', null, '../../etc', '115536179?x=1', '11553617x', 'ЕИК 115536179']) + assert.throws(() => deedUrl(bad), /unsafe|ЕИК/i, String(bad)); +}); + +test('assertTrHost refuses every host but the register', () => { + assert.doesNotThrow(() => assertTrHost(`https://${TR_HOST}/CR/api/Deeds/115536179`)); + for (const bad of [ + 'https://evil.example/CR/api/Deeds/115536179', + `http://${TR_HOST}/CR/api/Deeds/115536179`, // plaintext — never + `https://${TR_HOST}.evil.example/x`, // suffix trick + `https://register.cacbg.bg/x`, // the OTHER register: different host, different TLS posture + ]) { + assert.throws(() => assertTrHost(bad), /host/i, bad); + } +}); + +test('a 429 throws RateLimitError and is NEVER retried', async () => { + let calls = 0; + const httpGet = async () => { + calls++; + return { status: 429, headers: {}, body: Buffer.from('') }; + }; + await assert.rejects( + () => politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {}, tries: 5 }), + RateLimitError, + ); + // The whole point: „5 retries with growing backoff" and „429 stops the run" are only consistent + // if the retry set EXCLUDES 429. One call, not five. + assert.equal(calls, 1, 'a 429 must not be retried'); +}); + +test('5xx IS retried with growing backoff, up to the try budget', async () => { + let calls = 0; + const waits = []; + const httpGet = async () => { + calls++; + return { status: 503, headers: {}, body: Buffer.from('') }; + }; + const res = await politeTrGet(deedUrl('115536179'), { + httpGet, + sleep: async (ms) => void waits.push(ms), + tries: 4, + }); + assert.equal(calls, 4); + assert.equal(res.status, 503, 'the last response is returned, not thrown'); + assert.deepEqual( + waits.map((w, i) => (i === 0 ? true : w > waits[i - 1])), + [true, true, true], + `backoff must grow: ${waits.join(',')}`, + ); +}); + +test('a network throw is retried, then rethrown when the budget is spent', async () => { + let calls = 0; + const httpGet = async () => { + calls++; + throw new Error('ECONNRESET'); + }; + await assert.rejects( + () => politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {}, tries: 3 }), + /ECONNRESET/, + ); + assert.equal(calls, 3); +}); + +test('a 429 arriving mid-retry aborts immediately instead of finishing the budget', async () => { + const seq = [503, 503, 429, 200]; + let calls = 0; + const httpGet = async () => { + const status = seq[calls++]; + return { status, headers: {}, body: Buffer.from('') }; + }; + await assert.rejects( + () => politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {}, tries: 5 }), + RateLimitError, + ); + assert.equal(calls, 3, 'must stop AT the 429, not carry on to the 200 behind it'); +}); + +test('200 and 404 return without any retry', async () => { + for (const status of [200, 404]) { + let calls = 0; + const httpGet = async () => { + calls++; + return status === 200 ? okRes() : { status, headers: {}, body: Buffer.from('') }; + }; + const res = await politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {} }); + assert.equal(res.status, status); + assert.equal(calls, 1); + } +}); + +test('politeTrGet refuses a foreign host before making any request', async () => { + let calls = 0; + const httpGet = async () => { + calls++; + return okRes(); + }; + await assert.rejects( + () => politeTrGet('https://evil.example/CR/api/Deeds/1', { httpGet }), + /host/i, + ); + assert.equal(calls, 0, 'the refusal must happen before the network call'); +}); + +test('the crawler identifies itself and points at somewhere to complain', () => { + // The rate limiter is the operator's only lever on us. Crawling anonymously takes away their + // ability to distinguish us from an abusive client, or to ask us to stop. + assert.match(TR_USER_AGENT, /sigma/i); + assert.match(TR_USER_AGENT, /https:\/\/github\.com\//); +}); + +test('the default transport is node:https, not fetch', () => { + // Measured 2026-08-05: the identical live request returns 500/empty via undici fetch and 200 with + // the full deed via node:https. Pinning the choice so a future "modernise to fetch" tidy-up has to + // confront it rather than silently break every lookup. + assert.equal(typeof httpsGet, 'function'); + assert.match(httpsGet.toString(), /https\.get/); +}); + +test('RateLimitError carries the url and is distinguishable from a generic failure', async () => { + const httpGet = async () => ({ status: 429, headers: {}, body: Buffer.from('') }); + const err = await politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {} }).catch( + (e) => e, + ); + assert.ok(err instanceof RateLimitError); + assert.ok(err instanceof Error); + assert.match(err.message, /115536179/); +}); diff --git a/scripts/tr/eik.mjs b/scripts/tr/eik.mjs new file mode 100644 index 00000000..5a4d5b12 --- /dev/null +++ b/scripts/tr/eik.mjs @@ -0,0 +1,61 @@ +// ЕИК validity — the Node twin of the rule that already lives in SQL. +// +// `eik_valid` in scripts/normalize-raw.sql decides which bidders get an ЕИК-keyed identity at all, so +// it defines the ЕИК space the whole matcher works in. Until now that rule was callable ONLY from +// SQL, which is why the registry leg needs this: the crawler must decide, in Node, whether a code is +// worth a lookup. The two implementations are pinned against each other by a test that lifts the CASE +// expression straight out of the .sql file and runs both over the same values (eik.test.mjs) — a copy +// of the rule would drift from the thing it is copying. +// +// The rule (ЗТРРЮЛНЦ / БУЛСТАТ): +// 9-digit — weight digits 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by 3..10; +// a second 10 becomes 0. Digit 9 must equal the control. +// 13-digit — the leading 9 must themselves be a valid 9-digit ЕИК, then weight digits 9..12 by +// 2,7,3,5 (fallback 4,9,5,7; a second 10 becomes 0). Digit 13 must equal that control. +// +// Everything here is string-in, string-out. An ЕИК is an identifier, not a number: public bodies carry +// codes of exactly the `000…` shape, and a numeric round-trip drops the leading zeros and silently +// turns one company's identifier into another's. + +/** Weighted control digit over `digits`, with the standard second-pass fallback. @returns {number} */ +function control(digits, primary, fallback) { + const sum = (ws) => ws.reduce((acc, w, i) => acc + w * digits[i], 0) % 11; + const first = sum(primary); + if (first < 10) return first; + const second = sum(fallback); + return second < 10 ? second : 0; +} + +/** + * Is this a structurally valid ЕИК (9 or 13 digits, correct control digit)? + * Service codes (`000000000`, `0000000000000`) are rejected outright, matching the SQL: they pass the + * arithmetic but are placeholders, and letting them through collapsed unrelated foreign suppliers onto + * one node (#195). + * @param {unknown} eik @returns {boolean} + */ +export function eikChecksumValid(eik) { + const s = String(eik ?? ''); + if (s === '000000000' || s === '0000000000000') return false; + if (!/^\d+$/.test(s)) return false; + if (s.length !== 9 && s.length !== 13) return false; + + const d = [...s].map(Number); + const c9 = control(d.slice(0, 8), [1, 2, 3, 4, 5, 6, 7, 8], [3, 4, 5, 6, 7, 8, 9, 10]); + if (c9 !== d[8]) return false; + if (s.length === 9) return true; + + const c13 = control(d.slice(8, 12), [2, 7, 3, 5], [4, 9, 5, 7]); + return c13 === d[12]; +} + +/** + * Normalise a raw ЕИК string to digits only, or null when it is not one. + * Mirrors the SQL's `eik_clean`: strips a leading „ЕИК " label and surrounding whitespace, and does + * NOT otherwise repair the value. Validity is a separate question — `eikChecksumValid`. + * @param {unknown} raw @returns {string|null} + */ +export function normalizeEik(raw) { + const s = String(raw ?? '').trim(); + const stripped = (s.startsWith('ЕИК ') ? s.slice(4) : s).trim(); + return /^\d{9}$|^\d{13}$/.test(stripped) ? stripped : null; +} diff --git a/scripts/tr/eik.test.mjs b/scripts/tr/eik.test.mjs new file mode 100644 index 00000000..7a80a191 --- /dev/null +++ b/scripts/tr/eik.test.mjs @@ -0,0 +1,135 @@ +// node:test — the ЕИК checksum, and its PARITY with the SQL that already owns this rule. +// +// Why parity matters: `eik_valid` in scripts/normalize-raw.sql decides which bidders get an ЕИК-keyed +// identity at all, so it defines the ЕИК space the whole matcher works in. A JS twin that disagrees +// would silently accept a code the pipeline rejects (or vice versa) and the registry lookup would be +// made against an entity the rest of the system does not believe exists. The parity test below runs +// both implementations over the same values through node:sqlite, so they cannot drift apart. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { DatabaseSync } from 'node:sqlite'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { eikChecksumValid, normalizeEik } from './eik.mjs'; + +// Real ЕИК, verifiable against the public registers — the only external anchor this rule has. +const REAL = [ + '000696327', // Община София + '831661388', // Министерство на регионалното развитие + '115536179', // „ПИМК" ООД + '207695026', // „Профит Екстра" ЕООД +]; + +test('accepts real 9-digit ЕИК', () => { + for (const e of REAL) assert.equal(eikChecksumValid(e), true, e); +}); + +test('rejects a wrong-digit twin of every real ЕИК', () => { + // The failure this rule exists to stop (#195): a typo twin passing as a distinct real company and + // collapsing unrelated suppliers onto one node. Mutating the LAST digit must always break it. + for (const e of REAL) { + const bad = e.slice(0, 8) + ((Number(e[8]) + 1) % 10); + assert.equal(eikChecksumValid(bad), false, `${bad} (twin of ${e})`); + } +}); + +test('rejects service codes, wrong lengths and non-digits', () => { + for (const bad of [ + '000000000', + '0000000000000', + '', + null, + undefined, + '12345678', // 8 + '1234567890', // 10 — never a valid ЕИК length, which is what makes the ЕГН guard sound + '11553617x', + '115 536 179', + ]) { + assert.equal(eikChecksumValid(bad), false, String(bad)); + } +}); + +test('13-digit branch requires BOTH the 9-digit prefix and the 13th control digit', () => { + // A клон/поделение code: the leading 9 must themselves be a valid ЕИК, then weights 2,7,3,5. + const base = '115536179'; + const valid13 = thirteen(base, '001'); + assert.equal(valid13.length, 13, valid13); + assert.equal(eikChecksumValid(valid13), true, valid13); + // break the 13th digit + const broken = valid13.slice(0, 12) + ((Number(valid13[12]) + 1) % 10); + assert.equal(eikChecksumValid(broken), false, broken); + // a valid 13th control over an INVALID 9-prefix must still fail — both halves are load-bearing + assert.equal(eikChecksumValid(thirteen('115536170', '001')), false); +}); + +// Build a 13-digit ЕИК with a correct final control digit: 9-digit prefix + 3 free digits + control. +// The 2,7,3,5 weights run over positions 9..12 — that is the prefix's OWN control digit plus the three +// free ones — and position 13 is the result. +function thirteen(prefix9, three) { + const s = prefix9 + three; + const d = [...s].map(Number); + const w = (ws) => ws.reduce((a, x, i) => a + x * d[8 + i], 0) % 11; + let c = w([2, 7, 3, 5]); + if (c === 10) { + c = w([4, 9, 5, 7]); + if (c === 10) c = 0; + } + return s + c; +} + +test('normalizeEik strips the „ЕИК " prefix and surrounding whitespace, like the SQL does', () => { + assert.equal(normalizeEik('ЕИК 115536179'), '115536179'); + assert.equal(normalizeEik(' 115536179 '), '115536179'); + assert.equal(normalizeEik('115536179'), '115536179'); + assert.equal(normalizeEik('не е ЕИК'), null); + assert.equal(normalizeEik(null), null); +}); + +test('normalizeEik preserves leading zeros (public bodies are exactly 000…)', () => { + // Lose these and the crawler fetches a DIFFERENT company's deed. String in, string out — never a + // numeric round-trip. + assert.equal(normalizeEik('000696327'), '000696327'); + assert.equal(typeof normalizeEik('000696327'), 'string'); +}); + +// ── the anti-drift pin ──────────────────────────────────────────────────────── +test('JS twin agrees with normalize-raw.sql eik_valid on every value', () => { + const sql = readFileSync(fileURLToPath(new URL('../normalize-raw.sql', import.meta.url)), 'utf8'); + const expr = extractEikValidExpression(sql); + + const db = new DatabaseSync(':memory:'); + db.exec('CREATE TABLE probe (eik_clean TEXT)'); + const ins = db.prepare('INSERT INTO probe VALUES (?)'); + + const cases = [...REAL, ...REAL.map((e) => e.slice(0, 8) + ((Number(e[8]) + 1) % 10))]; + for (const e of REAL) cases.push(thirteen(e, '001'), thirteen(e, '002')); + // Every 9-digit code over a fixed prefix — exercises BOTH weight passes incl. the second-10 → 0 fallback. + for (let i = 0; i < 100; i++) cases.push('2011223' + String(i).padStart(2, '0')); + for (const bad of ['000000000', '0000000000000', '12345678', '1234567890', '11553617x']) + cases.push(bad); + + for (const c of cases) ins.run(c); + const rows = db.prepare(`SELECT eik_clean AS e, ${expr} AS v FROM probe`).all(); + db.close(); + + assert.ok(rows.length >= 120, `expected a broad probe set, got ${rows.length}`); + for (const { e, v } of rows) { + assert.equal( + eikChecksumValid(e), + v === 1, + `disagreement on ${JSON.stringify(e)} (SQL said ${v})`, + ); + } +}); + +/** + * Lift the `CASE … END AS eik_valid` expression straight out of normalize-raw.sql, so the test + * compares against the FILE and not a copy of it. A copy would drift with the thing it is pinning. + */ +function extractEikValidExpression(sql) { + const start = sql.indexOf(' CASE\n WHEN eik_clean IS NULL'); + assert.ok(start > 0, 'eik_valid CASE not found in normalize-raw.sql — the pin lost its anchor'); + const end = sql.indexOf('AS eik_valid', start); + assert.ok(end > start, 'eik_valid terminator not found'); + return sql.slice(start, end); +} diff --git a/scripts/tr/paths.mjs b/scripts/tr/paths.mjs new file mode 100644 index 00000000..e325d293 --- /dev/null +++ b/scripts/tr/paths.mjs @@ -0,0 +1,54 @@ +// Paths and path sanitizers for the Търговски регистър leg (issue #279, ADR-0033). +// +// Everything this leg writes lives under scratch/tr/, git-ignored, behind the same refuse-to-run rail +// the CACBG crawl uses — a deed carries third-party personal data (owner and manager names, the +// company's street address), so it is ADR-0010 decision 6 territory, extended by ADR-0033 to a second +// source with a stated retention. +// +// scratch/tr/deeds/.json raw response, atomic write +// scratch/tr/tr-cache.sqlite the index — ЕИК, dates, codes, verdicts. NO names. +// +// The constants below are only DEFAULTS for the CLI. Every function that touches the filesystem takes +// its path explicitly, so tests drive temp directories without mutating process state. + +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { assertScratchIgnored } from '../cacbg/guard.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); + +export const TR_SCRATCH = path.join(ROOT, 'scratch', 'tr'); +export const TR_RAW = path.join(TR_SCRATCH, 'deeds'); +export const TR_DB = path.join(TR_SCRATCH, 'tr-cache.sqlite'); + +/** Refuse to run unless scratch/tr is git-ignored. Call before the first fetch. */ +export function assertTrScratchIgnored() { + assertScratchIgnored('tr'); +} + +// A bare ЕИК: 9 or 13 digits, nothing else. Deliberately NOT `path.basename`-normalised — an ЕИК that +// needed normalising did not come from where we think it did, and silently repairing it is how you end +// up fetching a different company's deed. +const EIK_SHAPE = /^(?:\d{9}|\d{13})$/; + +/** + * Sanitize an ЕИК before it becomes a path segment or a URL segment. + * + * Returns the value VERBATIM — as a string, always. Bulgarian public bodies carry codes of exactly the + * `000…` shape, so a numeric round-trip anywhere on this path silently rewrites the identifier and the + * crawler fetches somebody else's deed (R8). + * + * Shape only. Whether the code's CHECKSUM is valid is a different question with a different remedy, + * answered by eik.mjs — conflating the two would report a real-but-invalid code as a path attack. + * @param {unknown} eik @returns {string} + */ +export function safeEik(eik) { + const s = String(eik ?? ''); + if (!EIK_SHAPE.test(s)) throw new Error(`unsafe ЕИК: ${JSON.stringify(eik)}`); + return s; +} + +/** Absolute path of the cached raw deed for an ЕИК, under `rawDir` (default TR_RAW). */ +export function deedPath(eik, rawDir = TR_RAW) { + return path.join(rawDir, `${safeEik(eik)}.json`); +} diff --git a/scripts/tr/paths.test.mjs b/scripts/tr/paths.test.mjs new file mode 100644 index 00000000..3401260d --- /dev/null +++ b/scripts/tr/paths.test.mjs @@ -0,0 +1,64 @@ +// node:test — path sanitizers and the refuse-to-run rail for the Trade Register leg. +// +// The deed cache holds third-party personal data (owner/manager names, company addresses), so the +// same rail the CACBG crawl runs behind applies here: everything is written under scratch/, and +// scratch/ must be git-ignored, asserted BEFORE any fetch. ADR-0010 decision 6 as extended by ADR-0033. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { TR_SCRATCH, TR_RAW, TR_DB, safeEik, assertTrScratchIgnored } from './paths.mjs'; +import { assertScratchIgnored } from '../cacbg/guard.mjs'; + +test('the TR scratch tree sits under scratch/ and is git-ignored in this repo', () => { + assert.ok(TR_SCRATCH.split(path.sep).includes('scratch'), TR_SCRATCH); + assert.ok(TR_RAW.startsWith(TR_SCRATCH)); + assert.ok(TR_DB.startsWith(TR_SCRATCH)); + assert.doesNotThrow(() => assertTrScratchIgnored()); +}); + +test('the guard is the CACBG one generalized, not a second copy', () => { + // A duplicated safety rail drifts from the original. assertScratchIgnored now takes the + // subdirectory, and its existing no-argument callers keep working unchanged. + assert.doesNotThrow(() => assertScratchIgnored()); + assert.doesNotThrow(() => assertScratchIgnored('tr')); + // .gitignore ignores `scratch/` WHOLESALE, so every subdirectory of it passes — an unknown name is + // not a way to reach the failure branch. Escape scratch/ instead (path.join normalises this to + // `docs/.probe`, which is tracked) to prove the guard still refuses when the target is not ignored. + assert.throws(() => assertScratchIgnored(path.join('..', 'docs')), /REFUSE TO RUN/); +}); + +test('safeEik accepts only a bare 9/13-digit code and returns it verbatim', () => { + assert.equal(safeEik('115536179'), '115536179'); + assert.equal(safeEik('1155361790001'), '1155361790001'); + // Leading zeros survive — public bodies are exactly this shape, and losing them fetches a + // DIFFERENT company's deed. + assert.equal(safeEik('000696327'), '000696327'); +}); + +test('safeEik refuses anything that could leave the intended path or URL', () => { + for (const bad of [ + '', + null, + undefined, + '..', + '../115536179', + '/115536179', + '115536179/../x', + '115536179?x=1', + '115536179#f', + '11553617x', + '11553617', + '1155361790', + 'ЕИК 115536179', + ' 115536179', + '115536179 ', + ]) { + assert.throws(() => safeEik(bad), /unsafe/i, JSON.stringify(bad)); + } +}); + +test('safeEik does not validate the CHECKSUM — that is a separate question', () => { + // Path safety and identity validity are different concerns: a shape-valid but checksum-invalid code + // must still be rejectable by the caller with a specific reason, not conflated into „unsafe path". + assert.equal(safeEik('115536170'), '115536170'); +}); From f8b198f32297a5701730a7e428ac834b7947871e Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 17:57:41 +0300 Subject: [PATCH 05/69] feat(tr): deed parser and the six-rung evidence ladder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The libel-critical part of #279. Pure and offline: deed in, verdict out. Nothing is wired into the publishing path yet — that is the next commit. deed.mjs enforces one order of operations, and it is an invariant with tests on it rather than a comment: decode entities → split on record-container/hr--report → drop erased → strip tags within the entity → match tokens within ONE entity. Field CR_F_19_L routinely holds several съдружници in a single string; matching against the whole field lets one person's given name combine with another's surname, and the output is a named public claim about the wrong human being. The test proves both directions — the frankenstein name does NOT match, the correct declarant DOES, and the naive whole-field approach WOULD have fired. Without that last assertion a matcher that always returns false would pass the suite. evidence.mjs implements the ladder: bar_joint_stock · document · confirmed · refuted · unknown · outside_tr, first match wins. What it establishes is company IDENTITY, never that the official owns anything — the ownership claim is the official's own filed declaration. So the failure mode of a wrong match is a real official attached to the wrong company's contracts. Fixtures are real markup, copied from the live deed for ЕИК 115536179. Writing this against imagined markup is how the entity-boundary bug ships, and doing so caught two things the ADR had wrong. ADR-0033 corrected on implementation, both from re-measuring the full deed: • the erasure marker is `erasure-text-inline`, NOT `field-text--erased` — that class occurs zero times, and an erased container has no field-text paragraph at all. Both spellings are now honoured; assuming one costs a wrong publish. • fieldOperation is not the erasure signal (2 on erased fields here, 1 on the erased history records W0 sampled), so it is not relied on. Guards that can only withhold: an unknown legal form never falls through to a lower rung; a two-token declarant can never earn „Документ"; a Latin homoglyph is a non-match and is counted; an empty seat never confirms; a seat registered after the declared period does not confirm; refutation never applies to a family stake and is suppressed inside the 2011-2012 re-registration window; matched_fact is a closed vocabulary that cannot carry a name. --- ...-evidence-replaces-name-distinctiveness.md | 11 +- scripts/tr/deed.mjs | 284 ++++++++++++++++ scripts/tr/deed.test.mjs | 273 +++++++++++++++ scripts/tr/evidence.mjs | 224 ++++++++++++ scripts/tr/evidence.test.mjs | 318 ++++++++++++++++++ 5 files changed, 1108 insertions(+), 2 deletions(-) create mode 100644 scripts/tr/deed.mjs create mode 100644 scripts/tr/deed.test.mjs create mode 100644 scripts/tr/evidence.mjs create mode 100644 scripts/tr/evidence.test.mjs diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md index e4d8ff2d..2e449127 100644 --- a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -68,9 +68,16 @@ requests, 2026-08-04 — well inside the pacing above, so no 429 was encountered — `legalForm=4` and `legalForm=10` return a byte-identical catalogue. - The envelope's `fullName` carries the legal form (`"ПИМК" ООД`) while `CR_F_2_L` is bare. The joint-stock bar can therefore be derived from the ЗТРРЮЛНЦ-mandated suffix, at zero extra requests. -- Erasure is **structural, not textual**: an erased entity is a `record-container` block carrying the class - `field-text--erased`. Entities inside one field are separated by +- Erasure is **structural, not textual**. Entities inside one field are separated by `
` blocks delimited by `
`. + An erased entity is such a block carrying an erasure marker. **Correction, on implementation:** this ADR + first named that marker `field-text--erased`. Re-measured against the full live deed for ЕИК 115536179, + that class occurs **zero** times; the register emits `
` (with + ``) and the erased container carries **no `field-text` paragraph at + all**. The parser therefore treats *either* spelling as erasure — honouring both costs nothing, while + assuming one costs a wrong publish. `fieldOperation` is **not** the signal: it reads 2 on both erased + fields in that deed but 1 on the erased history records W0 sampled, so it is an undocumented enum we do + not rely on. - Erased versions are **live in the current deed**: the first company sampled carries a fully-erased `CR_F_23_L` dated 2013-07-16, which read naïvely becomes „latest ownership entry: 2013-07-16" and feeds the refutation rule. diff --git a/scripts/tr/deed.mjs b/scripts/tr/deed.mjs new file mode 100644 index 00000000..7709e31c --- /dev/null +++ b/scripts/tr/deed.mjs @@ -0,0 +1,284 @@ +// Pure parser for a Търговски регистър deed (issue #279, ADR-0033). No I/O, no network, no state. +// +// The envelope is clean JSON, but every field's VALUE is an HTML fragment with semi-structured +// Bulgarian inside. That inner parser is where the libel risk lives, so the order of operations below +// is an invariant with tests on it, not a suggestion: +// +// decode HTML entities +// → split into entities on record-container / hr--report +// → drop entities marked erased +// → strip tags WITHIN one entity +// → separate the name from the address/stake WITHIN that entity +// → match tokens ONLY within a single entity +// +// Any other order silently merges blocks. Field CR_F_19_L routinely holds several съдружници in one +// string; matching a declarant against the whole field lets one person's given name combine with +// another's surname, and the output is a named public claim about the wrong human being. + +const OWNERSHIP_FIELDS = ['CR_F_18_L', 'CR_F_19_L', 'CR_F_23_L']; +const MANAGER_FIELD = 'CR_F_7_L'; +export const ROLE_FIELDS = [MANAGER_FIELD, ...OWNERSHIP_FIELDS]; +export { OWNERSHIP_FIELDS, MANAGER_FIELD }; + +// ── html ────────────────────────────────────────────────────────────────────── +const ENTITIES = { + '"': '"', + ''': "'", + '&': '&', + '<': '<', + '>': '>', + ' ': ' ', +}; +/** Decode the entity set the register actually emits, plus numeric escapes. FIRST step, always. */ +function decodeEntities(s) { + return String(s) + .replace(/&(?:quot|apos|amp|lt|gt|nbsp);/g, (m) => ENTITIES[m]) + .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n))) + .replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCodePoint(parseInt(n, 16))); +} + +/** Strip tags and collapse whitespace, INSIDE one already-isolated entity. */ +function stripTags(html) { + return String(html) + .replace(//gi, ' ') + .replace(/<[^>]*>/g, '') + .replace(/\s+/g, ' ') + .trim(); +} + +// An entity is erased when the register says so. MEASURED on a live deed: the marker is +// `erasure-text-inline` (with `ui-icon-erased`) and the container carries no `field-text` paragraph at +// all. `field-text--erased` is accepted as well — it is the other spelling reported for this register, +// and honouring both costs nothing while assuming one costs a wrong publish. +const ERASED_MARKER = /erasure-text-inline|ui-icon-erased|field-text--erased/; + +/** + * Split one field's htmlData into the separate registered entities it holds. + * + * @param {string} html + * @param {{strict?:boolean}} [opts] `strict` turns the erased-with-content contradiction into a throw. + * @returns {{text:string, erased:boolean}[]} + */ +export function entityBlocks(html, { strict = false } = {}) { + if (html == null || String(html).trim() === '') return []; + const decoded = decodeEntities(html); // decode BEFORE splitting — see the order above + const chunks = decoded + .split(/]*>/i) + .flatMap((part) => part.split(/(?=]*class='[^']*record-container)/i)) + .map((c) => c.trim()) + .filter((c) => c !== ''); + + const out = []; + for (const chunk of chunks) { + const erased = ERASED_MARKER.test(chunk); + // Read the visible text WITHOUT the erasure notice, so „Заличено обстоятелство." never counts as + // content and an erased block reads as empty. + const withoutNotice = chunk.replace( + /]*class='[^']*erasure-text-inline[^']*'[^>]*>.*?<\/div>/gis, + ' ', + ); + const text = stripTags(withoutNotice); + if (erased && text !== '' && strict) { + throw new Error( + `REFUSE: an erased entity carries content (${JSON.stringify(text.slice(0, 60))}) — the ` + + `"erased ⇒ empty" assumption has drifted; stop rather than guess which state is in force`, + ); + } + if (text === '' && !erased) continue; // structural noise, not an entity + out.push({ text, erased }); + } + return out; +} + +/** Every field in the deed, flattened out of sections → subDeeds → groups. */ +function allFields(deed) { + const out = []; + for (const s of deed?.sections ?? []) + for (const sd of s?.subDeeds ?? []) + for (const g of sd?.groups ?? []) for (const f of g?.fields ?? []) out.push(f); + return out; +} + +const isoDay = (v) => (v ? String(v).slice(0, 10) : null); + +/** + * The LIVE entities of the requested field codes — the single entry point to live state. + * Erased entities are dropped here and nowhere else, so the rule is auditable in one place. + * @returns {{nameCode:string, entryDate:string|null, entryNumber:string|null, entities:string[]}[]} + */ +export function liveFields(deed, nameCodes, opts = {}) { + const want = new Set(nameCodes); + const out = []; + for (const f of allFields(deed)) { + if (!want.has(f.nameCode)) continue; + const entities = entityBlocks(f.htmlData, opts) + .filter((b) => !b.erased) + .map((b) => b.text); + if (entities.length === 0) continue; + out.push({ + nameCode: f.nameCode, + // TEXT, never a number: a fieldEntryNumber like 20130716101007 exceeds 2^53 once combined. + entryNumber: f.fieldEntryNumber == null ? null : String(f.fieldEntryNumber), + entryDate: isoDay(f.fieldEntryDate), + entities, + }); + } + return out; +} + +// ── names ───────────────────────────────────────────────────────────────────── +/** + * Name tokens: NFC, upper case, split on non-letters, keep tokens of length ≥2. + * + * Dropping 1-character tokens is what makes „Г. И. Петров" a ONE-token name rather than a three-token + * one — an abbreviated name can then never reach the ≥3 tokens rung 2 requires, instead of passing on + * initials that match half the register. Latin letters are kept (never folded onto Cyrillic + * look-alikes) so a homoglyph is a non-match rather than a false match, matching companyNameKey's + * posture in packages/shared/src/company-name-key.ts. + * + * A near-twin of the module-private `holderTokens` in scripts/cacbg/parse.mjs — deliberately + * re-implemented rather than imported, because that module pulls in fast-xml-parser and would tie + * these pure tests to a workspace install. Keep the two in step. + */ +export function personTokens(name) { + return String(name ?? '') + .normalize('NFC') + .toUpperCase() + .split(/[^\p{L}]+/u) + .filter((t) => [...t].length >= 2); +} + +/** + * Does EVERY token of the declarant's name appear as a whole token of this ONE entity? + * + * Full subset, not a majority: of 301 measured matches, 46 were two-token only, which is precisely + * the homonym risk. Whole-token, not substring: „ПЕТРОВ" inside „ПЕТРОВА" is a different person. + * + * Callers MUST pass a single entity's text (see entityBlocks). Passing a whole field is the + * cross-entity bug, and no signature can prevent it — the test does. + */ +export function fullSubsetMatch(declarantName, entityText) { + const want = personTokens(declarantName); + if (want.length === 0) return false; + const have = new Set(personTokens(entityText)); + return want.every((t) => have.has(t)); +} + +// ── seat ────────────────────────────────────────────────────────────────────── +// Strip a settlement-type prefix only as a WHOLE token: „гр."/„с."/„общ."/„обл."/„ж.к." followed by a +// dot and optional space. R9 — a loose prefix strip turns СОФИЯ into ОФИЯ and ГРАДЕЦ into АДЕЦ. +const SETTLEMENT_PREFIX = /^(?:ГР|С|ОБЩ|ОБЛ|Ж\.К)\.\s*/u; + +/** Normalise a settlement name for comparison. Empty in ⇒ empty out, and empty NEVER confirms. */ +export function normalizeSettlement(raw) { + let s = String(raw ?? '') + .normalize('NFC') + .toUpperCase() + .replace(/\([^)]*\)/g, ' ') // „(столица)" + .trim(); + s = s.split(/[,/]/)[0].trim(); // cut at the first comma or slash — „с. Марково, п.к. 4108" + s = s.replace(SETTLEMENT_PREFIX, ''); + return s + .replace(/[^\p{L}\s-]/gu, '') + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * The company's registered settlement, from CR_F_5_L's „Населено място:" segment ONLY. + * + * ADR-0010 item 3 (addresses are never extracted) is honoured here, and the field makes that a live + * concern rather than a formality: CR_F_5_L also carries street, postcode, phone, fax, e-mail and + * website. Nothing but the settlement and the entry date leaves this function. + * @returns {{settlement:string, entryDate:string|null}} + */ +export function registrySeat(deed) { + for (const f of liveFields(deed, ['CR_F_5_L'])) { + for (const text of f.entities) { + const m = text.match(/Населено място:\s*([^,]+)/u); + if (m) return { settlement: normalizeSettlement(m[1]), entryDate: f.entryDate }; + } + } + return { settlement: '', entryDate: null }; +} + +// ── legal form ──────────────────────────────────────────────────────────────── +// Codes observed empirically (#279 §3 + the spike's catalogue read). DELIBERATELY incomplete: the +// nomenclature endpoint does not settle the enum (legalForm=4 and =10 return a byte-identical +// catalogue), so anything absent here is `unknown` and WITHHOLDS. In particular whether ЕАД carries +// its own code is unresolved — ООД and ЕООД turned out NOT to share one (4 vs 10), so assuming ЕАД +// shares 5 with АД would be exactly the fail-open this bar exists to prevent. +const FORM_CODES = new Map([ + [1, 'closely_held'], // ЕТ + [4, 'closely_held'], // ООД + [5, 'joint_stock'], // АД + [6, 'joint_stock'], // КДА + [10, 'closely_held'], // ЕООД +]); + +// The фирма's legal form is its SUFFIX under ЗТРРЮЛНЦ, and the deed envelope's `fullName` carries it +// („ПИМК" ООД) while CR_F_2_L is bare („ПИМК"). So the bar has a second, independent signal at zero +// extra cost. КДА is included here and is absent from classify.mjs's JOINT_STOCK/FORM_TOKENS today. +const JOINT_SUFFIX = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; +const CLOSELY_SUFFIX = /(?:^|[\s"„“”«»])(ООД|ЕООД|ЕТ|ДЗЗД|КД|СД|КООПЕРАЦИЯ)[\s"„“”«»]*$/u; + +/** + * Legal-form verdict — a union of the numeric code and the mandated фирма suffix. + * Either signal saying joint-stock bars the link. Neither able to say ⇒ `unknown`, which withholds. + * @returns {{code:number|null, codeVerdict:string, suffixVerdict:string, verdict:string}} + */ +export function registryLegalForm(deed) { + const code = typeof deed?.legalForm === 'number' ? deed.legalForm : null; + const codeVerdict = (code != null && FORM_CODES.get(code)) || 'unknown'; + + const name = String(deed?.fullName ?? '') + .normalize('NFC') + .toUpperCase() + .trim(); + const suffixVerdict = JOINT_SUFFIX.test(name) + ? 'joint_stock' + : CLOSELY_SUFFIX.test(name) + ? 'closely_held' + : 'unknown'; + + const verdict = + codeVerdict === 'joint_stock' || suffixVerdict === 'joint_stock' + ? 'joint_stock' + : codeVerdict === 'closely_held' || suffixVerdict === 'closely_held' + ? 'closely_held' + : 'unknown'; + return { code, codeVerdict, suffixVerdict, verdict }; +} + +// ── refutation input ────────────────────────────────────────────────────────── +/** + * Latest entry date across the LIVE ownership fields, or null when none survives. + * + * The trap this avoids was present in the first company sampled: CR_F_23_L sits in the CURRENT deed + * dated 2013-07-16 carrying only „Заличено обстоятелство.". Counted naively it becomes the latest + * ownership entry and can refute a link it says nothing about. liveFields drops it. + */ +export function latestOwnershipEntryDate(deed) { + const dates = liveFields(deed, OWNERSHIP_FIELDS) + .map((f) => f.entryDate) + .filter(Boolean); + return dates.length ? dates.sort().at(-1) : null; +} + +/** + * The deed we got back must be the deed we asked for. + * + * R8: Bulgarian public bodies carry ЕИК of exactly the `000…` shape, so any numeric round-trip on the + * path silently rewrites the identifier. Without this rail, that failure publishes a claim about a + * different company under the right official's name. + */ +export function assertUicEcho(deed, requestedEik) { + const got = deed?.uic == null ? null : String(deed.uic); + if (got !== String(requestedEik)) { + throw new Error( + `REFUSE: deed uic echo mismatch — requested ${JSON.stringify(String(requestedEik))}, ` + + `deed reports ${JSON.stringify(got)}`, + ); + } + return deed; +} diff --git a/scripts/tr/deed.test.mjs b/scripts/tr/deed.test.mjs new file mode 100644 index 00000000..6ec634ef --- /dev/null +++ b/scripts/tr/deed.test.mjs @@ -0,0 +1,273 @@ +// node:test — the deed parser. Pure, offline, and the most dangerous code in this change. +// +// Every fixture below is REAL markup, copied verbatim from a live deed (ЕИК 115536179, fetched +// 2026-08-05) with person names replaced only where a test needs a specific shape. Writing this +// parser against imagined markup is how the entity-boundary bug ships. +// +// The failure this file exists to prevent: field CR_F_19_L holds THREE separate people in one string, +// separated by
. Matching a declarant's tokens against the whole field lets +// the given name of one person combine with the surname of another, and the result is a named public +// claim that a specific official owns a specific company — about the wrong person. ADR-0033 decision 2. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + entityBlocks, + liveFields, + personTokens, + fullSubsetMatch, + normalizeSettlement, + registrySeat, + registryLegalForm, + latestOwnershipEntryDate, + assertUicEcho, +} from './deed.mjs'; + +// ── real markup ─────────────────────────────────────────────────────────────── +const F19_THREE = `

ПИМК ХОЛДИНГ ГРУП АД, ЕИК/ПИК 202294392, Държава: БЪЛГАРИЯ, Размер на дяловото участие: 59980.00 лв.

ПЕНКО НЕСТОРОВ НЕСТОРОВ, Държава: БЪЛГАРИЯ, Размер на дяловото участие: 10.00 лв.

ИЛИЯН КОСТАДИНОВ ФИЛИПОВ, Държава: БЪЛГАРИЯ, Размер на дяловото участие: 10.00 лв.

`; + +const F23_ERASED = `
Заличено обстоятелство.
`; + +const F5_SEAT = `

Държава: БЪЛГАРИЯ
Област: Пловдив, Община: Родопи
Населено място: с. Марково, п.к. 4108
бул./ул. местност ЗАХАРИДЕВО № 043А Телефон: 032/901102 и 032/945149
Адрес на електронна поща: office@pimk-bg.eu

`; + +const F7_MANAGER = `

АНТОН ИЦКОВ ЙОРДАНОВ, Държава: БЪЛГАРИЯ

`; + +const field = (nameCode, htmlData, over = {}) => ({ + nameCode, + htmlData, + fieldEntryNumber: '20130716101007', + fieldEntryDate: '2013-07-16T10:10:07', + fieldOperation: 3, + fieldIdent: '00190', + ...over, +}); +const deedOf = (fields, over = {}) => ({ + uic: '115536179', + fullName: '"ПИМК" ООД', + legalForm: 4, + sections: [{ subDeeds: [{ groups: [{ fields }] }] }], + ...over, +}); + +// ── T1 — the entity boundary ────────────────────────────────────────────────── +test('entityBlocks splits one field into its separate registered entities', () => { + const blocks = entityBlocks(F19_THREE); + assert.equal(blocks.length, 3, 'three съдружници, three blocks'); + assert.match(blocks[0].text, /ПИМК ХОЛДИНГ ГРУП АД/); + assert.match(blocks[1].text, /ПЕНКО НЕСТОРОВ НЕСТОРОВ/); + assert.match(blocks[2].text, /ИЛИЯН КОСТАДИНОВ ФИЛИПОВ/); + // No block may carry another block's name — that is the whole point. + assert.ok(!blocks[1].text.includes('ФИЛИПОВ')); + assert.ok(!blocks[2].text.includes('ПЕНКО')); +}); + +test('T1 — tokens from two DIFFERENT people never combine into a match', () => { + // „ПЕНКО … НЕСТОРОВ" and „ИЛИЯН КОСТАДИНОВ ФИЛИПОВ" are both in field 19. A declarant assembled + // from one person's given name and another's patronymic+surname must NOT match. + const frankenstein = 'ПЕНКО КОСТАДИНОВ ФИЛИПОВ'; + const blocks = entityBlocks(F19_THREE); + assert.equal( + blocks.some((b) => fullSubsetMatch(frankenstein, b.text)), + false, + 'cross-entity match — this is the libel bug', + ); + // Whole-field matching is exactly what must not be done; prove the naive approach WOULD have fired, + // so this test cannot silently pass because the matcher is broken in some other way. + assert.equal(fullSubsetMatch(frankenstein, F19_THREE), true, 'naive whole-field match fires'); +}); + +test('T1 positive control — the CORRECT declarant does match', () => { + // Without this, a matcher that always returns false passes every negative test above (ADR-0027). + const blocks = entityBlocks(F19_THREE); + assert.equal( + blocks.some((b) => fullSubsetMatch('ИЛИЯН КОСТАДИНОВ ФИЛИПОВ', b.text)), + true, + ); + assert.equal( + blocks.some((b) => fullSubsetMatch('Пенко Несторов Несторов', b.text)), + true, + 'case/spacing drift still matches within the right entity', + ); +}); + +test('T1 — HTML entities are decoded BEFORE splitting, not after', () => { + // Decode-after-split leaves "-bearing names mangled; decode-before-split is the tested order. + const encoded = F19_THREE.replace('ПИМК ХОЛДИНГ ГРУП АД', '"ПИМК ХОЛДИНГ ГРУП" АД'); + const blocks = entityBlocks(encoded); + assert.equal(blocks.length, 3); + assert.match(blocks[0].text, /"ПИМК ХОЛДИНГ ГРУП" АД/); + assert.ok(!blocks[0].text.includes('"'), 'entities must be decoded'); +}); + +test('T1 — a corporate съдружник contributes its own tokens, not a person match', () => { + // „ПИМК ХОЛДИНГ ГРУП АД, ЕИК/ПИК 202294392" is a legal entity. A person whose name happens to + // overlap its words must not match it into existence. + const blocks = entityBlocks(F19_THREE); + assert.equal( + fullSubsetMatch('ПИМК ХОЛДИНГ ГРУП', blocks[0].text), + true, + 'literal overlap exists', + ); + // …which is why rung 2 requires ≥3 tokens of a PERSON name; the guard lives in evidence.mjs (T2). +}); + +// ── erasure ─────────────────────────────────────────────────────────────────── +test('an erased entity is detected structurally and dropped from live state', () => { + // MEASURED: the live deed marks erasure with `erasure-text-inline` and the container carries NO + //

at all. An earlier note claimed the marker was `field-text--erased`; that + // class does not occur in the sampled deed. Both are treated as erasure so either shape is safe. + const blocks = entityBlocks(F23_ERASED); + assert.equal(blocks.length, 1); + assert.equal(blocks[0].erased, true); + const live = liveFields(deedOf([field('CR_F_23_L', F23_ERASED, { fieldOperation: 2 })]), [ + 'CR_F_23_L', + ]); + assert.deepEqual(live, [], 'an erased field contributes no live entity'); +}); + +test('the OTHER erasure spelling is honoured too', () => { + const alt = `

СТАР СОБСТВЕНИК

Заличено обстоятелство.
`; + assert.equal(entityBlocks(alt)[0].erased, true); +}); + +test('an erased-looking block carrying real content REFUSES the deed (drift alarm)', () => { + // R7: "erased ⇒ empty" is an empirical observation (93 of 93). If the register ever emits an erased + // marker beside live content, the assumption is broken and we must stop, not silently drop a real + // owner or silently keep a removed one. + const contradictory = `

ЖИВО ИМЕ ТУК

Заличено обстоятелство.
`; + assert.throws(() => entityBlocks(contradictory, { strict: true }), /erased.*content|drift/i); +}); + +test('an empty htmlData yields no entities and does not throw', () => { + assert.deepEqual(entityBlocks(''), []); + assert.deepEqual(entityBlocks(null), []); +}); + +test('liveFields keeps only the requested codes and reports entry date/number', () => { + const d = deedOf([ + field('CR_F_7_L', F7_MANAGER, { nameCode: 'CR_F_7_L', fieldEntryDate: '2017-09-15T00:00:00' }), + field('CR_F_19_L', F19_THREE), + field('CR_F_5_L', F5_SEAT), + ]); + const live = liveFields(d, ['CR_F_7_L', 'CR_F_19_L']); + assert.deepEqual(live.map((f) => f.nameCode).sort(), ['CR_F_19_L', 'CR_F_7_L']); + const f19 = live.find((f) => f.nameCode === 'CR_F_19_L'); + assert.equal(f19.entities.length, 3); + assert.equal(f19.entryDate, '2013-07-16'); + assert.equal(f19.entryNumber, '20130716101007'); + assert.equal(typeof f19.entryNumber, 'string', 'entry numbers exceed 2^53 — never a number'); +}); + +// ── T2 — the token rule ─────────────────────────────────────────────────────── +test('personTokens keeps tokens of length ≥2 and folds case/spacing', () => { + assert.deepEqual(personTokens('Иван Петров Георгиев'), ['ИВАН', 'ПЕТРОВ', 'ГЕОРГИЕВ']); + assert.deepEqual(personTokens(' иван петров '), ['ИВАН', 'ПЕТРОВ']); + // An initial is not a token — „Г. И. Петров" is ONE token, so it can never reach three. + assert.deepEqual(personTokens('Г. И. Петров'), ['ПЕТРОВ']); + // A hyphenated surname is one token, not two. + assert.deepEqual(personTokens('Мария Иванова-Петрова'), ['МАРИЯ', 'ИВАНОВА', 'ПЕТРОВА']); +}); + +test('fullSubsetMatch requires EVERY declarant token, not a majority', () => { + const entity = 'ИЛИЯН КОСТАДИНОВ ФИЛИПОВ, Държава: БЪЛГАРИЯ'; + assert.equal(fullSubsetMatch('ИЛИЯН КОСТАДИНОВ ФИЛИПОВ', entity), true); + // 2-of-3 must fail: of 301 measured matches, 46 were two-token only — the homonym risk itself. + assert.equal(fullSubsetMatch('ИЛИЯН КОСТАДИНОВ ПЕТРОВ', entity), false); + assert.equal(fullSubsetMatch('ИЛИЯН ПЕТРОВ ФИЛИПОВ', entity), false); +}); + +test('a token must match a WHOLE token, never a substring', () => { + // „ПЕТРОВ" must not be found inside „ПЕТРОВА"; that is a different person. + assert.equal(fullSubsetMatch('ИВАН ПЕТРОВ ГЕОРГИЕВ', 'ИВАН ПЕТРОВА ГЕОРГИЕВА'), false); +}); + +// ── T5 — settlement normalization ───────────────────────────────────────────── +test('T5 — the settlement prefix is stripped only as a whole token', () => { + // R9: a naive prefix strip turns СОФИЯ into ОФИЯ and ГРАДЕЦ into АДЕЦ. + assert.equal(normalizeSettlement('гр. Русе'), 'РУСЕ'); + assert.equal(normalizeSettlement('с. Марково'), 'МАРКОВО'); + assert.equal(normalizeSettlement('София'), 'СОФИЯ'); + assert.equal(normalizeSettlement('СОФИЯ'), 'СОФИЯ'); + assert.equal(normalizeSettlement('Градец'), 'ГРАДЕЦ'); + assert.equal(normalizeSettlement('гр.Пловдив'), 'ПЛОВДИВ'); + assert.equal(normalizeSettlement('София (столица)'), 'СОФИЯ'); +}); + +test('T5 — an empty settlement never equals another empty settlement', () => { + // „both blank ⇒ confirmed" would rubber-stamp every link with no seat data at all. + assert.equal(normalizeSettlement(''), ''); + assert.equal(normalizeSettlement(null), ''); + assert.equal(normalizeSettlement(' '), ''); +}); + +test('registrySeat reads the „Населено място" segment and NOTHING else', () => { + const seat = registrySeat(deedOf([field('CR_F_5_L', F5_SEAT, { nameCode: 'CR_F_5_L' })])); + assert.equal(seat.settlement, 'МАРКОВО'); + assert.equal(seat.entryDate, '2013-07-16'); + // ADR-0010 item 3: the parser never returns an address, phone, e-mail or website — and the deed + // demonstrably carries all four. + const blob = JSON.stringify(seat); + for (const leak of ['ЗАХАРИДЕВО', '032/901102', 'office@pimk-bg.eu', 'п.к.', '4108', 'Родопи']) + assert.ok(!blob.includes(leak), `seat must not carry ${leak}`); +}); + +// ── T3 — the joint-stock bar ────────────────────────────────────────────────── +test('T3 — the legal-form verdict is a UNION of the code and the ЗТРРЮЛНЦ suffix', () => { + const jointByCode = registryLegalForm(deedOf([], { legalForm: 5, fullName: 'НЕЩО СИ' })); + assert.equal(jointByCode.verdict, 'joint_stock'); + + // An UNKNOWN code must not fall through: the suffix decides, and if it cannot, we withhold. + const unknownButEad = registryLegalForm(deedOf([], { legalForm: 99, fullName: '"ГАМА" ЕАД' })); + assert.equal(unknownButEad.verdict, 'joint_stock', 'barred by the mandated suffix'); + + const unknownButEood = registryLegalForm(deedOf([], { legalForm: 99, fullName: '"БЕТА" ЕООД' })); + assert.equal(unknownButEood.verdict, 'closely_held', 'the bar is not blanket'); + + const unknownAndUnreadable = registryLegalForm(deedOf([], { legalForm: 99, fullName: 'НЕЩО' })); + assert.equal(unknownAndUnreadable.verdict, 'unknown', 'unknown withholds — it never publishes'); +}); + +test('T3 — КДА is barred (it is not in the existing closelyHeldForm token list)', () => { + assert.equal( + registryLegalForm(deedOf([], { legalForm: 99, fullName: '"X" КДА' })).verdict, + 'joint_stock', + ); + assert.equal( + registryLegalForm(deedOf([], { legalForm: 6, fullName: 'X' })).verdict, + 'joint_stock', + ); +}); + +test('T3 — a real ООД deed reads as closely held, by code AND by suffix', () => { + const v = registryLegalForm(deedOf([])); + assert.equal(v.code, 4); + assert.equal(v.verdict, 'closely_held'); + assert.equal(v.suffixVerdict, 'closely_held', 'fullName carries the form: "ПИМК" ООД'); +}); + +// ── T7 — the UIC echo ───────────────────────────────────────────────────────── +test('T7 — a deed whose UIC does not echo the request is REFUSED', () => { + // R8: ЕИК leading zeros are significant (public bodies are exactly 000…). If anything on the path + // rewrites the identifier, this is the rail that catches it before a claim is made about the + // wrong company. + assert.doesNotThrow(() => assertUicEcho(deedOf([]), '115536179')); + assert.throws(() => assertUicEcho(deedOf([]), '000696327'), /uic|echo/i); + assert.throws(() => assertUicEcho(deedOf([], { uic: '696327' }), '000696327'), /uic|echo/i); + assert.throws(() => assertUicEcho(deedOf([], { uic: null }), '115536179'), /uic|echo/i); +}); + +// ── the refutation input ────────────────────────────────────────────────────── +test('latestOwnershipEntryDate ignores ERASED ownership fields', () => { + // The trap, present in the first company sampled: CR_F_23_L is live in the current deed, dated + // 2013-07-16, and contains only „Заличено обстоятелство.". Read naively it becomes „latest + // ownership entry: 2013-07-16" and can refute a link it knows nothing about. + const d = deedOf([ + field('CR_F_19_L', F19_THREE, { fieldEntryDate: '2011-05-02T00:00:00' }), + field('CR_F_23_L', F23_ERASED, { fieldEntryDate: '2013-07-16T10:10:07', fieldOperation: 2 }), + ]); + assert.equal(latestOwnershipEntryDate(d), '2011-05-02', 'the erased 2013 entry must not count'); +}); + +test('latestOwnershipEntryDate is null when no live ownership field survives', () => { + const d = deedOf([field('CR_F_23_L', F23_ERASED, { fieldOperation: 2 })]); + assert.equal(latestOwnershipEntryDate(d), null); +}); diff --git a/scripts/tr/evidence.mjs b/scripts/tr/evidence.mjs new file mode 100644 index 00000000..262e31eb --- /dev/null +++ b/scripts/tr/evidence.mjs @@ -0,0 +1,224 @@ +// The evidence ladder (issue #279 §5, ADR-0033 decision 1). Pure: deed in, verdict out. Zero network. +// +// Six outcomes, FIRST MATCH WINS: +// 1 bar_joint_stock АД / ЕАД / КДА — never published, whatever follows +// 2 document the declarant's full name is in a live CR_F_7/18/19/23_L entity +// 3 confirmed declared seat == registered seat, or the declarant wrote the ЕИК +// 4 refuted own stake only: absent from live state, and the live ownership record +// predates the declared period — the register covers it and does not name them +// 5 unknown everything else — held +// 6 outside_tr not in the register at all (ДЗЗД, БУЛСТАТ associations) — held +// +// WHAT THIS ESTABLISHES, precisely: the identity of the COMPANY — that the company behind the declared +// name is the same legal entity as the winner we matched. It does NOT establish that the official owns +// it; that claim comes from their own filed declaration and is not a heuristic at all. The failure mode +// of a wrong match is therefore not an invented ownership claim but a real official attached to the +// WRONG company's ЕИК, contracts and money. Still a false public statement about a named person, which +// is why rung 2 requires a full three-token subset match inside a single registry entity, and why the +// filters that can only withhold are kept (ADR-0033 decision 2). + +import { + liveFields, + fullSubsetMatch, + personTokens, + normalizeSettlement, + registrySeat, + registryLegalForm, + latestOwnershipEntryDate, + OWNERSHIP_FIELDS, + MANAGER_FIELD, + ROLE_FIELDS, +} from './deed.mjs'; + +/** + * Version of the RULES, not of the code. §8's monotonicity gate keys on this: a previously published + * link disappearing under an UNCHANGED rules version is a hard finding; under a changed one it is an + * expected diff. Bump it whenever a rung's meaning changes. + */ +export const RULES_VERSION = 'tr-rules-1'; + +/** Rung 2 needs a real three-part Bulgarian name (ЗГР чл. 9). Two tokens is the homonym risk itself. */ +const MIN_NAME_TOKENS = 3; + +// Court-registered companies were re-registered into the Търговски регистър in a single administrative +// push, which flattened their entry dates into this window. „Strictly before the declared period" +// certifies nothing when the date is an artefact of the migration rather than of the ownership, so the +// refutation rung is suppressed inside it (R13). A suppressed refutation falls through to `unknown` — +// held, not published, which is the safe direction. +const REREGISTRATION_START = '2011-01-01'; +const REREGISTRATION_END = '2012-12-31'; + +/** + * Find the declarant inside the live entities of the given field codes. + * Matching happens per ENTITY — never against a whole field — because one field routinely holds + * several people and combining tokens across them is the libel bug. + * @returns {{nameCode:string, entryNumber:string|null, entryDate:string|null}|null} + */ +function findPerson(deed, name, nameCodes) { + for (const f of liveFields(deed, nameCodes)) { + for (const entity of f.entities) { + if (fullSubsetMatch(name, entity)) { + return { nameCode: f.nameCode, entryNumber: f.entryNumber, entryDate: f.entryDate }; + } + } + } + return null; +} + +/** + * Decide the evidence for one link. + * + * @param {object} input + * @param {object|null} input.deed parsed deed JSON; null only when `outsideTr` + * @param {boolean} [input.outsideTr] the ЕИК is not in the register at all + * @param {string} input.declarantName the office-holder's name as filed + * @param {string[]} [input.declaredSeats] seats declared BY THIS PERSON FOR THIS COMPANY only — + * 4.9% of company-name keys carry more than one distinct + * declared seat, so a company-only key would let one + * person's seat confirm another person's link + * @param {boolean} [input.declaredEik] the declarant wrote the ЕИК in the declaration + * @param {number|null} [input.firstDeclaredYear] + * @param {'self'|'family'} [input.scope] + * @param {boolean} [input.nameGloballyUnique] AND-gate on the WEAKEST rung only + * @returns {{kind:string, publishable:boolean, registryRole:string|null, matchedFact:string|null, + * entryNumber:string|null, entryDate:string|null, rulesVersion:string, + * shortName:boolean, latinInName:boolean}} + */ +export function evidenceVerdict(input) { + const { + deed, + outsideTr = false, + declarantName, + declaredSeats = [], + declaredEik = false, + firstDeclaredYear = null, + scope = 'self', + nameGloballyUnique = true, + } = input; + + const tokens = personTokens(declarantName); + const telemetry = { + rulesVersion: RULES_VERSION, + // Counted, not silently dropped: a refusal we cannot see is a recall hole nobody can size. + shortName: tokens.length < MIN_NAME_TOKENS, + latinInName: /[A-Za-z]/.test(String(declarantName ?? '')), + }; + const verdict = (kind, publishable, extra = {}) => ({ + kind, + publishable, + registryRole: null, + matchedFact: null, + entryNumber: null, + entryDate: null, + ...telemetry, + ...extra, + }); + + if (outsideTr) return verdict('outside_tr', false); + if (deed == null) { + // Fail closed and loudly. A missing deed quietly downgraded to „unknown" is indistinguishable + // from a real hold, and hides a cache gap that should stop the run. + throw new Error('evidenceVerdict: deed is required unless outsideTr is set'); + } + + // ── rung 1 ────────────────────────────────────────────────────────────────── + // A union of the numeric code and the ЗТРРЮЛНЦ suffix; either saying joint-stock bars the link, and + // neither able to say means we withhold rather than guess. + const form = registryLegalForm(deed); + if (form.verdict === 'joint_stock') return verdict('bar_joint_stock', false); + if (form.verdict === 'unknown') return verdict('unknown', false); + + // ── rung 2 ────────────────────────────────────────────────────────────────── + // Only a full three-token name may assert. A Latin homoglyph makes the name a non-match rather than + // a false match — company-name-key.ts's posture, applied to people. + const eligibleForDocument = !telemetry.shortName && !telemetry.latinInName; + if (eligibleForDocument) { + const owner = findPerson(deed, declarantName, OWNERSHIP_FIELDS); + if (owner) { + return verdict('document', true, { + registryRole: 'owner', + matchedFact: `role:owner:${owner.nameCode}`, + entryNumber: owner.entryNumber, + entryDate: owner.entryDate, + }); + } + const manager = findPerson(deed, declarantName, [MANAGER_FIELD]); + if (manager) { + return verdict('document', true, { + registryRole: 'manager', + matchedFact: `role:manager:${manager.nameCode}`, + entryNumber: manager.entryNumber, + entryDate: manager.entryDate, + }); + } + } + + // ── rung 3 ────────────────────────────────────────────────────────────────── + // The weakest publishing rung, so it carries the extra AND-gate: a nationally shared company name + // cannot ride it (ADR-0017's holding, carried forward). The stronger „Документ" rung above is not + // gated — the register named this person in THIS company, which makes the name key moot. + if (nameGloballyUnique) { + if (declaredEik) return verdict('confirmed', true, { matchedFact: 'eik' }); + + const seat = registrySeat(deed); + // Empty NEVER confirms — otherwise every link with no seat data on either side rubber-stamps. + if (seat.settlement !== '') { + const declared = declaredSeats.map(normalizeSettlement).filter((s) => s !== ''); + // R10: the registered seat must have been in force for the declared period. Seats move, and a + // company that relocated INTO the declared settlement afterwards would otherwise confirm falsely. + const seatCoversPeriod = + firstDeclaredYear == null || + seat.entryDate == null || + seat.entryDate <= `${firstDeclaredYear}-12-31`; + if (seatCoversPeriod && declared.includes(seat.settlement)) { + return verdict('confirmed', true, { + matchedFact: `seat:${seat.settlement}`, + entryDate: seat.entryDate, + }); + } + } + } + + // ── rung 4 ────────────────────────────────────────────────────────────────── + // OWN stakes only. For a family stake the registered owner is the relative, whose name we neither + // store nor check, so absence of the OFFICIAL from the deed is evidence of nothing. An early branch, + // not a caller convention. + if (scope === 'self' && firstDeclaredYear != null) { + const stillPresent = findPerson(deed, declarantName, ROLE_FIELDS); + const latest = latestOwnershipEntryDate(deed); + const inRereg = + latest != null && latest >= REREGISTRATION_START && latest <= REREGISTRATION_END; + if (!stillPresent && latest != null && !inRereg && latest < `${firstDeclaredYear}-01-01`) { + return verdict('refuted', false, { entryDate: latest }); + } + } + + // ── rung 5 ────────────────────────────────────────────────────────────────── + return verdict('unknown', false); +} + +/** + * Reconcile a DECLARED termination against the live deed (#279 §7). + * + * „Terminated" is an inference from silence — the commonest cause is a finished mandate, not a sale — + * so ADR-0021 E11's withdrawal is checked against the register before it takes effect. + * + * Phase 1 uses `terminated` ONLY. `label` is computed but deliberately not rendered: „и към днешна + * дата" asserts a present tense about a named person on evidence whose freshness is bounded by the + * cache refresh cycle, and it is deferred behind an LIA addendum (ADR-0033 decision 4). + * + * @returns {{terminated:boolean, label:'owner_today'|'manager_today'|null}} + */ +export function reconcileTermination({ deed, declarantName, scope = 'self' }) { + // Family first, structurally: there is nothing to look for, and looking would be an attempt to + // identify the relative. + if (scope !== 'self' || deed == null) return { terminated: true, label: null }; + + if (findPerson(deed, declarantName, OWNERSHIP_FIELDS)) { + return { terminated: false, label: 'owner_today' }; + } + if (findPerson(deed, declarantName, [MANAGER_FIELD])) { + return { terminated: true, label: 'manager_today' }; + } + return { terminated: true, label: null }; +} diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs new file mode 100644 index 00000000..7c43b797 --- /dev/null +++ b/scripts/tr/evidence.test.mjs @@ -0,0 +1,318 @@ +// node:test — the evidence ladder (ADR-0033 decision 1). Pure: deed in, verdict out. +// +// Six outcomes, first match wins. What each rung is allowed to CONCLUDE is the whole subject: +// the registry proves the identity of the COMPANY, never that the official owns it — the ownership +// claim comes from the official's own filed declaration. So a wrong match here does not invent an +// ownership claim, it attaches a real official to the wrong company's ЕИК, contracts and money. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { RULES_VERSION, evidenceVerdict, reconcileTermination } from './evidence.mjs'; + +const container = (t) => + `

${t}

`; +const joined = (...ts) => ts.map(container).join(`
`); +const ERASED = `
Заличено обстоятелство.
`; + +const fld = (nameCode, htmlData, entryDate = '2011-05-02T00:00:00') => ({ + nameCode, + htmlData, + fieldEntryNumber: '20110502101007', + fieldEntryDate: entryDate, + fieldOperation: 3, +}); +const deed = (fields, over = {}) => ({ + uic: '201122335', + fullName: '"АЛФА СТРОЙ" ООД', + legalForm: 4, + sections: [{ subDeeds: [{ groups: [{ fields }] }] }], + ...over, +}); + +const OWNER_DEED = deed([ + fld('CR_F_19_L', joined('ИВАН ПЕТРОВ ТЕСТОВ, Държава: БЪЛГАРИЯ', 'МАРИЯ СТОЯНОВА ИВАНОВА')), + fld('CR_F_5_L', container('Държава: БЪЛГАРИЯ
Населено място: гр. Пловдив, п.к. 4000')), +]); + +const base = { + deed: OWNER_DEED, + declarantName: 'Иван Петров Тестов', + declaredSeats: [], + declaredEik: false, + firstDeclaredYear: 2021, + scope: 'self', + nameGloballyUnique: true, +}; + +test('RULES_VERSION is a stable, non-empty identifier — §8 hangs off it', () => { + assert.equal(typeof RULES_VERSION, 'string'); + assert.ok(RULES_VERSION.length > 0); +}); + +// ── rung 1: the joint-stock bar wins over everything ────────────────────────── +test('rung 1 — a joint-stock company is barred even when the person IS in the deed', () => { + const ad = deed([fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))], { + legalForm: 5, + fullName: '"ГАМА ИНВЕСТ" АД', + }); + const v = evidenceVerdict({ ...base, deed: ad }); + assert.equal(v.kind, 'bar_joint_stock'); + assert.equal(v.publishable, false); +}); + +test('rung 1 — an UNKNOWN legal form withholds; it never falls through to a lower rung', () => { + const odd = deed([fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))], { + legalForm: 99, + fullName: 'НЕЩО БЕЗ ФОРМА', + }); + const v = evidenceVerdict({ ...base, deed: odd }); + assert.equal(v.kind, 'unknown'); + assert.equal(v.publishable, false); +}); + +// ── rung 2: „Документ" ──────────────────────────────────────────────────────── +test('rung 2 — a full-name match in a live ownership field publishes, with the role kept', () => { + const v = evidenceVerdict(base); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); + assert.equal(v.registryRole, 'owner'); + assert.equal(v.matchedFact, 'role:owner:CR_F_19_L'); + assert.equal(v.entryNumber, '20110502101007'); + assert.equal(v.entryDate, '2011-05-02'); +}); + +test('rung 2 — a manager-only match publishes but records the weaker role', () => { + const mgr = deed([ + fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ, Държава: БЪЛГАРИЯ')), + fld('CR_F_19_L', container('ДРУГО ЛИЦЕ ТУК')), + ]); + const v = evidenceVerdict({ ...base, deed: mgr }); + assert.equal(v.kind, 'document'); + assert.equal(v.registryRole, 'manager'); + assert.equal(v.matchedFact, 'role:manager:CR_F_7_L'); +}); + +test('rung 2 — a TWO-token declarant can never earn „Документ"', () => { + // 46 of 301 measured matches were two-token only, which is exactly the homonym risk. Falls to a + // lower rung rather than publishing on a name that half a register could satisfy. + const two = deed([fld('CR_F_19_L', container('ИВАН ТЕСТОВ, Държава: БЪЛГАРИЯ'))]); + const v = evidenceVerdict({ ...base, deed: two, declarantName: 'Иван Тестов' }); + assert.notEqual(v.kind, 'document'); + assert.equal(v.shortName, true, 'the refusal is counted, not silently dropped'); +}); + +test('rung 2 — the match must fall inside ONE entity (the libel guard, end to end)', () => { + const two = deed([ + fld('CR_F_19_L', joined('ПЕНКО НЕСТОРОВ НЕСТОРОВ', 'ИЛИЯН КОСТАДИНОВ ФИЛИПОВ')), + ]); + const v = evidenceVerdict({ ...base, deed: two, declarantName: 'ПЕНКО КОСТАДИНОВ ФИЛИПОВ' }); + assert.notEqual(v.kind, 'document'); +}); + +test('rung 2 — an ERASED ownership entry cannot produce a document match', () => { + const gone = deed([fld('CR_F_23_L', ERASED, '2013-07-16T10:10:07')]); + const v = evidenceVerdict({ ...base, deed: gone }); + assert.notEqual(v.kind, 'document'); +}); + +test('rung 2 — a Latin homoglyph in the name is a NON-match, and is counted', () => { + // company-name-key.ts deliberately does not fold Cyrillic↔Latin; person names take the same posture. + const v = evidenceVerdict({ ...base, declarantName: 'ИBAH ПЕТРОВ ТЕСТОВ' }); // Latin B, A, H + assert.notEqual(v.kind, 'document'); + assert.equal(v.latinInName, true); +}); + +// ── rung 3: „Потвърдено" ────────────────────────────────────────────────────── +test('rung 3 — a declared seat matching the registered seat confirms the company', () => { + const other = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив, п.к. 4000'), '2015-01-01T00:00:00'), + ]); + const v = evidenceVerdict({ ...base, deed: other, declaredSeats: ['Пловдив'] }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.publishable, true); + assert.equal(v.matchedFact, 'seat:ПЛОВДИВ'); +}); + +test('rung 3 — a declared ЕИК confirms the company on its own', () => { + const other = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК'))]); + const v = evidenceVerdict({ ...base, deed: other, declaredEik: true }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'eik'); +}); + +test('rung 3 — an EMPTY declared seat never confirms', () => { + const noSeat = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК'))]); + const v = evidenceVerdict({ ...base, deed: noSeat, declaredSeats: ['', ' '] }); + assert.notEqual(v.kind, 'confirmed'); +}); + +test('rung 3 — a seat registered AFTER the declared period does not confirm', () => { + // R10, and W0 measured that seats move: a company that relocated INTO the declared settlement after + // the fact would otherwise produce a false „Потвърдено". + const moved = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив'), '2024-06-01T00:00:00'), + ]); + const v = evidenceVerdict({ + ...base, + deed: moved, + declaredSeats: ['Пловдив'], + firstDeclaredYear: 2021, + }); + assert.notEqual(v.kind, 'confirmed'); +}); + +test('rung 3 — the weakest rung ALSO requires global name uniqueness (ADR-0017 carried forward)', () => { + const other = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: other, + declaredSeats: ['Пловдив'], + nameGloballyUnique: false, + }); + assert.notEqual(v.kind, 'confirmed', 'a nationally shared name cannot ride the weakest rung'); +}); + +test('rung 3 — name uniqueness does NOT gate the stronger „Документ" rung', () => { + const v = evidenceVerdict({ ...base, nameGloballyUnique: false }); + assert.equal( + v.kind, + 'document', + 'the registry named the person in THIS company; the name key is moot', + ); +}); + +// ── rung 4: „Оборена" ───────────────────────────────────────────────────────── +test('rung 4 — absent from a deed whose ownership predates the declaration refutes the link', () => { + const older = deed([ + fld('CR_F_19_L', container('СЪВСЕМ ДРУГ СОБСТВЕНИК'), '2015-03-01T00:00:00'), + ]); + const v = evidenceVerdict({ ...base, deed: older, firstDeclaredYear: 2021 }); + assert.equal(v.kind, 'refuted'); + assert.equal(v.publishable, false); +}); + +test('rung 4 — the comparison is date-to-DATE, not date-to-year', () => { + // R17: „strictly before the first declared year" means before YYYY-01-01. An entry inside the first + // declared year does NOT cover the period and must not refute. + const inYear = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2021-06-15T00:00:00')]); + assert.notEqual( + evidenceVerdict({ ...base, deed: inYear, firstDeclaredYear: 2021 }).kind, + 'refuted', + ); + const justBefore = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2020-12-31T00:00:00')]); + assert.equal( + evidenceVerdict({ ...base, deed: justBefore, firstDeclaredYear: 2021 }).kind, + 'refuted', + ); +}); + +test('rung 4 — NEVER applies to a family stake', () => { + // The owner there is the relative, whose name we neither store nor check (ADR-0010 item 4, + // ADR-0032 decision 2), so „the official is not in the deed" says nothing at all. + const older = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2015-03-01T00:00:00')]); + const v = evidenceVerdict({ ...base, deed: older, scope: 'family', firstDeclaredYear: 2021 }); + assert.notEqual(v.kind, 'refuted'); + assert.equal(v.kind, 'unknown'); +}); + +test('rung 4 — suppressed inside the 2011–2012 re-registration window', () => { + // R13: court-registered companies had every entry date flattened into the re-registration window, + // so „strictly before" certifies nothing there. + const flattened = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2011-11-04T00:00:00')]); + const v = evidenceVerdict({ ...base, deed: flattened, firstDeclaredYear: 2021 }); + assert.notEqual(v.kind, 'refuted'); + assert.equal(v.kind, 'unknown'); +}); + +// ── rungs 5 and 6 ───────────────────────────────────────────────────────────── +test('rung 5 — everything else is „Неизвестна" and stays hidden', () => { + const recent = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2023-01-01T00:00:00')]); + const v = evidenceVerdict({ ...base, deed: recent, firstDeclaredYear: 2021 }); + assert.equal(v.kind, 'unknown'); + assert.equal(v.publishable, false); +}); + +test('rung 6 — outside the register is its own outcome, and is not publishable', () => { + const v = evidenceVerdict({ ...base, deed: null, outsideTr: true }); + assert.equal(v.kind, 'outside_tr'); + assert.equal(v.publishable, false); +}); + +test('a missing deed that is NOT marked outside-ТР is an error, not a silent hold', () => { + // Fail closed: a cache gap must be visible, never quietly downgraded to „unknown". + assert.throws(() => evidenceVerdict({ ...base, deed: null, outsideTr: false }), /deed/i); +}); + +// ── the seal ────────────────────────────────────────────────────────────────── +test('matched_fact stays inside the closed vocabulary — it can never carry a name', () => { + const CLOSED = /^(?:seat:[\p{Lu} -]+|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + for (const v of [ + evidenceVerdict(base), + evidenceVerdict({ ...base, deed: deed([fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))]) }), + evidenceVerdict({ ...base, declaredEik: true }), + evidenceVerdict({ + ...base, + deed: deed([ + fld('CR_F_19_L', container('ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]), + declaredSeats: ['Пловдив'], + }), + ]) { + if (v.matchedFact == null) continue; + assert.match(v.matchedFact, CLOSED, `matched_fact escaped the vocabulary: ${v.matchedFact}`); + assert.ok(!/ИВАН|ПЕТРОВ|ТЕСТОВ/.test(v.matchedFact), 'a NAME reached matched_fact'); + } +}); + +test('every verdict carries the rules version that produced it', () => { + assert.equal(evidenceVerdict(base).rulesVersion, RULES_VERSION); +}); + +// ── §7 reconciliation ───────────────────────────────────────────────────────── +test('reconcileTermination — still a registered owner ⇒ NOT terminated', () => { + const r = reconcileTermination({ + deed: OWNER_DEED, + declarantName: 'Иван Петров Тестов', + scope: 'self', + }); + assert.equal(r.terminated, false); + assert.equal(r.label, 'owner_today'); +}); + +test('reconcileTermination — manager only ⇒ terminated as a stake, but the tie continues', () => { + const mgr = deed([ + fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ')), + fld('CR_F_19_L', container('ДРУГ')), + ]); + const r = reconcileTermination({ deed: mgr, declarantName: 'Иван Петров Тестов', scope: 'self' }); + assert.equal(r.terminated, true); + assert.equal(r.label, 'manager_today'); +}); + +test('reconcileTermination — absent from the live deed ⇒ the declared termination stands', () => { + const none = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ'))]); + const r = reconcileTermination({ + deed: none, + declarantName: 'Иван Петров Тестов', + scope: 'self', + }); + assert.equal(r.terminated, true); + assert.equal(r.label, null); +}); + +test('reconcileTermination — a FAMILY stake is never reconciled, by an early branch', () => { + // Structural, not a caller convention: the relative's name is not stored, so there is nothing to + // look for, and looking would be a de-anonymisation attempt. + const r = reconcileTermination({ + deed: OWNER_DEED, + declarantName: 'Иван Петров Тестов', + scope: 'family', + }); + assert.equal(r.terminated, true); + assert.equal(r.label, null); +}); From 98ac4f5aaa03572645ffe38894d70f54b5f18c20 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 22:23:32 +0300 Subject: [PATCH 06/69] =?UTF-8?q?feat(tr):=20the=20deed=20crawler=20?= =?UTF-8?q?=E2=80=94=20paced,=20resumable,=20and=20stopping=20on=20a=20429?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One request per candidate ЕИК, sequential, 1 per 3 s. What it refuses to do is the substance, and every refusal is driven offline through injected I/O. • The pace flag cannot make it faster, only slower. Tuning around a rate limiter empirically is what spec §3.3's „NEVER bulk-scrape" forbids, so --min-interval-ms below the documented 3 s is rejected outright. • A 429 ends the run with exit 2 and records NOTHING about the ЕИК that hit it — that ЕИК is unknown, not absent, and the throttle is a fact about us, not about the company. A later re-run resumes exactly there. • The candidate set is closed: it never follows a link out of a deed. • A checksum-invalid code is dropped before any request. • The returned deed's UIC must echo the request, or it is refused and not cached — otherwise one company's deed lands under another's ЕИК. Two things measured against the live register during this work: „Outside the Търговски регистър" does NOT look like #279 §3 describes. An ЕИК that is not a търговец answers HTTP 200 with a ZERO-BYTE body — not a 404, not HTML. Verified on Община София (000696327): empty on two consecutive requests while a real company returned its full deed in the same window. The rule that keeps R6 honest is therefore the STATUS, not the emptiness: empty under 200 is the register's answer and may be cached permanently; empty under 5xx is a failure and stays transient. Both directions are tested. Without this the ~4 non-търговец ЕИК would never resolve and the run could never exit 0. The breaker threshold is 5, not 10, because the unit is ЕИК and each unresolved one costs the full 5-attempt budget. At 10 the breaker would spend ~50 requests against an already-failing endpoint — the exact volume at which the register was observed to begin a sustained 429, i.e. the safety mechanism would itself have tripped the block. At 5 the worst case is ~25. The breaker test caught itself being fake: it had been generating made-up ЕИК of which only 1 in 10 passed the checksum, so the crawler dropped them all and the assertion passed on zero requests. It now generates genuinely valid codes and asserts the REQUEST budget, not just the candidate count. --- ...-evidence-replaces-name-distinctiveness.md | 6 + scripts/tr/fetch-deeds.mjs | 258 ++++++++++++ scripts/tr/fetch-deeds.test.mjs | 373 ++++++++++++++++++ 3 files changed, 637 insertions(+) create mode 100644 scripts/tr/fetch-deeds.mjs create mode 100644 scripts/tr/fetch-deeds.test.mjs diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md index 2e449127..b3315c99 100644 --- a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -82,6 +82,12 @@ requests, 2026-08-04 — well inside the pacing above, so no 429 was encountered `CR_F_23_L` dated 2013-07-16, which read naïvely becomes „latest ownership entry: 2013-07-16" and feeds the refutation rule. - Seats move. The same company's `CR_F_5_L` shows entry dates 2010-11-04 and 2014-01-23. +- **An ЕИК that is not a търговец answers `HTTP 200` with a ZERO-BYTE body** — not the 404, and not the + HTML, that §3 predicts. Measured on Община София (`000696327`): empty on two consecutive requests, + while a real company returned its full 34,398-byte deed in the same window, so it is the register's + answer rather than an outage. This is what the „извън ТР" rung actually looks like on the wire. The + distinction that keeps R6 honest is therefore the **status**, not the empty body: empty under 200 is + a documented negative and may be cached permanently; empty under 5xx is a failure and stays transient. The as-of capability that `?entryDate=` unlocks would fix the two weakest rungs, but it **re-baselines every control number in #279 §10**. It is therefore deliberately out of scope here and becomes a separate change diff --git a/scripts/tr/fetch-deeds.mjs b/scripts/tr/fetch-deeds.mjs new file mode 100644 index 00000000..0e14915d --- /dev/null +++ b/scripts/tr/fetch-deeds.mjs @@ -0,0 +1,258 @@ +// The deed crawler (issue #279 §3, ADR-0033). One request per candidate ЕИК, sequential and paced. +// +// This is the only component in the project that touches a public register at volume, so what it +// REFUSES to do is the substance: +// +// • It never goes faster than 1 request / 3 s, and the flag that sets the pace cannot be used to go +// faster — only slower. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the +// limiter is the operator's only way to state a preference, and tuning around it empirically is +// what that rule exists to prevent. +// • It never retries a 429. The block is sustained (see client.mjs), so a 429 ends the run with its +// own exit code and records NOTHING about the ЕИК that hit it — that ЕИК is unknown, not absent. +// • It never follows a link out of a deed. The candidate set is closed: whatever the caller passes +// in, nothing more. This is what keeps a bounded lookup from drifting into a crawl. +// • It only writes „outside the register" on a DOCUMENTED negative — measured to be an HTTP 200 +// with an empty body, not the 404 the issue predicts. A 5xx or a timeout is transient, and caching +// it as permanent would turn an outage into data that §8 never revisits. +// +// Resumable by construction: the cache is consulted first, so an interrupted run picks up exactly +// where it stopped and a complete cache costs zero requests. + +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { assertTrScratchIgnored, TR_DB, TR_RAW, safeEik, deedPath } from './paths.mjs'; +import { eikChecksumValid } from './eik.mjs'; +import { deedUrl, politeTrGet, RateLimitError, httpsGet } from './client.mjs'; +import { openCache, upsertDeed, markOutsideTr, pendingEiks } from './cache.mjs'; +import { + assertUicEcho, + registryLegalForm, + registrySeat, + latestOwnershipEntryDate, +} from './deed.mjs'; + +/** The documented polite pace: 1 request / 3 s (#279 §3). A floor, never a target to tune down. */ +export const MIN_INTERVAL_MS = 3000; + +/** + * Consecutive unresolved ЕИК before the run gives up rather than keep hammering. + * + * Deliberately small, because the unit is ЕИК and not requests: each unresolved candidate costs up to + * 5 attempts (#279 §3's documented retry budget), so the breaker's real cost is `BREAKER_TRIP × 5` + * requests against an endpoint that is already failing. At 10 that would be ~50 — the exact volume at + * which the register was observed to start returning a sustained 429. At 5 the worst case is ~25, + * which the same observation saw pass without a block. + */ +export const BREAKER_TRIP = 5; +/** Attempts per candidate, per #279 §3. Exported so the breaker's request budget is derivable. */ +export const TRIES_PER_EIK = 5; + +export function parseTrOptions(argv) { + const get = (name, def) => { + const i = argv.indexOf(`--${name}`); + return i >= 0 && argv[i + 1] ? argv[i + 1] : def; + }; + const posInt = (raw, name) => { + const n = Number(raw); + if (!Number.isInteger(n) || n < 1) + throw new Error(`--${name} must be a positive integer, got ${JSON.stringify(raw)}`); + return n; + }; + + const eiksFile = get('eiks-file', ''); + if (!eiksFile) throw new Error('--eiks-file is required (the closed candidate set)'); + + const limitRaw = get('limit', ''); + const intervalRaw = get('min-interval-ms', ''); + const minIntervalMs = intervalRaw ? posInt(intervalRaw, 'min-interval-ms') : MIN_INTERVAL_MS; + // Slower is always allowed; faster is not a knob. Making this un-passable is the point. + if (minIntervalMs < MIN_INTERVAL_MS) { + throw new Error( + `--min-interval-ms may not go below the documented pace of ${MIN_INTERVAL_MS}ms — ` + + `the register's rate limit is a stated preference, not an obstacle to tune around`, + ); + } + const maxAgeRaw = get('max-age-days', ''); + return { + eiksFile, + limit: limitRaw ? posInt(limitRaw, 'limit') : Infinity, + minIntervalMs, + maxAgeDays: maxAgeRaw ? posInt(maxAgeRaw, 'max-age-days') : null, + }; +} + +/** Read the closed candidate set: one ЕИК per line, blanks and `#` comments ignored. */ +export function readEiksFile(file) { + return fs + .readFileSync(file, 'utf8') + .split('\n') + .map((l) => l.trim()) + .filter((l) => l !== '' && !l.startsWith('#')); +} + +function atomicWrite(file, buf) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const tmp = `${file}.tmp-${process.pid}`; + fs.writeFileSync(tmp, buf); + fs.renameSync(tmp, file); +} + +/** + * Crawl the candidate ЕИК. Returns the intended process exit code, so the decision is testable + * without a global side effect: + * 0 — every candidate resolved (or the run was deliberately bounded by --limit) + * 1 — at least one candidate is unresolved (transient failure, refused deed, breaker tripped) + * 2 — the register rate-limited us; the run stopped and nothing was marked + * + * Every I/O edge is injectable so the whole policy is exercised offline. + */ +export async function run({ + httpGet = httpsGet, + sleep = (ms) => new Promise((r) => setTimeout(r, ms)), + now = () => new Date(), + guard = assertTrScratchIgnored, + dbFile = TR_DB, + rawDir = TR_RAW, + argv = process.argv, +} = {}) { + guard(); + const { eiksFile, limit, minIntervalMs, maxAgeDays } = parseTrOptions(argv); + + const requested = readEiksFile(eiksFile); + // A shape- or checksum-invalid code is dropped BEFORE any request: it cannot name a real company, + // so asking about it would spend the register's budget to learn nothing. + const candidates = []; + let invalid = 0; + for (const raw of requested) { + try { + const eik = safeEik(raw); + if (!eikChecksumValid(eik)) throw new Error('checksum'); + candidates.push(eik); + } catch { + invalid++; + } + } + + const db = openCache(dbFile); + try { + const pending = pendingEiks(db, candidates, { maxAgeDays, now: now() }); + const todo = Number.isFinite(limit) ? pending.slice(0, limit) : pending; + console.log( + `candidates ${candidates.length} · invalid ${invalid} · cached ${candidates.length - pending.length} · to fetch ${todo.length}`, + ); + + let unresolved = 0; + let consecutive = 0; + let first = true; + + for (const eik of todo) { + if (!first) await sleep(minIntervalMs); // pace BETWEEN requests, not before the first + first = false; + + let res; + try { + res = await politeTrGet(deedUrl(eik), { httpGet, sleep, tries: TRIES_PER_EIK }); + } catch (err) { + if (err instanceof RateLimitError) { + // Stop the whole run. Recording anything here would attribute the register's throttle to + // this ЕИК, which is a fact about us, not about the company. + console.error(`${err.message}\nSTOPPING — re-run later; progress so far is cached.`); + return 2; + } + console.error( + ` ${eik}: ${err instanceof Error ? err.message : err} (transient, not cached)`, + ); + unresolved++; + consecutive++; + if (consecutive >= BREAKER_TRIP) { + console.error(`breaker: ${consecutive} consecutive failures — aborting the run`); + return 1; + } + continue; + } + + // ── the documented negatives ──────────────────────────────────────────── + // MEASURED 2026-08-05: an ЕИК that is not a търговец answers **HTTP 200 with a ZERO-BYTE body**, + // not a 404 and not the HTML #279 §3 predicts. Verified on Община София (000696327): empty on + // two consecutive requests, while a real company returned its full 34,398-byte deed in the same + // window — so it is the register's answer, not an outage. + // + // The distinction that keeps R6 honest is the STATUS, not the empty body: an empty body under + // 200 is the register saying „no deed"; an empty body under 5xx is a failure and stays + // transient. Getting this backwards either caches a false negative forever or leaves ~4 ЕИК + // permanently unresolved so the run can never exit 0. + if (res.status === 200 && res.body.length === 0) { + markOutsideTr(db, eik, 'HTTP 200, empty body — no deed in the Търговски регистър', now()); + consecutive = 0; + continue; + } + if (res.status === 404) { + markOutsideTr(db, eik, 'HTTP 404 — not in the Търговски регистър (BULSTAT/ДЗЗД?)', now()); + consecutive = 0; + continue; + } + if (res.status !== 200) { + console.error(` ${eik}: HTTP ${res.status} after retries (transient, not cached)`); + unresolved++; + consecutive++; + if (consecutive >= BREAKER_TRIP) { + console.error(`breaker: ${consecutive} consecutive failures — aborting the run`); + return 1; + } + continue; + } + + let deed; + try { + deed = JSON.parse(res.body.toString('utf8')); + // The deed we got back must be the deed we asked for, or every claim derived from it names + // the wrong company (R8). + assertUicEcho(deed, eik); + } catch (err) { + console.error(` ${eik}: REFUSED — ${err instanceof Error ? err.message : err}`); + unresolved++; + consecutive++; + continue; + } + + // The raw response is the ONLY place names live; it stays under git-ignored scratch. + atomicWrite(deedPath(eik, rawDir), res.body); + const seat = registrySeat(deed); + upsertDeed(db, { + eik, + status: 'fetched', + httpStatus: 200, + fetchedAt: now().toISOString(), + rawPath: path.relative(rawDir, deedPath(eik, rawDir)), + bodySha256: crypto.createHash('sha256').update(res.body).digest('hex'), + legalFormCode: registryLegalForm(deed).code, + legalFormVerdict: registryLegalForm(deed).verdict, + seatNormalized: seat.settlement || null, + seatEntryDate: seat.entryDate, + latestOwnEntryDate: latestOwnershipEntryDate(deed), + }); + consecutive = 0; + } + + if (unresolved > 0) { + console.error(`${unresolved} candidate(s) unresolved — the cache is incomplete`); + return 1; + } + return 0; + } finally { + db.close(); + } +} + +// CLI entry. Kept off the import path so the module stays testable. +if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) { + run() + .then((code) => { + process.exitCode = code; + }) + .catch((err) => { + console.error(err); + process.exitCode = 1; + }); +} diff --git a/scripts/tr/fetch-deeds.test.mjs b/scripts/tr/fetch-deeds.test.mjs new file mode 100644 index 00000000..8b8296d7 --- /dev/null +++ b/scripts/tr/fetch-deeds.test.mjs @@ -0,0 +1,373 @@ +// node:test — the deed crawler, driven entirely through injected I/O. No network, no real scratch. +// +// This is the only component that touches a public register at volume, so what it must NOT do is the +// substance: never exceed the pace, never retry a 429, never turn a transient wall into permanent +// data, and never re-request what it already holds. Spec §3.3 permits a bounded per-ЕИК lookup and +// forbids bulk scraping; the difference between the two is enforced here. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { + parseTrOptions, + run, + MIN_INTERVAL_MS, + BREAKER_TRIP, + TRIES_PER_EIK, +} from './fetch-deeds.mjs'; + +const DEED = (uic) => ({ + uic, + fullName: '"АЛФА СТРОЙ" ООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: `

ИВАН ПЕТРОВ ТЕСТОВ

`, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + { + nameCode: 'CR_F_5_L', + htmlData: `

Населено място: гр. Пловдив

`, + fieldEntryNumber: '20110502101008', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], +}); + +const ok = (uic) => ({ + status: 200, + headers: {}, + body: Buffer.from(JSON.stringify(DEED(uic)), 'utf8'), +}); +const status = (s) => ({ status: s, headers: {}, body: Buffer.from('') }); + +// Two valid ЕИК (real checksums) plus one that is shape-valid but checksum-invalid. +const A = '201122335'; +const B = '203445566'; +const C = '204556676'; +const BAD_CHECKSUM = '201122336'; + +function ctx() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-crawl-')); + return { + dir, + dbFile: path.join(dir, 'tr-cache.sqlite'), + rawDir: path.join(dir, 'deeds'), + cleanup: () => fs.rmSync(dir, { recursive: true, force: true }), + }; +} +const eiksFile = (dir, eiks) => { + const f = path.join(dir, 'eiks.txt'); + fs.writeFileSync(f, eiks.join('\n') + '\n'); + return f; +}; + +/** Drive run() with a recording transport. `routes` maps ЕИК → response (or a function of attempt). */ +function harness(c, eiks, routes, extraArgv = []) { + const calls = []; + const waits = []; + const httpGet = async (url) => { + const eik = url.split('/').pop(); + calls.push(eik); + const r = routes[eik]; + return typeof r === 'function' ? r(calls.filter((e) => e === eik).length) : (r ?? status(404)); + }; + return { + calls, + waits, + promise: run({ + httpGet, + sleep: async (ms) => void waits.push(ms), + now: () => new Date('2026-08-05T12:00:00Z'), + guard: () => {}, + dbFile: c.dbFile, + rawDir: c.rawDir, + argv: ['node', 'fetch-deeds.mjs', '--eiks-file', eiksFile(c.dir, eiks), ...extraArgv], + }), + }; +} + +const openCacheRO = (dbFile) => new DatabaseSync(dbFile); +const rows = (dbFile) => { + const db = openCacheRO(dbFile); + const r = db.prepare('SELECT eik, status, outside_reason FROM deeds ORDER BY eik').all(); + db.close(); + return r; +}; + +// ── options ─────────────────────────────────────────────────────────────────── +test('parseTrOptions: defaults are the documented polite pace', () => { + const o = parseTrOptions(['node', 'x', '--eiks-file', '/tmp/e.txt']); + assert.equal(o.eiksFile, '/tmp/e.txt'); + assert.equal(o.limit, Infinity); + assert.equal(o.minIntervalMs, MIN_INTERVAL_MS); + assert.ok(MIN_INTERVAL_MS >= 3000, 'the documented pace is 1 request / 3 s'); +}); + +test('parseTrOptions: rejects a pace FASTER than the documented one', () => { + // Tuning around a limiter empirically is precisely what spec §3.3 forbids; make it un-passable. + assert.throws( + () => parseTrOptions(['node', 'x', '--eiks-file', '/e', '--min-interval-ms', '100']), + /min-interval/i, + ); +}); + +test('parseTrOptions: --eiks-file is required, and numeric flags are validated', () => { + assert.throws(() => parseTrOptions(['node', 'x']), /eiks-file/i); + for (const bad of ['0', '-1', 'abc', '1.5']) + assert.throws( + () => parseTrOptions(['node', 'x', '--eiks-file', '/e', '--limit', bad]), + /limit/i, + bad, + ); +}); + +// ── pacing ──────────────────────────────────────────────────────────────────── +test('requests are sequential and spaced by at least the documented interval', async () => { + const c = ctx(); + try { + const h = harness(c, [A, B, C], { [A]: ok(A), [B]: ok(B), [C]: ok(C) }); + assert.equal(await h.promise, 0); + assert.deepEqual(h.calls, [A, B, C], 'sequential, in order — never concurrent'); + const paces = h.waits.filter((w) => w >= MIN_INTERVAL_MS); + assert.ok(paces.length >= 2, `expected a pace wait between requests, got ${h.waits.join(',')}`); + } finally { + c.cleanup(); + } +}); + +// ── 429 ─────────────────────────────────────────────────────────────────────── +test('a 429 ends the run with exit 2 and marks NOTHING', async () => { + const c = ctx(); + try { + const h = harness(c, [A, B, C], { [A]: ok(A), [B]: status(429), [C]: ok(C) }); + assert.equal(await h.promise, 2, 'a rate-limit block is its own exit code'); + assert.deepEqual(h.calls, [A, B], 'stops AT the 429 — C is never requested'); + // B must not be recorded at all: it is unknown, not absent, and certainly not outside the register. + assert.deepEqual( + rows(c.dbFile).map((r) => r.eik), + [A], + ); + } finally { + c.cleanup(); + } +}); + +test('after a 429 the run resumes exactly where it stopped', async () => { + const c = ctx(); + try { + const first = harness(c, [A, B, C], { [A]: ok(A), [B]: status(429), [C]: ok(C) }); + assert.equal(await first.promise, 2); + const second = harness(c, [A, B, C], { [A]: ok(A), [B]: ok(B), [C]: ok(C) }); + assert.equal(await second.promise, 0); + assert.deepEqual(second.calls, [B, C], 'A is already cached — not re-requested'); + } finally { + c.cleanup(); + } +}); + +// ── resumability ────────────────────────────────────────────────────────────── +test('a re-run over a complete cache makes ZERO requests', async () => { + const c = ctx(); + try { + assert.equal(await harness(c, [A, B], { [A]: ok(A), [B]: ok(B) }).promise, 0); + const again = harness(c, [A, B], { [A]: ok(A), [B]: ok(B) }); + assert.equal(await again.promise, 0); + assert.deepEqual(again.calls, [], 'nothing to fetch'); + } finally { + c.cleanup(); + } +}); + +// ── permanence ──────────────────────────────────────────────────────────────── +test('an empty 200 is the register saying „no deed" and is cached as outside-ТР', async () => { + // MEASURED against the live API: an ЕИК that is not a търговец (Община София, 000696327) answers + // HTTP 200 with a ZERO-BYTE body — not the 404 or HTML #279 §3 predicts. Reproduced twice, with a + // real company returning its full deed in the same window, so it is an answer and not an outage. + const c = ctx(); + try { + const empty = { status: 200, headers: {}, body: Buffer.alloc(0) }; + assert.equal(await harness(c, [A], { [A]: empty }).promise, 0); + const [row] = rows(c.dbFile); + assert.equal(row.status, 'outside_tr'); + assert.match(row.outside_reason, /empty body/i); + } finally { + c.cleanup(); + } +}); + +test('an empty body under a 5xx stays TRANSIENT — the status decides, not the emptiness', async () => { + // The pair that keeps R6 honest. Both responses have a zero-byte body; only the 200 is an answer. + const c = ctx(); + try { + assert.equal(await harness(c, [A], { [A]: status(503) }).promise, 1); + assert.deepEqual(rows(c.dbFile), [], 'a 5xx must never become permanent „outside ТР"'); + } finally { + c.cleanup(); + } +}); + +test('a 404 is a DOCUMENTED negative and is cached as outside-ТР', async () => { + const c = ctx(); + try { + assert.equal(await harness(c, [A], { [A]: status(404) }).promise, 0); + const [row] = rows(c.dbFile); + assert.equal(row.status, 'outside_tr'); + assert.match(row.outside_reason, /404/); + } finally { + c.cleanup(); + } +}); + +test('a persistent 5xx is TRANSIENT and is never cached as outside-ТР', async () => { + // R6: „outside the register" is permanent by intent. Writing it after a transient wall turns a + // temporary outage into permanent data that §8 never re-examines. + const c = ctx(); + try { + const code = await harness(c, [A], { [A]: status(503) }).promise; + assert.equal(code, 1, 'an unresolved ЕИК makes the run incomplete'); + assert.deepEqual(rows(c.dbFile), [], 'nothing may be recorded from a 5xx'); + } finally { + c.cleanup(); + } +}); + +test('a deed whose UIC does not echo the request is REFUSED, not cached', async () => { + // R8, at the crawl boundary: if anything rewrote the identifier we would be caching one company's + // deed under another company's ЕИК, and every downstream claim about it would name the wrong firm. + const c = ctx(); + try { + const code = await harness(c, [A], { [A]: ok('999999999') }).promise; + assert.equal(code, 1); + assert.deepEqual(rows(c.dbFile), []); + assert.deepEqual(fs.existsSync(c.rawDir) ? fs.readdirSync(c.rawDir) : [], []); + } finally { + c.cleanup(); + } +}); + +test('a checksum-invalid ЕИК is skipped without ever being requested', async () => { + const c = ctx(); + try { + const h = harness(c, [BAD_CHECKSUM, A], { [A]: ok(A) }); + assert.equal(await h.promise, 0); + assert.deepEqual(h.calls, [A], 'the invalid code costs the register nothing'); + } finally { + c.cleanup(); + } +}); + +// ── output ──────────────────────────────────────────────────────────────────── +test('the raw deed is written under the raw dir and the index records only non-PII', async () => { + const c = ctx(); + try { + assert.equal(await harness(c, [A], { [A]: ok(A) }).promise, 0); + assert.deepEqual(fs.readdirSync(c.rawDir), [`${A}.json`]); + const db = openCacheRO(c.dbFile); + const row = db.prepare('SELECT * FROM deeds WHERE eik = ?').get(A); + db.close(); + assert.equal(row.status, 'fetched'); + assert.equal(row.legal_form_verdict, 'closely_held'); + assert.equal(row.seat_normalized, 'ПЛОВДИВ'); + assert.equal(row.body_sha256.length, 64); + // The raw file holds the names; the index must not. + assert.ok(!JSON.stringify(row).includes('ТЕСТОВ')); + assert.ok(fs.readFileSync(path.join(c.rawDir, `${A}.json`), 'utf8').includes('ТЕСТОВ')); + } finally { + c.cleanup(); + } +}); + +test('--limit bounds a run without marking the remainder as anything', async () => { + const c = ctx(); + try { + const h = harness(c, [A, B, C], { [A]: ok(A), [B]: ok(B), [C]: ok(C) }, ['--limit', '2']); + assert.equal(await h.promise, 0, 'a deliberately bounded run is not an incomplete one'); + assert.deepEqual(h.calls, [A, B]); + assert.deepEqual( + rows(c.dbFile).map((r) => r.eik), + [A, B], + ); + } finally { + c.cleanup(); + } +}); + +/** + * N genuinely checksum-valid 9-digit ЕИК. Generated, not hand-written: an earlier version of the + * breaker test below used made-up codes, of which only 1 in 10 was valid — so the crawler dropped + * them all before requesting anything and the assertion passed on zero calls. A test that exercises + * nothing is worse than no test (ADR-0027). + */ +function validEiks(n) { + const control = (p8) => { + const d = [...p8].map(Number); + let s = d.reduce((a, x, i) => a + x * (i + 1), 0) % 11; + if (s === 10) { + s = d.reduce((a, x, i) => a + x * (i + 3), 0) % 11; + if (s === 10) s = 0; + } + return s; + }; + const out = []; + for (let i = 0; out.length < n; i++) { + const p8 = String(20000000 + i); + out.push(p8 + control(p8)); + } + return out; +} + +test('the ЕИК generator used by the breaker test really produces valid codes', () => { + const eiks = validEiks(15); + assert.equal(eiks.length, 15); + assert.equal(new Set(eiks).size, 15); + // Proves the breaker test below actually reaches the network path rather than being filtered out. + assert.deepEqual( + eiks.filter((e) => e.length !== 9), + [], + ); +}); + +test('the circuit breaker aborts a sustained wall of failures', async () => { + // A long run against a broken endpoint must stop hammering, even though each individual 5xx is + // „transient". Distinct from the 429 path: that stops instantly and deliberately. + const c = ctx(); + try { + const many = validEiks(BREAKER_TRIP + 10); + const routes = Object.fromEntries(many.map((e) => [e, status(503)])); + const h = harness(c, many, routes); + assert.equal(await h.promise, 1); + + const attempted = [...new Set(h.calls)]; + assert.equal(attempted.length, BREAKER_TRIP, 'the breaker cuts the run at its threshold'); + assert.ok(attempted.length < many.length, 'and therefore short of the full candidate set'); + // The number that actually matters is REQUESTS, not candidates: each unresolved ЕИК costs the + // full retry budget, so the breaker's real cost is BREAKER_TRIP × TRIES_PER_EIK. Keep that under + // the ~50 at which the register was observed to start returning a sustained 429 — otherwise the + // safety mechanism is itself what trips the block. + assert.equal(h.calls.length, BREAKER_TRIP * TRIES_PER_EIK); + assert.ok( + h.calls.length <= 25, + `a failing run must not spend ${h.calls.length} requests before giving up`, + ); + assert.deepEqual(rows(c.dbFile), [], 'a wall of 5xx records nothing'); + } finally { + c.cleanup(); + } +}); From 800b2c1545edcdfe4a566108ddf6f7aa478f18a8 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 23:23:21 +0300 Subject: [PATCH 07/69] =?UTF-8?q?feat(db):=20migration=200006=20=E2=80=94?= =?UTF-8?q?=20the=20Trade=20Register=20evidence=20seal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A side table, not columns on interest_links, for three reasons that are easy to forget once the code is written: SQLite's ADD COLUMN has no IF NOT EXISTS and these migrations are applied by a bare `d1 execute --file` with no ledger, so a re-apply must be a no-op; load.mjs rebuilds the CACBG tables from 0003 alone, so any column added here would have to be duplicated into 0003 and kept in step forever; and §8 describes the seal as an attached artefact of a link. A seal is written for EVERY link, not only published ones — the seals on held and withdrawn links are what make the review queue reviewable. matched_fact is a closed vocabulary ('seat:' | 'role:owner:' | 'role:manager:' | 'eik') and must never carry the matched name; the deed's names are read only to produce a boolean and never leave git-ignored scratch. A schema cannot enforce that, so the audit will. Wired everywhere the 0003 table list already is, and the two FK orderings are opposite by necessity: the seal ships AFTER interest_links and is wiped BEFORE it. D1 enforces foreign keys, so getting either backwards breaks the re-seed. • ship-related-persons.mjs — TABLES and WIPE_ORDER, with a test asserting both orderings and a third asserting every shipped table is also wiped (a table that ships but is never wiped accumulates seals for links that no longer exist). • related-persons.ts CONFLICT_TABLES — so an environment without 0006 degrades to an empty surface rather than a 500. • load.mjs — applies 0003 AND 0006, and drops the seal first. • BOTH related-persons-data.yml and deploy.yml hardcode migration filenames; each needed its own step. Missing either leaves a D1 without the table. ship-reseed.test.mjs builds its fixture from the migrations, so 0006 joins it — the new table is part of the FK graph that test exists to police. --- .github/workflows/deploy.yml | 6 +++ .github/workflows/related-persons-data.yml | 6 +++ .../0006_interest_link_evidence.sql | 41 ++++++++++++++ packages/db/src/migrations.test.ts | 54 +++++++++++++++++++ packages/db/src/queries/related-persons.ts | 3 ++ scripts/cacbg/load.mjs | 10 ++++ scripts/ship-related-persons.mjs | 4 ++ scripts/ship-related-persons.test.mjs | 21 ++++++++ scripts/ship-reseed.test.mjs | 5 ++ 9 files changed, 150 insertions(+) create mode 100644 packages/db/migrations/0006_interest_link_evidence.sql diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2c3f42ae..f52927c9 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -145,6 +145,12 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0003_related_persons_foundation.sql + # 0006 attaches the Trade Register evidence seal (#279, ADR-0033). Both migrations are + # applied by name here because wrangler's migration ledger is empty on this D1 (the base + # schema was created out-of-band), so `d1 migrations apply` would collide on 0000. + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0006_interest_link_evidence.sql # The base schema was created out-of-band with `d1 execute --file`, so wrangler's migration # ledger is empty and `d1 migrations apply` would collide on 0000. Probe the actual table # instead. SQLite has no `ADD COLUMN IF NOT EXISTS`; a completion-marker table is created only diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index deb24884..fa84f21e 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -185,6 +185,12 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0003_related_persons_foundation.sql + # 0006 attaches the Trade Register evidence seal (#279, ADR-0033). Both migrations are + # applied by name here because wrangler's migration ledger is empty on this D1 (the base + # schema was created out-of-band), so `d1 migrations apply` would collide on 0000. + pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0006_interest_link_evidence.sql - name: Ship свързани-лица tables → D1 # ship-related-persons.mjs shells out to `wrangler d1 execute` with cwd=apps/web. Two things it diff --git a/packages/db/migrations/0006_interest_link_evidence.sql b/packages/db/migrations/0006_interest_link_evidence.sql new file mode 100644 index 00000000..a7b83033 --- /dev/null +++ b/packages/db/migrations/0006_interest_link_evidence.sql @@ -0,0 +1,41 @@ +-- Trade Register evidence seal for a свързани-лица link (#279 §8, ADR-0033). +-- +-- WHY A SIDE TABLE and not columns on interest_links, since both were on the table: +-- 1. SQLite's ALTER TABLE ... ADD COLUMN has no IF NOT EXISTS, and migrations here are applied by a +-- bare `wrangler d1 execute --file` with no applied-migrations tracking (see the workflow steps), +-- so a re-apply MUST be a no-op. `CREATE TABLE IF NOT EXISTS` is; ADD COLUMN is not. +-- 2. scripts/cacbg/load.mjs rebuilds the CACBG tables from 0003 ALONE. A column added here would have +-- to be duplicated into 0003 and kept in step with it forever, and the day the two diverge the +-- loader silently drops the evidence for every link it writes. +-- 3. §8 describes the seal as an attached artefact of a link, which is what this is. +-- +-- A seal is written for EVERY link, not only published ones. The seals on held and withdrawn links are +-- what make the review queue reviewable — without them „why is this one hidden?" has no answer. +-- +-- PII rail: `matched_fact` is a CLOSED VOCABULARY — 'seat:' | 'role:owner:' | +-- 'role:manager:' | 'eik'. It must NEVER carry the matched name. The registry deed's names are +-- read only to produce a boolean and never leave git-ignored scratch (ADR-0033 decision 5); storing one +-- here would put third-party personal data on the served surface, which #279 §9 forbids outright. The +-- audit enforces the vocabulary with a pattern check, because a schema cannot. + +CREATE TABLE IF NOT EXISTS interest_link_evidence ( + link_key TEXT PRIMARY KEY REFERENCES interest_links(link_key), + -- document | confirmed | refuted | bar_joint_stock | unknown | outside_tr (ADR-0033 decision 1) + evidence_kind TEXT NOT NULL, + registry_role TEXT, -- owner | manager | NULL — only meaningful for evidence_kind='document' + matched_fact TEXT, -- the closed vocabulary above. NEVER a name. + -- TEXT, not INTEGER: a fieldEntryNumber like 20130716101007 is already 14 digits and exceeds the + -- exact-integer range once it round-trips through JSON/JS. + entry_number TEXT, + entry_date TEXT, -- ISO date of the registry entry the evidence rests on + lookup_date TEXT NOT NULL, -- when the deed was fetched — the freshness bound on the claim + rules_version TEXT NOT NULL, -- evidence.mjs RULES_VERSION; §8's monotonicity gate keys on this + -- live | terminated_owner_still | terminated_manager_still | terminated. + -- RE-DERIVED on every run and deliberately NOT part of the seal's permanence: it asserts a present + -- tense about a named person, and its freshness is bounded by the cache refresh cycle (ADR-0033 R3). + live_status TEXT NOT NULL, + sealed_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- The surface filters on evidence_kind ('document' | 'confirmed' publish); the audit scans by kind too. +CREATE INDEX IF NOT EXISTS idx_ile_kind ON interest_link_evidence(evidence_kind); diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index e5296aaa..b92c174b 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -11,6 +11,7 @@ const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); const migration2 = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); const backfill = resolve(root, 'scripts/backfill-current-value-currency.sql'); const precompute = resolve(root, 'scripts/precompute.sql'); @@ -150,4 +151,57 @@ describe('served migrations', () => { rmSync(dir, { recursive: true, force: true }); } }); + + // 0006 attaches the Trade Register evidence seal to a link (#279, ADR-0033). A SIDE TABLE rather than + // columns on interest_links, for two reasons that are easy to forget: SQLite's ADD COLUMN has no + // IF NOT EXISTS and migrations are applied by a bare `d1 execute --file` with no tracking, so a + // re-apply must be a no-op; and load.mjs rebuilds the CACBG tables from 0003 alone, so any column + // added here would have to be duplicated into 0003 and kept in step forever. + it('0006 adds the evidence seal and re-applying it is a no-op', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-0006-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + for (const m of [migration0, migration1, migration2, migration3, migration6]) + readScript(dbPath, m); + + expect( + sqlite( + dbPath, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='interest_link_evidence';", + ).trim(), + ).toBe('1'); + + // A seal for a real link survives a second apply — the migration must not drop and recreate. + sqlite( + dbPath, + `INSERT INTO persons (id, name) VALUES ('person:a', 'Иван Петров Тестов'); + INSERT INTO bidders (id, name, eik_normalized, eik_valid) + VALUES ('eik:201122335', 'АЛФА СТРОЙ ООД', '201122335', 1); + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:k', 'k', 'person:a', 'eik:201122335', '201122335', 'АЛФА СТРОЙ ООД', + 'test', 'document', 'owns', 'private_ownership', 'published'); + INSERT INTO interest_link_evidence + (link_key, evidence_kind, matched_fact, lookup_date, rules_version, live_status) + VALUES ('k', 'document', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live');`, + ); + readScript(dbPath, migration6); // idempotent re-apply + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_link_evidence;').trim()).toBe('1'); + + // The FK is real: a seal for a link that does not exist is rejected. D1 enforces foreign keys, + // so this is what stops the ship path inserting seals before (or after wiping) their links. + expect(() => + sqlite( + dbPath, + `PRAGMA foreign_keys=ON; + INSERT INTO interest_link_evidence + (link_key, evidence_kind, matched_fact, lookup_date, rules_version, live_status) + VALUES ('nope', 'document', 'eik', '2026-08-05', 'tr-rules-1', 'live');`, + ), + ).toThrow(/FOREIGN KEY/i); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/packages/db/src/queries/related-persons.ts b/packages/db/src/queries/related-persons.ts index 05c3f228..ce15ff2b 100644 --- a/packages/db/src/queries/related-persons.ts +++ b/packages/db/src/queries/related-persons.ts @@ -19,6 +19,9 @@ const CONFLICT_TABLES = [ 'declared_interests', 'interest_link_authorities', 'related_persons_internal', + // 0006 (#279, ADR-0033). Listed here for the same reason as the rest: on an environment where 0006 + // has not been applied yet, the evidence join must degrade to an empty surface rather than a 500. + 'interest_link_evidence', ]; // „D1_ERROR: no such table: interest_links: SQLITE_ERROR" → capture the table name and test membership. const MISSING_TABLE = /no such table:\s*(?:main\.)?"?([a-z_]+)"?/i; diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 4dc360b2..984ead20 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -30,6 +30,13 @@ const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '. const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); const STAGING = process.env.CACBG_STAGING || path.join(ROOT, 'scratch/cacbg/staging'); const MIGRATION = path.join(ROOT, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// 0006 attaches the Trade Register evidence seal (#279, ADR-0033). Applied here as well as 0003 +// because this loader rebuilds the CACBG tables from the migrations on every run — a seal table +// missing from the work DB would make every evidence write fail at ship time instead of at load. +const MIGRATION_EVIDENCE = path.join( + ROOT, + 'packages/db/migrations/0006_interest_link_evidence.sql', +); const REPORT = path.join(STAGING, 'findings.md'); const MATCHER_VERSION = 'cnk-1+classify-1'; // bump when the normalizer or classify logic changes const { companyNameKey, isMatchableKey } = @@ -71,6 +78,8 @@ const usedSuppressions = new Set(); // Full idempotent rebuild that also picks up schema changes: drop the CACBG tables (children first — // FK-safe) and re-apply the migration. Nothing to preserve — suppressions are external now. for (const t of [ + // FIRST: interest_link_evidence references interest_links, so it must go before its parent. + 'interest_link_evidence', 'interest_link_authorities', 'interest_links', 'declared_interests', @@ -80,6 +89,7 @@ for (const t of [ ]) db.exec(`DROP TABLE IF EXISTS ${t}`); db.exec(fs.readFileSync(MIGRATION, 'utf8')); +db.exec(fs.readFileSync(MIGRATION_EVIDENCE, 'utf8')); // A link is suppressed when its fingerprint is in the list. Only compute the HMAC when the list is // non-empty (size>0 ⇒ salt present, else the loader above threw), so the empty common path skips crypto. const isSuppressed = (linkKey) => { diff --git a/scripts/ship-related-persons.mjs b/scripts/ship-related-persons.mjs index 54da33de..6ecd1e59 100644 --- a/scripts/ship-related-persons.mjs +++ b/scripts/ship-related-persons.mjs @@ -28,6 +28,8 @@ export const TABLES = [ 'declarations', 'declared_interests', 'interest_links', + // AFTER interest_links: a seal references its link, so inserting it first fails the FK (#279). + 'interest_link_evidence', 'interest_link_authorities', ]; // DELETE order for the pre-insert wipe — children before parents. related_persons_internal (PII, never @@ -35,6 +37,8 @@ export const TABLES = [ // carrying internal rows would block DELETE FROM declarations. export const WIPE_ORDER = [ 'interest_link_authorities', + // BEFORE interest_links, for the mirror reason: deleting a link whose seal survives fails the FK. + 'interest_link_evidence', 'related_persons_internal', 'interest_links', 'declared_interests', diff --git a/scripts/ship-related-persons.test.mjs b/scripts/ship-related-persons.test.mjs index b15612d1..76e8c18a 100644 --- a/scripts/ship-related-persons.test.mjs +++ b/scripts/ship-related-persons.test.mjs @@ -10,6 +10,7 @@ import { sqlLiteral, sqlIdent, TABLES, + WIPE_ORDER, } from './ship-related-persons.mjs'; test('sqlLiteral escapes quotes, strips NUL, and NULLs non-finite/absent', () => { @@ -70,11 +71,31 @@ test('TABLES ships parents before children and covers the served related-persons 'declared_interests', 'interest_links', 'interest_link_authorities', + 'interest_link_evidence', ]) { assert.ok(TABLES.includes(t), `missing ${t}`); } }); +test('the evidence seal ships AFTER the links it references, and is wiped BEFORE them', () => { + // D1 enforces foreign keys, so ordering is not cosmetic: inserting a seal before its link fails, + // and deleting a link while a seal still references it fails the re-seed at interest_links. + assert.ok( + TABLES.indexOf('interest_link_evidence') > TABLES.indexOf('interest_links'), + 'a seal inserted before its link violates the FK', + ); + assert.ok( + WIPE_ORDER.indexOf('interest_link_evidence') < WIPE_ORDER.indexOf('interest_links'), + 'a link deleted while its seal survives violates the FK', + ); +}); + +test('every shipped table is wiped, and every wiped table is real', () => { + // A table added to TABLES but forgotten in WIPE_ORDER accumulates stale rows on every re-ship — + // the surface would then carry evidence for links that no longer exist. + for (const t of TABLES) assert.ok(WIPE_ORDER.includes(t), `${t} ships but is never wiped`); +}); + test('assertD1TargetAuthorized: declared env + allowlisted name + (name↔id) before a remote wipe (T48)', () => { const ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'; const ok = { diff --git a/scripts/ship-reseed.test.mjs b/scripts/ship-reseed.test.mjs index 4864029d..9c5b6617 100644 --- a/scripts/ship-reseed.test.mjs +++ b/scripts/ship-reseed.test.mjs @@ -15,6 +15,9 @@ import { wipeSql, TABLES } from './ship-related-persons.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); const MIG = resolve(HERE, '..', 'packages/db/migrations/0003_related_persons_foundation.sql'); +// 0006 adds interest_link_evidence, which references interest_links — so it is part of the FK graph +// this test exists to police, and the wipe must clear it before its parent (#279, ADR-0033). +const MIG_EVIDENCE = resolve(HERE, '..', 'packages/db/migrations/0006_interest_link_evidence.sql'); // Stub only the two 0000 tables 0002's FKs reference (bidders, authorities) — a PK column is all a FK needs — // then apply the real 0002 and seed one row down every FK chain so a parent delete has live children. @@ -25,6 +28,7 @@ function newPopulatedDb() { CREATE TABLE bidders(id TEXT PRIMARY KEY); CREATE TABLE authorities(id TEXT PRIMARY KEY); .read ${MIG} +.read ${MIG_EVIDENCE} INSERT INTO bidders(id) VALUES('eik:1'); INSERT INTO authorities(id) VALUES('auth:1'); INSERT INTO persons(id,name) VALUES('p1','П Тест'); @@ -33,6 +37,7 @@ INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VAL INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('il1','p1|1','p1','eik:1','1','e','v1','B_distinctive','owns','published'); INSERT INTO interest_link_authorities(link_key,authority_id,authority_name) VALUES('p1|1','auth:1','A'); INSERT INTO related_persons_internal(id,declaration_id,related_name,related_kind) VALUES('rp1','d1','X','related_person'); +INSERT INTO interest_link_evidence(link_key,evidence_kind,matched_fact,lookup_date,rules_version,live_status) VALUES('p1|1','document','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'); `; execFileSync('sqlite3', ['-bail', db], { input: setup, stdio: 'pipe' }); return { dir, db }; From 5f64f5cfda40c2a6bf65e2ece7e2e9ad6bf62249 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 07:56:13 +0300 Subject: [PATCH 08/69] =?UTF-8?q?fix(cacbg):=20closelyHeldForm=20did=20not?= =?UTF-8?q?=20exclude=20=D0=9A=D0=94=D0=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit КДА — командитно дружество с акции — issues shares like an АД, so the same two reasons apply: the shareholder book is not public, making a declared parcel unverifiable, and the parcel may be immaterial. It was absent from both JOINT_STOCK and FORM_TOKENS, so a КДА read as CLOSELY HELD and its declared holder could reach the public ownership surface as an owner — the exact class of claim the exclusion exists to prevent. Two independent effects, hence both lists: JOINT_STOCK is the materiality bar, and FORM_TOKENS feeds the content-word count in nameDistinctiveness, where a form token miscounted as a content word inflates distinctiveness toward publishing. Found while implementing #279, whose rung 1 names КДА explicitly alongside АД and ЕАД, but the gap predates it and is a bug on its own — hence a separate commit. The mirror cases stay correct: „КДА-ТРЕЙД ЕООД" and „КДА ГРУП ООД" are ООД. --- scripts/cacbg/classify.mjs | 8 +++++--- scripts/cacbg/classify.test.mjs | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/scripts/cacbg/classify.mjs b/scripts/cacbg/classify.mjs index fe979345..f8e7dcd7 100644 --- a/scripts/cacbg/classify.mjs +++ b/scripts/cacbg/classify.mjs @@ -16,6 +16,7 @@ const FORM_TOKENS = new Set([ 'КД', 'СД', 'АДСИЦ', + 'КДА', 'КООПЕРАЦИЯ', 'ФОНДАЦИЯ', 'СДРУЖЕНИЕ', @@ -71,15 +72,16 @@ const norm = (s) => .replace(/[\s.\-–—]+/g, ' ') .trim(); -// Joint-stock / listed legal form (АД / ЕАД / АДСИЦ) as the TRAILING form token. In BG company names the +// Joint-stock / share-issuing legal form (АД / ЕАД / АДСИЦ / КДА) as the TRAILING form token. In BG company names the // legal form is always the suffix, so anchor to the end (optionally followed by quotes/whitespace); a whole // token bounded on the left by string edge, whitespace or quotes — NOT hyphens/dots, so „АД-ХОК ЕООД" (a // hyphenated ООД name) is not misread. Anchoring to the suffix is what stops „АД ГРУП ООД" (an ООД whose // NAME begins with the token „АД") being wrongly excluded as joint-stock — the form there is ООД. -const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ)[\s"„“”«»]*$/u; +const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; /** * Materiality by legal form. The public ownership surface is CLOSELY-HELD companies only (ООД/ЕООД/ЕТ/ - * КД/СД/ДЗЗД or a form-unspecified name from the closely-held table). Joint-stock forms (АД/ЕАД/АДСИЦ) are + * КД/СД/ДЗЗД or a form-unspecified name from the closely-held table). Joint-stock forms (АД/ЕАД/АДСИЦ/КДА, + * the last a командитно дружество с акции — it issues shares, so it belongs with them) are * public-float securities — a declared parcel of listed shares is NOT a material ownership conflict, and * presenting it as one defames (the „11 Trace shares → €88M" trap). Excludes only an explicit АД-form token, * so it withholds rather than fabricates. @returns {boolean} true ⇒ material/closely-held. diff --git a/scripts/cacbg/classify.test.mjs b/scripts/cacbg/classify.test.mjs index 7d2bddd1..b62d3c7e 100644 --- a/scripts/cacbg/classify.test.mjs +++ b/scripts/cacbg/classify.test.mjs @@ -71,6 +71,25 @@ test('closelyHeldForm: ООД/ЕООД/ЕТ material; АД/ЕАД/АДСИЦ (li assert.equal(closelyHeldForm('АД СТИЛ ЕООД'), true); }); +test('closelyHeldForm: КДА (командитно дружество с акции) is joint-stock and must be excluded', () => { + // КДА issues shares like an АД — the shareholder book is not public, so a declared parcel is neither + // verifiable nor necessarily material, which is the whole basis of the exclusion. It was missing from + // both JOINT_STOCK and FORM_TOKENS, so a КДА read as closely-held and its holder could be published + // as an owner. #279 rung 1 names it explicitly alongside АД and ЕАД. + assert.equal(closelyHeldForm('ФИНАНС КДА'), false); + assert.equal(closelyHeldForm('"АЛФА ИНВЕСТ" КДА'), false); + assert.equal(closelyHeldForm('АЛФА КДА, гр. София'), false); // seat suffix must not rescue it + // …and the mirror: „КДА" inside a word or as a leading token is not the form. + assert.equal(closelyHeldForm('КДА-ТРЕЙД ЕООД'), true); + assert.equal(closelyHeldForm('КДА ГРУП ООД'), true); +}); + +test('nameDistinctiveness: КДА counts as a legal form, not a content word', () => { + // FORM_TOKENS feeds the content-word count. A form token counted as content inflates distinctiveness, + // which is the direction that publishes prematurely. + assert.equal(nameDistinctiveness('ФИНАНС КДА'), 'generic'); +}); + test('closelyHeldForm: a trailing седалище after the form does not flip an АД to closely-held (libel)', () => { // The declarant appended the seat to the name cell. Without stripping it, the end-anchored form test // misses the АД and returns closely-held=true → a listed-АД parcel presented as a material conflict. From 4226ae348437d5e0d33e88c3414d43a892b0ad53 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 08:10:06 +0300 Subject: [PATCH 09/69] feat(cacbg): the evidence ladder replaces the publish tiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decision swap (#279, ADR-0033). load.mjs no longer asks whether a declared name looks distinctive; it asks what the Trade Register says about that ЕИК. publishTier() and seatConfirmed() are deleted — one production caller each — and publish_tier now carries the evidence kind. FAIL CLOSED, twice. A missing cache refuses the load outright. A PARTIAL cache refuses it too, and that is the one that matters: it does not fail loudly downstream. An 80%-restored cache yields ~80 published links, which clears the ship floor of 50, ships a decimated surface and wipes the rest of the live links. --allow-partial-tr is the deliberate override, and an uncached ЕИК under it is „unknown" (held), never a reason to publish. The workflow now passes --min-links 250 explicitly rather than relying on the default of 50. tr-census.mjs is deleted. Its promote() moved C_hold → published on pure name uniqueness — identity by name coincidence, which is what this change abolishes. Two design errors the fixtures caught, both about how far ADR-0017's gate reaches: • Gating the declared-ЕИК leg on name uniqueness would discard the strongest identifier we have exactly where the name is useless — a фирма backing two ЕИК is the case ADR-0028 was written for. Now ungated. • Gating the seat leg on nameDistinctiveness would empty the rung of its purpose: rescuing a GENERIC name is what the seat is for. Only national non-uniqueness blocks it. A consequence worth stating plainly: #279 NARROWS the family surface. The registered owner of a family stake is the relative, whose name we deliberately never store, so rung 2 („Документ") can never fire for a family link by construction — its identity can only be confirmed by something the OFFICIAL declared, the seat or the ЕИК. ADR-0032's decision is untouched (family publishes on the named surface exactly like self); it now needs the same evidence as everything else. The fixture's seat-less family case is therefore held, and a seat-carrying one was added as the positive control, without which „family published: 0" would be indistinguishable from a dead path. Fixture declarants gained patronymics. Bulgarian names are three-part by statute (ЗГР чл. 9) and rung 2 requires the full triple, so two-token fixture names were not merely unrealistic — they could never earn „Документ" and were quietly testing the wrong path. A seal is written for every link, held and withdrawn included, so the review queue can explain itself; live_status is re-derived each run and never sealed. load.mjs also emits candidate-eiks.txt — every resolved ЕИК, not just published ones — which is the crawler's closed input set. --- .github/workflows/related-persons-data.yml | 9 +- .github/workflows/scripts-test.yml | 2 +- scripts/cacbg/classify.mjs | 26 +- scripts/cacbg/classify.test.mjs | 15 - scripts/cacbg/load-collision.test.mjs | 75 +++- .../cacbg/load-divestment-nonwinner.test.mjs | 92 +++- scripts/cacbg/load-homonym.test.mjs | 92 +++- scripts/cacbg/load.mjs | 194 ++++++++- scripts/cacbg/load.test.mjs | 392 +++++++++++++++--- scripts/cacbg/tr-census.mjs | 195 --------- scripts/cacbg/tr-census.test.mjs | 121 ------ scripts/tr/evidence.mjs | 11 +- scripts/tr/evidence.test.mjs | 32 ++ 13 files changed, 811 insertions(+), 445 deletions(-) delete mode 100644 scripts/cacbg/tr-census.mjs delete mode 100644 scripts/cacbg/tr-census.test.mjs diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index fa84f21e..16f35c7e 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -204,7 +204,14 @@ jobs: node scripts/wrangler-render.mjs apps/web/wrangler.jsonc cp apps/web/wrangler.deploy.jsonc apps/web/wrangler.jsonc export PATH="$GITHUB_WORKSPACE/apps/web/node_modules/.bin:$GITHUB_WORKSPACE/node_modules/.bin:$PATH" - node scripts/ship-related-persons.mjs --work-db data/work/backfill.sqlite --remote --yes + # --min-links is passed EXPLICITLY, not left at its default of 50. Under #279 the surface is + # measured at ~329 links, so 50 no longer protects anything: a partially-restored Trade + # Register cache yields roughly 80 published links, which clears a floor of 50, ships a + # decimated surface and wipes the rest. The loader's coverage gate is the first defence and + # this is the second (ADR-0033 decision 7). Lower it deliberately, in a PR, if a genuinely + # smaller surface is ever expected. + node scripts/ship-related-persons.mjs --work-db data/work/backfill.sqlite --remote --yes \ + --min-links 250 - name: Reindex search projection → officials searchable (same batch the cron runs) # ship writes ONLY the свързани-лица domain tables (interest_links, persons, …), NOT search_index. diff --git a/.github/workflows/scripts-test.yml b/.github/workflows/scripts-test.yml index d3112c48..81dd4ec4 100644 --- a/.github/workflows/scripts-test.yml +++ b/.github/workflows/scripts-test.yml @@ -28,7 +28,7 @@ jobs: # so no `pnpm install` is required. - run: node --test scripts/*.test.mjs - # свързани-лица pipeline tests — the CACBG leg (parse/classify/load/audit/tr-census/extract) and the + # свързани-лица pipeline tests — the CACBG leg (parse/classify/load/audit/extract) and the # Търговски регистър leg (scripts/tr: ЕИК checksum, HTTP client, deed cache). These use node:sqlite # (DatabaseSync) and import the shared companyNameKey .ts via the register-ts resolve hook, so they # need Node 24 (node:sqlite + native TS type-stripping, no experimental flag) rather than the Node 22 diff --git a/scripts/cacbg/classify.mjs b/scripts/cacbg/classify.mjs index f8e7dcd7..8bc07bce 100644 --- a/scripts/cacbg/classify.mjs +++ b/scripts/cacbg/classify.mjs @@ -96,26 +96,12 @@ export function closelyHeldForm(name) { ); } -/** Seat proof: declared seat and winner settlement both present and equal ⇒ same entity (deterministic). */ -export function seatConfirmed(declSeat, winnerSettlement) { - const a = norm(declSeat); - const b = norm(winnerSettlement); - return a.length > 0 && b.length > 0 && a === b; -} - -/** - * Publish tier for a single-winner-ЕИК match: - * 'A_eik' — a declarant-provided ЕИК resolved the winner: the national unique identifier, - * deterministic even behind a generic/colliding name. Assigned by the loader for - * declared_eik matches (not this function — it needs the match method); ADR-0016. - * 'A_seat' — seat-confirmed: deterministic, publishable even for generic names. - * 'B_distinctive' — single-ЕИК + structurally distinctive name: publishable (disclosed heuristic). - * 'C_hold' — generic name, no seat/ЕИК proof: withhold pending TR name-census. - */ -export function publishTier({ seatOk, distinctiveness }) { - if (seatOk) return 'A_seat'; - return distinctiveness === 'distinctive' ? 'B_distinctive' : 'C_hold'; -} +// seatConfirmed() and publishTier() lived here until #279. The publish tiers they produced +// (A_seat / B_distinctive / C_hold) are superseded by the Trade Register evidence ladder in +// scripts/tr/evidence.mjs — identity now rests on a checkable registry fact rather than on the shape +// of the declared name (ADR-0033, superseding ADR-0009). The seat comparison moved with it, because a +// declared seat is now matched against the REGISTERED seat rather than against the winner row's +// settlement column. nameDistinctiveness above survives, narrowed to an AND-gate on the weakest rung. /** * Temporal relation of a contract to the years a stake was declared (asset decls are annual snapshots). diff --git a/scripts/cacbg/classify.test.mjs b/scripts/cacbg/classify.test.mjs index b62d3c7e..e0e8b2fa 100644 --- a/scripts/cacbg/classify.test.mjs +++ b/scripts/cacbg/classify.test.mjs @@ -2,8 +2,6 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { nameDistinctiveness, - seatConfirmed, - publishTier, temporalStatus, localityToken, closelyHeldForm, @@ -32,19 +30,6 @@ test('nameDistinctiveness: numbers / Latin / ≥3 words are distinctive; bare 1- assert.equal(nameDistinctiveness('„ДОМИНО" ЕООД'), 'generic'); // quoted single word + form }); -test('seatConfirmed: equal non-empty seats confirm; empty or mismatched do not', () => { - assert.equal(seatConfirmed('Шумен', 'ШУМЕН'), true); - assert.equal(seatConfirmed('София', 'Пловдив'), false); - assert.equal(seatConfirmed('', 'София'), false); // sparse winner/declared seat never confirms - assert.equal(seatConfirmed('София', ''), false); -}); - -test('publishTier: seat proof wins; else distinctiveness decides publish vs hold', () => { - assert.equal(publishTier({ seatOk: true, distinctiveness: 'generic' }), 'A_seat'); - assert.equal(publishTier({ seatOk: false, distinctiveness: 'distinctive' }), 'B_distinctive'); - assert.equal(publishTier({ seatOk: false, distinctiveness: 'generic' }), 'C_hold'); -}); - test('temporalStatus: contract within declared-year span is contemporaneous', () => { assert.equal(temporalStatus([2020, 2021, 2022], 2021), 'contemporaneous'); assert.equal(temporalStatus([2020, 2022], 2024), 'after_last_decl'); diff --git a/scripts/cacbg/load-collision.test.mjs b/scripts/cacbg/load-collision.test.mjs index 32991193..8d709ec2 100644 --- a/scripts/cacbg/load-collision.test.mjs +++ b/scripts/cacbg/load-collision.test.mjs @@ -16,7 +16,7 @@ import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad() { execFileSync( @@ -24,15 +24,81 @@ function runLoad() { ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], { cwd: ROOT, - env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + }, stdio: 'pipe', }, ); } + +/** + * Minimal Trade Register evidence for this fixture (#279, ADR-0033). Publishing now rests on a registry + * fact, so a loader test without a cache would only ever exercise the fail-closed path. Each winner's + * deed names its own declarant as съдружник, which is the „Документ" rung. + */ +function buildTrCache(owners) { + fs.mkdirSync(TR_RAW, { recursive: true }); + const cache = new DatabaseSync(TR_DB); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + for (const [eik, name] of Object.entries(owners)) { + const deed = { + uic: eik, + fullName: '"ФИКС" ЕООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: `

${name}

`, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], + }; + fs.writeFileSync(path.join(TR_RAW, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code,legal_form_verdict,latest_own_entry_date) VALUES(?,?,?,?,?,?,?,?)', + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + 4, + 'closely_held', + '2011-05-02', + ); + } + cache.close(); +} const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-collision-')); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); fs.mkdirSync(STAGING, { recursive: true }); @@ -92,6 +158,11 @@ before(() => { holdings.map((h) => JSON.stringify(h)).join('\n') + '\n', ); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + + buildTrCache({ + 100000001: 'ИВАН ПЪРВИ ТЕСТОВ', + 200000002: 'ПЕТЪР ВТОРИ ПРОБЕН', + }); }); after(() => fs.rmSync(dir, { recursive: true, force: true })); diff --git a/scripts/cacbg/load-divestment-nonwinner.test.mjs b/scripts/cacbg/load-divestment-nonwinner.test.mjs index 226be07c..fac967b5 100644 --- a/scripts/cacbg/load-divestment-nonwinner.test.mjs +++ b/scripts/cacbg/load-divestment-nonwinner.test.mjs @@ -17,7 +17,7 @@ import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad() { execFileSync( @@ -25,15 +25,86 @@ function runLoad() { ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], { cwd: ROOT, - env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + }, stdio: 'pipe', }, ); } + +/** + * Minimal Trade Register evidence for this fixture (#279, ADR-0033). Publishing now rests on a registry + * fact, so a loader test without a cache would only ever exercise the fail-closed path. Each winner's + * deed names its own declarant as съдружник — the „Документ" rung. + */ +function buildTrCache(owners) { + fs.mkdirSync(TR_RAW, { recursive: true }); + const cache = new DatabaseSync(TR_DB); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + for (const [eik, names] of Object.entries(owners)) { + const html = [] + .concat(names) + .map((n) => `

${n}

`) + .join(`
`); + const deed = { + uic: eik, + fullName: '"ФИКС" ЕООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: html, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], + }; + fs.writeFileSync(path.join(TR_RAW, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code,legal_form_verdict,latest_own_entry_date) VALUES(?,?,?,?,?,?,?,?)', + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + 4, + 'closely_held', + '2011-05-02', + ); + } + cache.close(); +} + const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-divest-')); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); fs.mkdirSync(STAGING, { recursive: true }); @@ -64,7 +135,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Диан Дивестов', + person: 'Диан Иванов Дивестов', position: '', entity: 'ДИВ ТЕХ 5 ЕООД', kind: 'shares', @@ -80,7 +151,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Диан Дивестов', + person: 'Диан Иванов Дивестов', position: '', entity: 'НЕПОБЕДИМ КОМПАНИ ООД', kind: 'shares', @@ -99,7 +170,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Верен Държателев', + person: 'Верен Иванов Държателев', position: '', entity: 'ДРУГ ВИН 6 ЕООД', kind: 'shares', @@ -114,6 +185,13 @@ before(() => { holdings.map((h) => JSON.stringify(h)).join('\n') + '\n', ); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + + buildTrCache({ + // Диан DIVESTED, so the live deed must name somebody else — otherwise §7's reconciliation + // correctly overturns his declared termination and the case stops testing what it is for. + 100000001: 'НОВ ИВАНОВ СОБСТВЕНИК', + 200000002: 'ВЕРЕН ИВАНОВ ДЪРЖАТЕЛЕВ', + }); // filings.jsonl — one record per declaration (as extract.mjs emits it), carrying the declaration type. The // divest horizon is built from this: Диан's 2022 assets declaration (listing only the non-winner) advances // his assets horizon to 2022 → the 2019 ДИВ ТЕХ 5 winner stake is withdrawn. Верен has only a 2019 filing. @@ -143,8 +221,8 @@ test('a later NON-winner ownership filing still withdraws a divested winner stak ) .get(eik, person); - const dian = link('100000001', 'Диан Дивестов'); - const veren = link('200000002', 'Верен Държателев'); + const dian = link('100000001', 'Диан Иванов Дивестов'); + const veren = link('200000002', 'Верен Иванов Държателев'); // The divested winner stake is dated to its last declaration and excluded from the public surface. assert.equal(dian.status, 'withdrawn'); diff --git a/scripts/cacbg/load-homonym.test.mjs b/scripts/cacbg/load-homonym.test.mjs index 395b62ed..7f38b12b 100644 --- a/scripts/cacbg/load-homonym.test.mjs +++ b/scripts/cacbg/load-homonym.test.mjs @@ -18,19 +18,94 @@ import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad() { execFileSync( 'node', ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], - { cwd: ROOT, env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, stdio: 'pipe' }, + { + cwd: ROOT, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + }, + stdio: 'pipe', + }, ); } + +/** + * Minimal Trade Register evidence for this fixture (#279, ADR-0033). Publishing now rests on a registry + * fact, so a loader test without a cache would only ever exercise the fail-closed path. Each winner's + * deed names its own declarant as съдружник — the „Документ" rung. + */ +function buildTrCache(owners) { + fs.mkdirSync(TR_RAW, { recursive: true }); + const cache = new DatabaseSync(TR_DB); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + for (const [eik, names] of Object.entries(owners)) { + const html = [] + .concat(names) + .map((n) => `

${n}

`) + .join(`
`); + const deed = { + uic: eik, + fullName: '"ФИКС" ЕООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: html, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], + }; + fs.writeFileSync(path.join(TR_RAW, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code,legal_form_verdict,latest_own_entry_date) VALUES(?,?,?,?,?,?,?,?)', + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + 4, + 'closely_held', + '2011-05-02', + ); + } + cache.close(); +} + const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-homonym-')); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); fs.mkdirSync(STAGING, { recursive: true }); @@ -59,7 +134,7 @@ before(() => { template: 'assets', category: '', institution: 'ОБЩИНА СОФИЯ', - person: 'Георги Иванов', + person: 'Георги Иванов Петров', position: 'Кмет', entity: 'ВИН ЕДНО 5 ЕООД', kind: 'shares', @@ -77,7 +152,7 @@ before(() => { template: 'assets', category: '', institution: 'МИНИСТЕРСТВО НА ТЕСТА', - person: 'Георги Иванов', + person: 'Георги Иванов Петров', position: 'Директор', entity: 'ВИН ДВЕ 6 ЕООД', kind: 'shares', @@ -95,7 +170,7 @@ before(() => { template: 'assets', category: '', institution: 'ОБЩИНА СОФИЯ', - person: 'Георги Иванов', + person: 'Георги Иванов Петров', position: 'Кмет', entity: 'ВИН ЕДНО 5 ЕООД', kind: 'shares', @@ -110,6 +185,11 @@ before(() => { holdings.map((h) => JSON.stringify(h)).join('\n') + '\n', ); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + + buildTrCache({ + 100000001: 'ГЕОРГИ ИВАНОВ ПЕТРОВ', + 200000002: 'ГЕОРГИ ИВАНОВ ПЕТРОВ', + }); }); after(() => fs.rmSync(dir, { recursive: true, force: true })); @@ -120,7 +200,7 @@ test('same-named officials at different institutions do NOT merge into one perso // Two DISTINCT persons named „Георги Иванов" — one per institution — never one merged identity. const persons = db - .prepare("SELECT id, name FROM persons WHERE name = 'Георги Иванов' ORDER BY id") + .prepare("SELECT id, name FROM persons WHERE name = 'Георги Иванов Петров' ORDER BY id") .all(); assert.equal(persons.length, 2, 'two distinct namesake officials, not one merged person'); assert.notEqual(persons[0].id, persons[1].id); diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 984ead20..5a37ae91 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -16,12 +16,13 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { nameDistinctiveness, - seatConfirmed, - publishTier, temporalStatus, localityToken, closelyHeldForm, } from './classify.mjs'; +import { openCache, readDeed, coverage } from '../tr/cache.mjs'; +import { TR_DB, TR_RAW } from '../tr/paths.mjs'; +import { evidenceVerdict, reconcileTermination, RULES_VERSION } from '../tr/evidence.mjs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; import { fingerprint, loadSuppressions, SUPPRESSION_KEY_VERSION } from './suppressions.mjs'; import { canonicalInstitution } from './institutions.mjs'; @@ -38,7 +39,15 @@ const MIGRATION_EVIDENCE = path.join( 'packages/db/migrations/0006_interest_link_evidence.sql', ); const REPORT = path.join(STAGING, 'findings.md'); -const MATCHER_VERSION = 'cnk-1+classify-1'; // bump when the normalizer or classify logic changes +// Bumped for #279: classify-2 (КДА added to the joint-stock bar) + tr-1 (identity now rests on a +// Trade Register fact, not on name distinctiveness). RULES_VERSION versions the EVIDENCE rules +// separately — §8's monotonicity gate keys on that one, not on this. +const MATCHER_VERSION = 'cnk-1+classify-2+tr-1'; +const TR_CACHE_DB = process.env.TR_CACHE_DB || TR_DB; +const TR_RAW_DIR = process.env.TR_RAW_DIR || TR_RAW; +// A deliberate, logged override for the coverage gate below. Without it a single permanently +// unreachable ЕИК would deadlock the pipeline forever; with it, the operator states that they know. +const ALLOW_PARTIAL_TR = process.argv.includes('--allow-partial-tr'); const { companyNameKey, isMatchableKey } = await import('../../packages/shared/src/company-name-key.ts'); @@ -403,6 +412,12 @@ const insLink = db.prepare( const insILA = db.prepare( 'INSERT OR IGNORE INTO interest_link_authorities(link_key,authority_id,authority_name,contract_count,value_eur,own) VALUES(?,?,?,?,?,?)', ); +// The evidence seal (#279 §8, migration 0006). Written for EVERY link, not only published ones — the +// seals on held and withdrawn links are what let the review queue explain itself. `matched_fact` is a +// closed vocabulary and must NEVER carry a name; the audit enforces that with a pattern check. +const insEvidence = db.prepare( + 'INSERT OR REPLACE INTO interest_link_evidence(link_key,evidence_kind,registry_role,matched_fact,entry_number,entry_date,lookup_date,rules_version,live_status) VALUES(?,?,?,?,?,?,?,?,?)', +); // classify one authority (whose name may be a ';'-joined blob) against the official's institutions. // exact = deterministic name equality; name_contains/locality = DISCLOSED heuristics (candidate, not proof). const OWN_RANK = { exact: 3, name_contains: 2, locality: 1, none: 0 }; @@ -426,6 +441,73 @@ function authOwn(authorityName, instNorms, instNormsLong, locTokens) { } // Distinct officials who declared each company (ЕИК). A private interest has ONE owner-declarant; a // public body's board is declared by MANY rotating members — the deterministic ex-officio tell (ADR-0019). +// ── Trade Register evidence: the candidate set, the fail-closed gate, and the deed reader ───────── +// Identity now rests on a checkable registry fact rather than on the shape of the declared name +// (#279, ADR-0033). Two consequences the loader has to enforce, both fail-closed: +// +// 1. NO cache ⇒ throw. Publishing without evidence is precisely what this change abolishes. +// 2. PARTIAL cache ⇒ throw. This is the silent one. An 80%-restored cache yields roughly 80 +// published links, which is ABOVE ship-related-persons.mjs's floor of 50 — so it would sail +// through that guard, ship a decimated surface, and wipe the rest of the live links. +// +// The candidate set is every resolved ЕИК across ALL aggregates, not just the ones that end up +// published: a link held for want of evidence still needs its deed to say so. +const candidateEiks = [...new Set([...agg.values()].map((r) => r.eik))].sort(); +fs.writeFileSync(path.join(STAGING, 'candidate-eiks.txt'), candidateEiks.join('\n') + '\n'); + +if (!fs.existsSync(TR_CACHE_DB)) { + db.close(); + throw new Error( + `REFUSE TO LOAD: no Trade Register cache at ${TR_CACHE_DB}. Every publishing decision now rests ` + + `on a registry fact (ADR-0033); without the cache there is no evidence to rest on. Run ` + + `scripts/tr/fetch-deeds.mjs --eiks-file ${path.join(STAGING, 'candidate-eiks.txt')} first.`, + ); +} +const trCache = openCache(TR_CACHE_DB); +const trCoverage = coverage(trCache, candidateEiks); +console.log( + `TR cache: ${trCoverage.covered}/${trCoverage.wanted} covered ` + + `(fetched ${trCoverage.fetched}, outside ТР ${trCoverage.outsideTr}, missing ${trCoverage.missing})`, +); +if (trCoverage.missing > 0 && !ALLOW_PARTIAL_TR) { + trCache.close(); + db.close(); + throw new Error( + `REFUSE TO LOAD: the Trade Register cache covers ${trCoverage.covered} of ${trCoverage.wanted} ` + + `candidate ЕИК. A partial cache does not fail loudly downstream — it publishes a decimated ` + + `surface that still clears the ship floor and then wipes the rest of the live links. Finish ` + + `the crawl, or pass --allow-partial-tr to state that a smaller surface is intended.`, + ); +} + +// Deeds are read from git-ignored scratch and cached in memory for the run. The parsed deed carries +// third-party names; they are used ONLY inside evidenceVerdict's boolean comparisons and never reach +// a column, a log line or the report (ADR-0033 decision 5). +// The lookup date sealed on every link: when the evidence was gathered, not when it was interpreted. +// It is the freshness bound the methodology page has to state, so it comes from the cache rather than +// from `now` — a re-run over an unchanged cache must not make the evidence look fresher than it is. +const trLookupDate = (() => { + const row = trCache.prepare('SELECT MAX(fetched_at) m FROM deeds').get(); + return row?.m ? String(row.m).slice(0, 10) : new Date().toISOString().slice(0, 10); +})(); + +const deedCache = new Map(); +function deedFor(eik) { + if (deedCache.has(eik)) return deedCache.get(eik); + const row = readDeed(trCache, eik); + let entry; + if (!row) entry = { deed: null, outsideTr: false, missing: true }; + else if (row.status === 'outside_tr') entry = { deed: null, outsideTr: true, missing: false }; + else { + const file = path.join(TR_RAW_DIR, row.rawPath ?? `${eik}.json`); + entry = fs.existsSync(file) + ? { deed: JSON.parse(fs.readFileSync(file, 'utf8')), outsideTr: false, missing: false } + : { deed: null, outsideTr: false, missing: true }; + } + deedCache.set(eik, entry); + return entry; +} + const declarantsByEik = new Map(); for (const rec of agg.values()) { if (rec.scope !== 'self') continue; // ex-officio tell counts SELF declarants of a public board only @@ -470,7 +552,20 @@ for (const rec of agg.values()) { a.count++; if (r.eur != null) a.value += r.eur; } - const seatOk = [...rec.seats].some((s) => seatConfirmed(s, rec.bidder.settlement)); + // ── the evidence ladder replaces the publish tiers (ADR-0033 decision 1) ──────────────────────── + // rec.seats is keyed on `pid|eik|scope`, so it already holds ONLY the seats this person declared for + // THIS company — which is what #279 §5 rung 3 requires: 4.9% of company-name keys carry more than one + // distinct declared seat, so a company-only key would let one person's seat confirm another's link. + const { deed, outsideTr, missing } = deedFor(rec.eik); + if (missing && !ALLOW_PARTIAL_TR) { + // Unreachable via the coverage gate above; kept as a belt-and-braces refusal so a future change + // that loosens the gate cannot silently publish a link with no evidence behind it. + trCache.close(); + db.close(); + throw new Error( + `no cached deed for ЕИК ${rec.eik} — the coverage gate should have caught this`, + ); + } // A declarant-provided ЕИК is the national unique identifier (ЗТРРЮЛНЦ) — it resolves the winner // deterministically even behind a generic or winner-colliding name, so a declared_eik match publishes // on its own basis (A_eik), never held for name-genericness. This is at least as certain as the seat @@ -478,14 +573,39 @@ for (const rec of agg.values()) { // Name-only methods (exact_name_key / extracted_name) still ride the distinctiveness/seat gate below: // a globally non-unique winner name (e.g. „Водоснабдяване и канализация ЕАД" → 2 valid ЕИК in different // towns) can never be name-distinctive, so it publishes only if the declared SEAT disambiguates, else held. + // The filters that can only WITHHOLD are retained as an AND-gate on the weakest rung only + // (ADR-0033 decision 2): a nationally shared company name cannot ride „Потвърдено". The stronger + // „Документ" rung is deliberately not gated — the register named this person in THIS company, which + // makes the name key moot. Near-zero recall cost, and it preserves ADR-0017's outcome. + // ADR-0017 carried forward, and NARROWED to what it actually held: a name backing more than one valid + // ЕИК cannot support a name-derived identity claim. It gates the SEAT leg of rung 3 only — never the + // declared-ЕИК leg (ADR-0028: the ЕИК is the identity), and never rung 2 (the register named this + // person in THIS company). nameDistinctiveness is deliberately NOT part of this gate: the seat rung + // exists precisely to rescue a generic name, so requiring distinctiveness would empty it. const nameUnique = nameGloballyUnique(rec.key); - const tier = - rec.method === 'declared_eik' - ? 'A_eik' - : publishTier({ - seatOk, - distinctiveness: nameUnique ? nameDistinctiveness(rec.key) : 'generic', - }); + // With --allow-partial-tr the operator has accepted an incomplete cache. An uncached ЕИК then yields + // no evidence at all, which is „Неизвестна" — held. It must never be read as a reason to publish. + const verdict = missing + ? { + kind: 'unknown', + publishable: false, + registryRole: null, + matchedFact: null, + entryNumber: null, + entryDate: null, + rulesVersion: RULES_VERSION, + } + : evidenceVerdict({ + deed, + outsideTr, + declarantName: rec.person, + declaredSeats: [...rec.seats], + declaredEik: rec.method === 'declared_eik', + firstDeclaredYear: declYears.length ? Math.min(...declYears) : null, + scope: rec.scope, + nameGloballyUnique: nameUnique, + }); + const tier = verdict.kind; const contemporaneous = [...years].some( (cy) => temporalStatus(declYears, cy) === 'contemporaneous', ) @@ -547,15 +667,33 @@ for (const rec of agg.values()) { // non-surfaced class ('internal'), not 'published'. cValue can legitimately be 0 with cCount>0 (contracts // whose amount is unknown/NULL) — that is a real conflict of unknown value, so gate on COUNT, not value. const surfaces = (iClass === 'private_ownership' || iClass === 'family_ownership') && cCount > 0; + // §7: „terminated" is an inference from SILENCE, and its commonest cause is a finished mandate, not a + // sale. Before ADR-0021 E11's withdrawal takes effect, a terminated OWN stake is reconciled against the + // live deed: a person still registered as an owner has not divested. Family stakes are never reconciled + // — the registered owner there is the relative, whose name we neither store nor check. + // PHASE 1 uses only `terminated`; the „и към днешна дата" label is computed and deliberately not + // rendered (ADR-0033 decision 4 — it asserts a present tense behind an LIA addendum). + const recon = + divested && rec.scope === 'self' + ? reconcileTermination({ deed, declarantName: rec.person, scope: rec.scope }) + : { terminated: divested, label: null }; + const terminatedEffective = recon.terminated; + + // Order matters and differs from the old ladder: `internal` is now decided BEFORE `published`, so a + // non-surfaced class (ex-officio board, management-only, zero-contract) can never land in `held`. + // `held` is the REVIEW QUEUE, and its population is exactly the evidence rungs that withhold — + // bar_joint_stock, unknown and outside_tr. const status = isSuppressed(linkKey) ? 'suppressed' - : divested - ? 'withdrawn' - : tier === 'C_hold' - ? 'held' - : surfaces - ? 'published' - : 'internal'; + : verdict.kind === 'refuted' + ? 'withdrawn' // §5.4 — own stakes only; evidence.mjs refuses to refute a family stake + : terminatedEffective + ? 'withdrawn' + : !surfaces + ? 'internal' + : verdict.publishable + ? 'published' + : 'held'; const yrs = [...years]; insLink.run( `il:${linkKey}`, @@ -580,6 +718,26 @@ for (const rec of agg.values()) { yrs.length ? String(Math.max(...yrs)) : null, status, ); + // live_status is RE-DERIVED every run and never treated as part of the permanent seal (ADR-0033 R3): + // it asserts a present tense whose freshness is bounded by the cache refresh cycle. + const liveStatus = !terminatedEffective + ? recon.label === 'owner_today' + ? 'terminated_owner_still' + : 'live' + : recon.label === 'manager_today' + ? 'terminated_manager_still' + : 'terminated'; + insEvidence.run( + linkKey, + verdict.kind, + verdict.registryRole, + verdict.matchedFact, + verdict.entryNumber, + verdict.entryDate, + trLookupDate, + verdict.rulesVersion, + liveStatus, + ); for (const [auth_id, a] of perAuth) insILA.run(linkKey, auth_id, a.name, a.count, a.value || null, a.own); } diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index d795bf59..d90b44a4 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -14,7 +14,7 @@ import { fingerprint } from './suppressions.mjs'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); const SUPP_SALT = 'test-salt-9f3a'; // stand-in for the CI secret SUPPRESSION_SALT -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad(extraEnv = {}) { execFileSync( @@ -22,17 +22,105 @@ function runLoad(extraEnv = {}) { ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], { cwd: ROOT, - env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING, ...extraEnv }, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + ...extraEnv, + }, stdio: 'pipe', }, ); } + +/** + * Build a Trade Register cache + raw deeds covering the fixture's winners. + * + * `spec[eik]` describes one deed: `owners` / `managers` are full names placed in SEPARATE registry + * entities (so the entity-boundary rule is exercised end to end), `form` is the numeric legalForm and + * `suffix` the ЗТРРЮЛНЦ form on fullName. Anything omitted from `spec` is still cached — as a deed + * naming somebody else — because the loader must FAIL CLOSED on a cache that does not cover every + * candidate, and a test that silently left ЕИК uncovered would exercise that path by accident. + */ +function buildTrCache(dbFile, rawDir, spec = {}, { omit = [] } = {}) { + fs.mkdirSync(rawDir, { recursive: true }); + const cache = new DatabaseSync(dbFile); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + const src = new DatabaseSync(DB, { readOnly: true }); + const eiks = src + .prepare('SELECT eik_normalized e FROM bidders WHERE eik_normalized IS NOT NULL') + .all() + .map((r) => r.e); + src.close(); + + const container = (t) => + `

${t}

`; + const joinEntities = (names) => names.map(container).join(`
`); + + for (const eik of eiks) { + if (omit.includes(eik)) continue; + const d = spec[eik] ?? {}; + if (d.outsideTr) { + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,fetched_at,outside_reason) VALUES(?,?,?,?)', + ) + .run(eik, 'outside_tr', '2026-08-05T00:00:00Z', 'HTTP 200, empty body'); + continue; + } + const fields = []; + const push = (nameCode, names, entryDate) => + names?.length && + fields.push({ + nameCode, + htmlData: joinEntities(names), + fieldEntryNumber: '20110502101007', + fieldEntryDate: `${entryDate ?? '2011-05-02'}T00:00:00`, + }); + push('CR_F_19_L', d.owners ?? ['НЯКОЙ ДРУГ СОБСТВЕНИК'], d.ownEntryDate); + push('CR_F_7_L', d.managers, d.ownEntryDate); + if (d.seat) push('CR_F_5_L', [`Населено място: ${d.seat}`], d.seatEntryDate ?? d.ownEntryDate); + const deed = { + uic: eik, + fullName: `"ФИКС" ${d.suffix ?? 'ООД'}`, + legalForm: d.form ?? 4, + sections: [{ subDeeds: [{ groups: [{ fields }] }] }], + }; + fs.writeFileSync(path.join(rawDir, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + `INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code, + legal_form_verdict,seat_normalized,latest_own_entry_date) + VALUES(?,?,?,?,?,?,?,?,?)`, + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + d.form ?? 4, + d.suffix && /АД|КДА/.test(d.suffix) ? 'joint_stock' : 'closely_held', + d.seat ? d.seat.replace(/^гр\.\s*/, '').toUpperCase() : null, + d.ownEntryDate ?? '2011-05-02', + ); + } + cache.close(); +} const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-load-')); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); fs.mkdirSync(STAGING, { recursive: true }); // minimal slice of the winner schema that load.mjs joins @@ -99,6 +187,12 @@ before(() => { -- not twice — the load-side sibling of the UI conflictHeadline per-ЕИК money dedup. INSERT INTO bidders VALUES ('eik:181818187','ПАРТНЬОРИ 5 ЕООД','181818187',1,'София'); INSERT INTO contracts VALUES ('c15','t1','eik:181818187','2022-06-01',600000); + -- FAMILY positive control (#279): identical in shape to Кмет's case but the official DECLARED the + -- company's seat, which is what confirms the company's identity when the registered owner is the + -- relative whose name we never hold. Without this case „family published: 0" would be + -- indistinguishable from a structurally dead path (ADR-0027's false-zero lesson). + INSERT INTO bidders VALUES ('eik:191919199','СЕМЕЕН ДОМ ЕООД','191919199',1,'Русе'); + INSERT INTO contracts VALUES ('c16','t4','eik:191919199','2023-07-01',180000); `); db.close(); @@ -128,7 +222,7 @@ before(() => { template: 'assets', category: '', institution: 'X', - person: 'Мария Иванова', + person: 'Мария Иванова Петрова', position: '', entity: '"ГЕНЕРИК" ООД', kind: 'shares', @@ -145,7 +239,7 @@ before(() => { template: 'assets', category: '', institution: 'Y', - person: 'Петър Николов', + person: 'Петър Иванов Николов', position: '', entity: 'СИЙ ЕООД', kind: 'shares', @@ -162,7 +256,7 @@ before(() => { template: 'assets', category: '', institution: 'Z', - person: 'Георги Стоянов', + person: 'Георги Иванов Стоянов', position: '', entity: 'СИЙ ЕООД', kind: 'shares', @@ -181,7 +275,7 @@ before(() => { template: 'assets', category: '', institution: 'W', - person: 'Стефан Колев', + person: 'Стефан Иванов Колев', position: '', entity: '"ГЕНЕРИК" ООД, ЕИК 222222229', kind: 'shares', @@ -198,7 +292,7 @@ before(() => { template: 'assets', category: '', institution: 'V', - person: 'Радка Илиева', + person: 'Радка Иванова Илиева', position: '', entity: '"ГЕНЕРИК" ООД, ЕИК 333333338', kind: 'shares', @@ -215,7 +309,7 @@ before(() => { template: 'interests', category: '', institution: 'U', - person: 'Борис Манолов', + person: 'Борис Иванов Манолов', position: 'член на съвет', entity: 'ХОЛДИНГ 9 ЕАД', kind: 'management', @@ -231,7 +325,7 @@ before(() => { template: 'interests', category: '', institution: 'U', - person: 'Виктор Асенов', + person: 'Виктор Иванов Асенов', position: 'член на съвет', entity: 'ХОЛДИНГ 9 ЕАД', kind: 'management', @@ -249,7 +343,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Николай Дивестов', + person: 'Николай Иванов Дивестов', position: '', entity: 'ДИВЕСТ 1 ЕООД', kind: 'shares', @@ -265,7 +359,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Николай Дивестов', + person: 'Николай Иванов Дивестов', position: '', entity: 'ДИВЕСТ 2 ЕООД', kind: 'shares', @@ -283,7 +377,7 @@ before(() => { template: 'assets', category: '', institution: 'ОБЩИНА ТЕСТ', - person: 'Кмет Тестов', + person: 'Кмет Иванов Тестов', position: 'кмет', entity: 'ЕВРОСТРОЙ 21 ЕООД', kind: 'shares', @@ -293,6 +387,24 @@ before(() => { holderRelation: 'related', controlHash: 'H11', }, + // FAMILY positive control: same as Кмет, but with the seat declared → rung 3 confirms the company. + { + folder: '2024', + xmlFile: 'K2.xml', + year: '2023', + template: 'assets', + category: '', + institution: 'ОБЩИНА ТЕСТ', + person: 'Кметица Иванова Втора', + position: 'кмет', + entity: 'СЕМЕЕН ДОМ ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: 'Русе', + holderRelation: 'related', + controlHash: 'H11b', + }, // SECURITIES: Акционер holds LISTED joint-stock shares (kind securities) → excluded, no ownership link. { folder: '2024', @@ -301,7 +413,7 @@ before(() => { template: 'assets', category: '', institution: 'S', - person: 'Акционер Тестов', + person: 'Акционер Иванов Тестов', position: '', entity: 'ЛИСТЕД ТЕСТ АД', kind: 'securities', @@ -320,7 +432,7 @@ before(() => { template: 'assets', category: '', institution: 'N2', - person: 'Нула Тестов', + person: 'Нула Иванов Тестов', position: '', entity: 'НУЛА ТЕХ 3 ЕООД', kind: 'shares', @@ -340,7 +452,7 @@ before(() => { template: 'assets', category: '', institution: 'T2', - person: 'Пълен Дивестов', + person: 'Пълен Иванов Дивестов', position: '', entity: 'ДИВЕСТ ЗЕРО 4 ЕООД', kind: 'shares', @@ -360,7 +472,7 @@ before(() => { template: 'assets', category: '', institution: 'N3', - person: 'Двусмислен Тестов', + person: 'Двусмислен Иванов Тестов', position: '', entity: 'ДИСТИНКТ ТЕХ 7 ЕООД', kind: 'shares', @@ -379,7 +491,7 @@ before(() => { template: 'assets', category: '', institution: 'МВР', - person: 'Канонов Тестов', + person: 'Канонов Иванов Тестов', position: '', entity: 'КАНОН ТЕХ 5 ЕООД', kind: 'shares', @@ -396,7 +508,7 @@ before(() => { template: 'assets', category: '', institution: 'Министерство на вътрешните работи', - person: 'Канонов Тестов', + person: 'Канонов Иванов Тестов', position: '', entity: 'КАНОН ТЕХ 5 ЕООД', kind: 'shares', @@ -414,7 +526,7 @@ before(() => { template: 'assets', category: '', institution: '', - person: 'Безинст Тестов', + person: 'Безинст Иванов Тестов', position: '', entity: 'БЕЗИНСТ ТЕХ 6 ЕООД', kind: 'shares', @@ -434,7 +546,7 @@ before(() => { template: 'interests', category: '', institution: 'INT', - person: 'Интер Тестов', + person: 'Интер Иванов Тестов', position: '', entity: 'ИНТЕР ТЕХ 8 ЕООД', kind: 'shares', @@ -454,7 +566,7 @@ before(() => { template: 'assets', category: '', institution: 'ТЕСТ ВЕДОМСТВО', - person: 'Алфа Партньоров', + person: 'Алфа Иванов Партньоров', position: '', entity: 'ПАРТНЬОРИ 5 ЕООД', kind: 'shares', @@ -471,7 +583,7 @@ before(() => { template: 'assets', category: '', institution: 'ДРУГО ВЕДОМСТВО', - person: 'Бета Партньоров', + person: 'Бета Иванов Партньоров', position: '', entity: 'ПАРТНЬОРИ 5 ЕООД', kind: 'shares', @@ -503,7 +615,7 @@ before(() => { xmlFile: 'ZE1.xml', year: '2023', template: 'assets', // Пълен's later EMPTY ASSET filing — same type as his 2019 asset stake ⇒ divests it - person: 'Пълен Дивестов', + person: 'Пълен Иванов Дивестов', institution: 'T2', }); // #226 (Todor B1) cross-type: Интер declares his stake ONLY in an INTERESTS declaration (2020); his later @@ -515,13 +627,39 @@ before(() => { xmlFile: 'INTA.xml', year: '2023', template: 'assets', - person: 'Интер Тестов', + person: 'Интер Иванов Тестов', institution: 'INT', }); fs.writeFileSync( path.join(STAGING, 'filings.jsonl'), filings.map((f) => JSON.stringify(f)).join('\n') + '\n', ); + + // The Trade Register evidence each link now has to rest on (#279, ADR-0033). Shaped so every + // existing case keeps the INTENT it was written for, under the new rule rather than the old one: + // • a person the register names as owner/manager → „Документ" + // • a declared seat matching the registered seat → „Потвърдено" + // • a declared ЕИК → „Потвърдено" (never name-gated, ADR-0028) + // • nobody we can match and nothing to confirm → „Неизвестна", held + buildTrCache(TR_DB, TR_RAW, { + 111111119: { managers: ['ИВАН ПЕТРОВ ТЕСТОВ'] }, // manages → document/manager (class keeps it internal) + 444444447: { seat: 'гр. Бургас' }, // Петър declared Бургас → confirmed; Георги declared none → held + 555555556: { + managers: ['БОРИС ИВАНОВ МАНОЛОВ', 'ВИКТОР ИВАНОВ АСЕНОВ'], + suffix: 'ЕАД', + form: 5, + }, + 666666665: { owners: ['СЪВСЕМ ДРУГ СОБСТВЕНИК'] }, // Николай absent → his divestment stands + 777777773: { owners: ['НИКОЛАЙ ИВАНОВ ДИВЕСТОВ'] }, // still the registered owner → document + 888888884: { owners: ['РОДНИНА КМЕТОВА'] }, // family: the RELATIVE owns it, not the official + 999999998: { suffix: 'АД', form: 5 }, + 101010104: { owners: ['ДРУГ СОБСТВЕНИК'] }, // Пълен absent → divest-to-zero stands + 131313136: { owners: ['КАНОНОВ ИВАНОВ ТЕСТОВ'] }, + 161616163: { owners: ['ИНТЕР ИВАНОВ ТЕСТОВ'] }, + 181818187: { owners: ['АЛФА ИВАНОВ ПАРТНЬОРОВ', 'БЕТА ИВАНОВ ПАРТНЬОРОВ'] }, + 121212129: { owners: ['НУЛА ИВАНОВ ТЕСТОВ'] }, + 191919199: { owners: ['РОДНИНА ВТОРА'], seat: 'гр. Русе' }, // family + declared seat → confirmed + }); }); after(() => fs.rmSync(dir, { recursive: true, force: true })); @@ -540,7 +678,7 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // management_role never surfaces → status 'internal', NOT 'published' (a direct D1 reader must not see a // non-surfaced official+company row labelled published; the served query also filters by interest_class). assert.equal(ivan.status, 'internal'); - assert.equal(ivan.publish_tier, 'B_distinctive'); + assert.equal(ivan.publish_tier, 'document'); // the register names him a manager of this company assert.equal(ivan.relation, 'manages'); assert.equal(ivan.interest_class, 'management_role'); // manages, sole declarant → ambiguous, not headline assert.equal(ivan.own_institution, 'exact'); @@ -557,8 +695,8 @@ test('resolves publish/held/quarantine tiers deterministically', () => { assert.equal(blob.value_eur, 25000); // bare collision name (no ЕИК in text) → quarantined, Мария gets no link - assert.equal(link('222222229', 'Мария Иванова'), undefined); - assert.equal(link('333333338', 'Мария Иванова'), undefined); + assert.equal(link('222222229', 'Мария Иванова Петрова'), undefined); + assert.equal(link('333333338', 'Мария Иванова Петрова'), undefined); // the only links onto the colliding ЕИК come from declared_eik (Стефан/Радка), never exact_name_key assert.equal( db @@ -569,14 +707,14 @@ test('resolves publish/held/quarantine tiers deterministically', () => { 0, ); - const petar = link('444444447', 'Петър Николов'); - assert.equal(petar.publish_tier, 'A_seat'); // generic name rescued by seat match + const petar = link('444444447', 'Петър Иванов Николов'); + assert.equal(petar.publish_tier, 'confirmed'); // declared seat == registered seat assert.equal(petar.status, 'published'); assert.equal(petar.interest_class, 'private_ownership'); // declared a share → the headline conflict signal // two officials manage the SAME company → deterministically classed ex-officio (public board), not private - const boris = link('555555556', 'Борис Манолов'); - const viktor = link('555555556', 'Виктор Асенов'); + const boris = link('555555556', 'Борис Иванов Манолов'); + const viktor = link('555555556', 'Виктор Иванов Асенов'); assert.equal(boris.interest_class, 'ex_officio_board'); assert.equal(viktor.interest_class, 'ex_officio_board'); assert.equal(boris.relation, 'manages'); @@ -584,14 +722,14 @@ test('resolves publish/held/quarantine tiers deterministically', () => { assert.equal(boris.status, 'internal'); assert.equal(viktor.status, 'internal'); - const georgi = link('444444447', 'Георги Стоянов'); - assert.equal(georgi.publish_tier, 'C_hold'); // generic, no seat → held + const georgi = link('444444447', 'Георги Иванов Стоянов'); + assert.equal(georgi.publish_tier, 'unknown'); // same company, but he declared no seat → nothing confirms assert.equal(georgi.status, 'held'); // E11 divestment: Николай's 2019 stake in ДИВЕСТ 1 is superseded by a 2022 filing that omits it → withdrawn; // his current ДИВЕСТ 2 stake stays published. A later ownership filing that drops a company ends that link. - const gone = link('666666665', 'Николай Дивестов'); - const kept = link('777777773', 'Николай Дивестов'); + const gone = link('666666665', 'Николай Иванов Дивестов'); + const kept = link('777777773', 'Николай Иванов Дивестов'); assert.equal(gone.status, 'withdrawn'); // divested — excluded from the published surface assert.equal(gone.interest_class, 'private_ownership'); assert.equal(gone.last_declared_year, '2019'); // dated to its last declaration, never asserted "current" @@ -601,15 +739,15 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // certain ЕИК (declared_eik) behind a colliding name, no seat → the declarant-provided ЕИК is the // national unique identifier, so identity is deterministic → publishes as A_eik, NOT held for // name-genericness (ADR-0016; the ЕИК is at least as certain as the seat that rescues Радка below). - const stefan = link('222222229', 'Стефан Колев'); + const stefan = link('222222229', 'Стефан Иванов Колев'); assert.equal(stefan.match_method, 'declared_eik'); // ЕИК resolution IS certain - assert.equal(stefan.publish_tier, 'A_eik'); // ЕИК = unique identifier → deterministic, not name-gated + assert.equal(stefan.publish_tier, 'confirmed'); // the declared ЕИК confirms the company, never name-gated assert.equal(stefan.status, 'published'); // private_ownership (20% ООД share) → surfaces // same colliding name, resolved by her declared ЕИК (with seat as extra corroboration) → the ЕИК is the // identity, so A_eik (not A_seat); publishable. A_seat's own path stays covered by Петър above. - const radka = link('333333338', 'Радка Илиева'); + const radka = link('333333338', 'Радка Иванова Илиева'); assert.equal(radka.match_method, 'declared_eik'); - assert.equal(radka.publish_tier, 'A_eik'); + assert.equal(radka.publish_tier, 'confirmed'); assert.equal(radka.status, 'published'); // FAMILY: a close relative's declared stake in a winner that sold to the official's OWN institution. @@ -617,10 +755,17 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // relative's declared stake in a procurement winner is the same public-interest signal. class // family_ownership, relation 'related', own_institution exact. It has real contract money (€250k, cCount>0) // so the zero-contract gate keeps it. - const family = link('888888884', 'Кмет Тестов'); + const family = link('888888884', 'Кмет Иванов Тестов'); assert.equal(family.relation, 'related'); assert.equal(family.interest_class, 'family_ownership'); - assert.equal(family.status, 'published'); // ADR-0032: family surfaces like self + // #279 NARROWS the family surface, and this is where it shows. The registered owner of a family + // stake is the RELATIVE, whose name we deliberately never store (ADR-0010 item 4, ADR-0032 #2) — so + // rung 2 („Документ") can never fire for a family link, by construction. Its identity can only be + // confirmed by something the OFFICIAL declared: the seat, or the ЕИК. Кмет declared neither, so his + // link is now HELD rather than published. ADR-0032's decision is untouched — family publishes on the + // named surface exactly like self — but it now needs the same registry evidence as everything else. + assert.equal(family.status, 'held'); + assert.equal(family.publish_tier, 'unknown'); assert.equal(family.own_institution, 'exact'); // relative's company sold to the official's own institution assert.equal(family.contemporaneous, 1); assert.equal(family.contract_value_eur, 250000); @@ -645,8 +790,8 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // SUM(contract_value_eur) over published links double-counts that winner — the load-side twin of the UI // conflictHeadline bug. Prove the fixture actually exercises the collision (naive per-link sum strictly // exceeds the per-ЕИК-deduped sum), then assert the reported totals equal the deduped figure. - const alfa = link('181818187', 'Алфа Партньоров'); - const beta = link('181818187', 'Бета Партньоров'); + const alfa = link('181818187', 'Алфа Иванов Партньоров'); + const beta = link('181818187', 'Бета Иванов Партньоров'); assert.equal(alfa.status, 'published'); assert.equal(beta.status, 'published'); assert.equal(alfa.interest_class, 'private_ownership'); @@ -683,14 +828,14 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // ZERO-CONTRACT gate (I5): a distinctive winner with NO contracts is collected but never published — the // card would read „0 договори · 0 €", which is no procurement conflict. status 'internal', not 'published'. - const zero = link('121212129', 'Нула Тестов'); + const zero = link('121212129', 'Нула Иванов Тестов'); assert.equal(zero.contract_count, 0); assert.equal(zero.interest_class, 'private_ownership'); // it IS own material ownership … - assert.equal(zero.publish_tier, 'B_distinctive'); // … and tier-B by name … + assert.equal(zero.publish_tier, 'document'); // … with registry evidence … assert.equal(zero.status, 'internal'); // … but the zero-contract gate withholds it from the surface // SECURITIES/materiality: a self holding of LISTED joint-stock shares forms NO ownership link. - assert.equal(link('999999998', 'Акционер Тестов'), undefined); + assert.equal(link('999999998', 'Акционер Иванов Тестов'), undefined); // but it is still recorded as a declared interest (census), tagged kind securities assert.equal( db.prepare("SELECT kind FROM declared_interests WHERE entity_raw='ЛИСТЕД ТЕСТ АД'").get().kind, @@ -700,7 +845,7 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // B1 divest-to-ZERO: Пълен owned ДИВЕСТ ЗЕРО in 2019, then filed an EMPTY declaration in 2023 (no holdings // row — only a filings.jsonl entry). The empty filing advances his horizon to 2023, so the 2019 stake is // WITHDRAWN. Without the filing horizon (pre-B1) his scope-max would be 2019 and this would stay published. - const divZero = link('101010104', 'Пълен Дивестов'); + const divZero = link('101010104', 'Пълен Иванов Дивестов'); assert.equal(divZero.interest_class, 'private_ownership'); assert.equal(divZero.last_declared_year, '2019'); // dated to its last declaration, never asserted current assert.equal(divZero.status, 'withdrawn'); // caught by the empty later filing (B1) @@ -710,18 +855,18 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // that asset-declaration silence as a sale and WITHDRAWS the stake; the per-type horizon must not — no later // INTERESTS filing omits the company. This link must stay PUBLISHED. (Guards against dropping a true link: // 13% of holders declare a stake only in the interests declaration.) - const crossType = link('161616163', 'Интер Тестов'); + const crossType = link('161616163', 'Интер Иванов Тестов'); assert.equal(crossType.interest_class, 'private_ownership'); assert.equal(crossType.status, 'published'); // NOT withdrawn — the later asset filing is a different type // B4 UNKNOWN holder: an ambiguous holder cell forms NO link at all (counted nowhere) — it must never // reach the leaderboard, self or family (ADR-0032). Двусмислен gets no interest_link. - assert.equal(link('111111119', 'Двусмислен Тестов'), undefined); + assert.equal(link('111111119', 'Двусмислен Иванов Тестов'), undefined); // but the person + declared_interest are still recorded (census), and it is neither self nor family. assert.equal( db .prepare( - "SELECT COUNT(*) n FROM interest_links il JOIN persons p ON p.id=il.person_id WHERE p.name='Двусмислен Тестов'", + "SELECT COUNT(*) n FROM interest_links il JOIN persons p ON p.id=il.person_id WHERE p.name='Двусмислен Иванов Тестов'", ) .get().n, 0, @@ -738,10 +883,10 @@ test('resolves publish/held/quarantine tiers deterministically', () => { .n, 1, ); - const kanon = link('131313136', 'Канонов Тестов'); + const kanon = link('131313136', 'Канонов Иванов Тестов'); assert.equal(kanon.status, 'published'); // distinctive, private ownership, has a contract // N10 empty-institution: an empty institution cannot distinguish homonyms → Безинст forms NO link. - assert.equal(link('141414141', 'Безинст Тестов'), undefined); + assert.equal(link('141414141', 'Безинст Иванов Тестов'), undefined); db.close(); }); @@ -772,14 +917,15 @@ test('re-run is idempotent and honors the suppression list (contested link stays 'suppressed', ); // idempotent: still exactly the same number of links + persons after a clean rebuild. - // 16 links: 12 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero - // 'withdrawn' + Интер's per-type-kept published link) + 1 family + Канонов's canonicalized single link + + // 17 links: 12 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero + // 'withdrawn' + Интер's per-type-kept published link) + 2 family (Кмет's, now held for want of + // registry evidence, and Кметица's seat-confirmed one) + Канонов's canonicalized single link + // Алфа & Бета (two officials on one winner, ПАРТНЬОРИ 5); Мария (quarantined), Акционер (securities), // Двусмислен (unknown holder) & Безинст (empty institution) none. - assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 16); - // 19 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, + assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 17); + // 20 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, // zero-contract Нула and the two ПАРТНЬОРИ co-owners; Канонов's two institution-variant filings fold to ONE. - assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 19); + assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 20); db.close(); }); @@ -879,3 +1025,135 @@ test('a suppression on a ROTATED key_version FAILS the build (B3 no silent salt (err) => /key_version/.test(String(err.stderr ?? '') + String(err.message ?? '')), ); }); + +// ── the fail-closed evidence gates (ADR-0033 decision 7) ───────────────────────────────────────── +// Both directions of "no evidence ⇒ no publish", because the partial case is the dangerous one: it does +// NOT fail loudly downstream. An 80%-restored cache yields roughly 80 published links, which clears +// ship-related-persons.mjs's floor of 50 — so it would ship a decimated surface and then wipe the rest +// of the live links. The loader is the only place that can still tell the difference. +test('a MISSING Trade Register cache refuses the whole load', () => { + const gone = path.join(dir, 'no-such-cache.sqlite'); + assert.throws( + () => runLoad({ TR_CACHE_DB: gone }), + /REFUSE TO LOAD[\s\S]*no Trade Register cache/, + ); +}); + +test('a PARTIAL cache refuses the load rather than publishing a decimated surface', () => { + const partialDb = path.join(dir, 'partial-cache.sqlite'); + const partialRaw = path.join(dir, 'partial-deeds'); + // Cover everything EXCEPT two winners — the shape a resumed-but-unfinished crawl leaves behind. + buildTrCache( + partialDb, + partialRaw, + { 111111119: { managers: ['ИВАН ПЕТРОВ ТЕСТОВ'] } }, + { omit: ['444444447', '777777773'] }, + ); + assert.throws( + () => runLoad({ TR_CACHE_DB: partialDb, TR_RAW_DIR: partialRaw }), + /REFUSE TO LOAD[\s\S]*covers \d+ of \d+/, + ); +}); + +test('--allow-partial-tr is the deliberate, stated override', () => { + // Without an override a single permanently unreachable ЕИК would deadlock the pipeline forever. + const partialDb = path.join(dir, 'partial-ok.sqlite'); + const partialRaw = path.join(dir, 'partial-ok-deeds'); + buildTrCache(partialDb, partialRaw, {}, { omit: ['121212129'] }); + assert.doesNotThrow(() => + execFileSync( + 'node', + [ + '--import', + path.join(HERE, 'register-ts.mjs'), + path.join(HERE, 'load.mjs'), + '--allow-partial-tr', + ], + { + cwd: ROOT, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: partialDb, + TR_RAW_DIR: partialRaw, + }, + stdio: 'pipe', + }, + ), + ); + runLoad(); // restore the full-cache state for any later reader +}); + +test('the candidate ЕИК list is written for the crawler, covering held links too', () => { + runLoad(); + const listed = fs + .readFileSync(path.join(STAGING, 'candidate-eiks.txt'), 'utf8') + .split('\n') + .filter(Boolean); + const db = open(); + const all = db + .prepare('SELECT DISTINCT eik FROM interest_links') + .all() + .map((r) => r.eik); + db.close(); + // Every resolved ЕИК, not just the published ones: a link held for want of evidence still needs a + // deed to explain why it is held. + for (const e of all) assert.ok(listed.includes(e), `${e} missing from candidate-eiks.txt`); + assert.equal(new Set(listed).size, listed.length, 'no duplicates — each ЕИК costs one request'); +}); + +test('every link carries an evidence seal, and no seal carries a name', () => { + runLoad(); + const db = open(); + const links = db.prepare('SELECT link_key, publish_tier, status FROM interest_links').all(); + const seals = db.prepare('SELECT * FROM interest_link_evidence').all(); + assert.equal(seals.length, links.length, 'a seal for EVERY link, held and withdrawn included'); + + const CLOSED = /^(?:seat:[\p{Lu} -]+|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + for (const s of seals) { + if (s.matched_fact != null) assert.match(s.matched_fact, CLOSED, s.matched_fact); + assert.ok(s.rules_version.length > 0); + assert.match(s.lookup_date, /^\d{4}-\d{2}-\d{2}$/); + } + // The rail, stated as an assertion rather than a convention. Scope it precisely: `link_key` carries + // the OFFICIAL's own identity by design — they are named on the public surface from their own + // declaration — so the rail is about everyone else. No name that exists only inside a registry deed + // (a co-owner, a manager, the relative who actually holds a family stake) may reach a sealed column. + const evidenceOnly = seals.map(({ link_key: _ignored, ...rest }) => rest); + const blob = JSON.stringify(evidenceOnly).toUpperCase(); + for (const thirdParty of [ + 'РОДНИНА', // the relative who owns the family company + 'СЪВСЕМ ДРУГ СОБСТВЕНИК', // a registry co-owner nobody declared + 'НЯКОЙ ДРУГ', + 'ДРУГ СОБСТВЕНИК', + ]) + assert.ok(!blob.includes(thirdParty), `a third-party name reached the seal: ${thirdParty}`); + // …and the declarant's name must not be duplicated into the evidence fields either. + for (const surname of ['ТЕСТОВ', 'ДИВЕСТОВ', 'ПАРТНЬОРОВ']) + assert.ok(!blob.includes(surname), `a declarant name reached an evidence column: ${surname}`); + db.close(); +}); + +test('a family stake publishes ONLY when the official confirmed the company themselves', () => { + // The positive control for the narrowest published path. A family link can never earn „Документ" — + // the registered owner is the relative, whose name we never hold — so it stands or falls on the seat + // or ЕИК the OFFICIAL declared. Without this case „family published: 0" would be indistinguishable + // from a structurally dead path (ADR-0027). + runLoad(); + const db = open(); + const withSeat = db + .prepare( + "SELECT il.status, il.publish_tier FROM interest_links il JOIN persons p ON p.id=il.person_id WHERE il.eik='191919199' AND p.name='Кметица Иванова Втора'", + ) + .get(); + assert.equal(withSeat.status, 'published'); + assert.equal(withSeat.publish_tier, 'confirmed'); + const seal = db + .prepare( + "SELECT matched_fact FROM interest_link_evidence WHERE link_key LIKE '%191919199|family'", + ) + .get(); + assert.equal(seal.matched_fact, 'seat:РУСЕ'); + db.close(); +}); diff --git a/scripts/cacbg/tr-census.mjs b/scripts/cacbg/tr-census.mjs deleted file mode 100644 index e7b8c66d..00000000 --- a/scripts/cacbg/tr-census.mjs +++ /dev/null @@ -1,195 +0,0 @@ -// TR name-uniqueness census (ADR-0015). Promotes tier-C held interest_links — generic company names -// with a single WINNER namesake — to 'published' iff the name is GLOBALLY unique in the Trade Register. -// -// Source: the Commercial Register open-data dump on data.egov.bg (DPA-safe: ЕГН/ЛНЧ hashed out, company -// name + ЕИК retained). We build companyNameKey(name) → {distinct ЕИК} over ALL active entities using -// the SAME normalizer as the matcher, so the key spaces are identical. Promotion is deterministic: -// key count == 1 AND that ЕИК == the matched winner ЕИК. Anything else stays held. No heuristic. -// -// Field-detection is structural (find the 9/13-digit ЕИК; find the company-name field), so it is robust -// to the exact open-data column names — confirm the mapping against the real dump before a production run. -// -// Run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/tr-census.mjs --dump -import { DatabaseSync } from 'node:sqlite'; -import fs from 'node:fs'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); -const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); -const arg = (n) => { - const i = process.argv.indexOf(`--${n}`); - return i >= 0 ? process.argv[i + 1] : undefined; -}; -const DUMP = arg('dump') || process.env.TR_DUMP; -const { companyNameKey } = await import('../../packages/shared/src/company-name-key.ts'); - -const isEik = (v) => typeof v === 'string' && /^\d{9}$|^\d{13}$/.test(v.trim()); - -// The register stores CompanyName WITHOUT the legal form and LegalForm as a code; our matcher keys on the -// name WITH its Bulgarian form (bidders.name = «…» ЕООД), so we must reconstruct it or every lookup misses. -// Codes seen in the real dump: EOOD OOD ASSOC CC K EAD ET AD KCHT FOUND KD SD DPK EDPK IAD IEAD. -const LEGAL_FORM = { - EOOD: 'ЕООД', - OOD: 'ООД', - EAD: 'ЕАД', - AD: 'АД', - ET: 'ЕТ', - KD: 'КД', - K: 'КД', - SD: 'СД', - KCHT: 'КДА', - IAD: 'ИАД', - IEAD: 'ИЕАД', - DPK: 'ПК', - EDPK: 'ЕПК', - CC: 'кооперация', - ASSOC: 'СНЦ', - FOUND: 'фондация', -}; -// A Bulgarian trade name reconstructed to match the matcher's key space. Unknown form → append the raw -// code (harmless: it just forms its own key that no bidder lookup will hit). -const bgName = (companyName, legalForm) => - `${String(companyName ?? '').trim()} ${LEGAL_FORM[legalForm] ?? String(legalForm ?? '').trim()}`.trim(); - -// Extract {eik, name} from ONE TR deed's attribute object ({CompanyName, LegalForm, UIC, DeedStatus, …}). -// Also tolerates a flat {eik/uic, name} record (test fixtures). ЕИК = the 9/13-digit UIC. -export function extractEntity(rec) { - const uic = rec.UIC ?? rec.uic ?? rec.eik ?? rec.EIK; - const eik = isEik(String(uic ?? '')) ? String(uic).trim() : null; - const name = - rec.CompanyName != null ? bgName(rec.CompanyName, rec.LegalForm) : (rec.name ?? null); - return { eik, name: name || null }; -} - -// Read a TR dump as deed-attribute records. The real data.egov.bg export nests them at -// Message[].Body[].Deeds[].Deed[], each carrying its fields under `$` (fast-xml/attr style). Falls back to -// a plain JSON array / {data:[…]} / JSONL of already-flat records so tests and other shapes still work. -function* records(dump) { - const raw = fs.readFileSync(dump, 'utf8'); - const trimmed = raw.trimStart(); - if (trimmed.startsWith('{') || trimmed.startsWith('[')) { - const parsed = JSON.parse(raw); - const deeds = parsed?.Message?.[0]?.Body?.[0]?.Deeds?.[0]?.Deed; - if (Array.isArray(deeds)) { - for (const d of deeds) yield d?.['$'] ?? d; - return; - } - const arr = Array.isArray(parsed) - ? parsed - : (parsed.data ?? parsed.records ?? parsed.result ?? []); - yield* arr; - } else { - for (const line of raw.split('\n')) { - const t = line.trim(); - if (t) yield JSON.parse(t); - } - } -} - -// Build companyNameKey → Set(ЕИК) over one or more dump files (accumulates — call with every daily file to -// approach the full register). Deduped by ЕИК within a key, so the same company appearing on many days -// (or a name→one company) counts once; two DISTINCT companies sharing a name yield size>1 = not unique. -export function buildCensus(dumps) { - const census = new Map(); - for (const dump of Array.isArray(dumps) ? dumps : [dumps]) { - for (const rec of records(dump)) { - const { eik, name } = extractEntity(rec); - if (!eik || !name) continue; - const key = companyNameKey(name); - if (!census.has(key)) census.set(key, new Set()); - census.get(key).add(eik); - } - } - return census; -} - -// Promote held tier-C links the census proves globally unique: name-key maps to EXACTLY one ЕИК AND that -// ЕИК is the matched winner's (a national namesake, even at a different ЕИК, keeps the link held). With -// dryRun, report the would-promote set without writing — used to validate against a PARTIAL census, which -// must never actually promote (an un-ingested namesake would make a false-unique claim = libel). -export function promote(db, census, { dryRun = false, minEik = null, forcePartial = false } = {}) { - // Partial-census guard (libel gate). A census smaller than the real ТР register can make a genuinely - // non-unique name look unique (`eiks.size === 1`) → a false, libelous attribution. A real promote must - // ASSERT coverage in code, not by convention: pass `--min-eik ≥ register size` (the census's distinct-ЕИК - // count must meet it) or the explicit `--force-partial` escape hatch. dryRun is exempt — it writes nothing - // and exists precisely to inspect a partial census. - if (!dryRun && !forcePartial) { - if (!Number.isInteger(minEik) || minEik < 1) { - throw new Error( - 'refusing to promote without --min-eik : a partial census silently fabricates ' + - 'false-unique attributions. Pass --min-eik ≥ the known ТР register size, or --force-partial to override.', - ); - } - const distinctEik = new Set(); - for (const s of census.values()) for (const e of s) distinctEik.add(e); - if (distinctEik.size < minEik) { - throw new Error( - `refusing to promote from a partial census: ${distinctEik.size} distinct ЕИК < --min-eik ${minEik}. ` + - 'Ingest more ТР dumps until coverage meets the register size, or pass --force-partial to override.', - ); - } - } - const held = db - .prepare( - "SELECT link_key, eik, entity_key FROM interest_links WHERE status='held' AND publish_tier='C_hold'", - ) - .all(); - const upd = db.prepare( - "UPDATE interest_links SET status='published', match_method='exact_name_key+tr_census' WHERE link_key=?", - ); - const would = []; - if (!dryRun) db.exec('BEGIN'); - for (const l of held) { - const eiks = census.get(l.entity_key); - if (eiks && eiks.size === 1 && eiks.has(l.eik)) { - would.push(l.link_key); - if (!dryRun) upd.run(l.link_key); - } - } - if (!dryRun) db.exec('COMMIT'); - return { promoted: would.length, stillHeld: held.length - would.length, would }; -} - -// Resolve --dump: a single file, a comma-list, or --dump-dir (every *.json/*.jsonl inside). -function dumpFiles() { - const dir = arg('dump-dir'); - if (dir) - return fs - .readdirSync(dir) - .filter((f) => /\.(json|jsonl)$/.test(f)) - .map((f) => path.join(dir, f)); - return DUMP - ? DUMP.split(',') - .map((s) => s.trim()) - .filter(Boolean) - : []; -} - -// CLI entry (guarded so importing this module in tests has no side effects) -const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); -if (isMain) { - const files = dumpFiles(); - const dryRun = process.argv.includes('--dry-run'); - const forcePartial = process.argv.includes('--force-partial'); - const minEik = arg('min-eik') !== undefined ? Number(arg('min-eik')) : null; - if (!files.length) { - console.log( - 'no --dump/--dump-dir provided; census not run. Provide TR open-data files to promote tier-C links.', - ); - } else { - const db = new DatabaseSync(DB); - const census = buildCensus(files); - const eikTotal = [...census.values()].reduce((n, s) => n + s.size, 0); - console.log( - `census: ${files.length} file(s) → ${census.size} distinct name-keys / ${eikTotal} ЕИК`, - ); - const { promoted, stillHeld, would } = promote(db, census, { dryRun, minEik, forcePartial }); - console.log( - `${dryRun ? 'DRY-RUN would promote' : 'tier-C promotions'}: ${promoted} published, ${stillHeld} still held (non-unique or namesake mismatch)`, - ); - if (dryRun && would.length) - console.log(' would-promote link_keys:\n ' + would.join('\n ')); - db.close(); - } -} diff --git a/scripts/cacbg/tr-census.test.mjs b/scripts/cacbg/tr-census.test.mjs deleted file mode 100644 index 76d2fbb0..00000000 --- a/scripts/cacbg/tr-census.test.mjs +++ /dev/null @@ -1,121 +0,0 @@ -// TR census — real data.egov.bg deed shape + deterministic tier-C promotion. -// Run: node --import ./scripts/cacbg/register-ts.mjs --test scripts/cacbg/tr-census.test.mjs -import { test, before, after } from 'node:test'; -import assert from 'node:assert/strict'; -import { DatabaseSync } from 'node:sqlite'; -import fs from 'node:fs'; -import path from 'node:path'; -import os from 'node:os'; -import { extractEntity, buildCensus, promote } from './tr-census.mjs'; -const { companyNameKey } = await import('../../packages/shared/src/company-name-key.ts'); - -let dir, dump, dbPath; -const K = (s) => companyNameKey(s); - -before(() => { - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-census-')); - dump = path.join(dir, 'tr.json'); - // Real export shape: Message[].Body[].Deeds[].Deed[], each with its fields under `$`; CompanyName is - // bare and LegalForm is a code — the census must reconstruct «name + Bulgarian form» to match bidders. - const deeds = [ - { CompanyName: 'СИЙ', LegalForm: 'AD', UIC: '444444447' }, // globally unique → promote - { CompanyName: 'ОБЩА ФИРМА', LegalForm: 'OOD', UIC: '555555556' }, // one of two namesakes - { CompanyName: 'Обща Фирма', LegalForm: 'OOD', UIC: '666666663' }, // second namesake → NOT unique - { CompanyName: 'СИЙ', LegalForm: 'AD', UIC: '444444447' }, // same deed on another day → deduped - ]; - fs.writeFileSync( - dump, - JSON.stringify({ Message: [{ Body: [{ Deeds: [{ Deed: deeds.map((d) => ({ $: d })) }] }] }] }), - ); - - dbPath = path.join(dir, 'db.sqlite'); - const db = new DatabaseSync(dbPath); - db.exec(` - CREATE TABLE interest_links(link_key TEXT PRIMARY KEY, eik TEXT, entity_key TEXT, status TEXT, publish_tier TEXT, match_method TEXT); - INSERT INTO interest_links VALUES ('p1|444444447','444444447','${K('СИЙ АД')}','held','C_hold','exact_name_key'); - INSERT INTO interest_links VALUES ('p2|555555556','555555556','${K('ОБЩА ФИРМА ООД')}','held','C_hold','exact_name_key'); - `); - db.close(); -}); - -after(() => fs.rmSync(dir, { recursive: true, force: true })); - -test('extractEntity reconstructs «name + Bulgarian legal form» from a TR deed and reads UIC as ЕИК', () => { - assert.deepEqual(extractEntity({ CompanyName: 'СИЙ', LegalForm: 'AD', UIC: '444444447' }), { - eik: '444444447', - name: 'СИЙ АД', - }); - assert.deepEqual(extractEntity({ CompanyName: 'ТЕСТ', LegalForm: 'EOOD', UIC: '205057459' }), { - eik: '205057459', - name: 'ТЕСТ ЕООД', - }); - assert.equal(extractEntity({ CompanyName: 'X', LegalForm: 'AD', UIC: '2018060520' }).eik, null); // 10 digits ≠ ЕИК - assert.equal(extractEntity({ name: 'flat ООД', uic: '5555555560000' }).eik, '5555555560000'); // flat fallback + 13-digit -}); - -test('buildCensus indexes name-key → set of ЕИК over the nested export (dedup by ЕИК)', () => { - const c = buildCensus(dump); - assert.equal(c.get(K('СИЙ АД')).size, 1); // appears twice, one ЕИК → deduped - assert.equal(c.get(K('ОБЩА ФИРМА ООД')).size, 2); // two distinct ЕИК fold to one key → non-unique -}); - -test('promote publishes only globally-unique tier-C links; shared names stay held', () => { - const db = new DatabaseSync(dbPath); - // census covers 3 distinct ЕИК → assert coverage with --min-eik 3 (the partial-census gate). - const res = promote(db, buildCensus(dump), { minEik: 3 }); - assert.equal(res.promoted, 1); - const rows = new Map( - db - .prepare('SELECT link_key,status,match_method FROM interest_links') - .all() - .map((r) => [r.link_key, r]), - ); - assert.equal(rows.get('p1|444444447').status, 'published'); - assert.match(rows.get('p1|444444447').match_method, /tr_census/); - assert.equal(rows.get('p2|555555556').status, 'held'); // two namesakes → stays held - db.close(); -}); - -test('partial-census gate: a real promote refuses without --min-eik, and when coverage is below it', () => { - const db = new DatabaseSync(dbPath); - // isolate: p1 back to held (an earlier test may have published it — shared dbPath) - db.exec( - "UPDATE interest_links SET status='held', match_method='exact_name_key' WHERE link_key='p1|444444447'", - ); - const census = buildCensus(dump); // 3 distinct ЕИК - // no --min-eik → refuse (can't assert completeness → could fabricate a false-unique attribution) - assert.throws(() => promote(db, census, {}), /--min-eik/); - // --min-eik above the census's coverage → refuse (partial dump) - assert.throws(() => promote(db, census, { minEik: 4 }), /partial census|distinct ЕИК/); - // the refused runs published nothing — p1 is still held - assert.equal( - db.prepare("SELECT status FROM interest_links WHERE link_key='p1|444444447'").get().status, - 'held', - ); - db.close(); -}); - -test('partial-census gate: --force-partial is the explicit override', () => { - const db = new DatabaseSync(dbPath); - db.exec( - "UPDATE interest_links SET status='held', match_method='exact_name_key' WHERE link_key='p1|444444447'", - ); - const res = promote(db, buildCensus(dump), { forcePartial: true }); // no minEik, but forced - assert.equal(res.promoted, 1); - db.close(); -}); - -test('dry-run reports would-promote set without mutating the DB', () => { - const db = new DatabaseSync(dbPath); - // reset p1 back to held for an isolated dry-run check - db.exec( - "UPDATE interest_links SET status='held', match_method='exact_name_key' WHERE link_key='p1|444444447'", - ); - const res = promote(db, buildCensus(dump), { dryRun: true }); - assert.deepEqual(res.would, ['p1|444444447']); - assert.equal( - db.prepare("SELECT status FROM interest_links WHERE link_key='p1|444444447'").get().status, - 'held', - ); // untouched - db.close(); -}); diff --git a/scripts/tr/evidence.mjs b/scripts/tr/evidence.mjs index 262e31eb..f6619dad 100644 --- a/scripts/tr/evidence.mjs +++ b/scripts/tr/evidence.mjs @@ -157,9 +157,16 @@ export function evidenceVerdict(input) { // The weakest publishing rung, so it carries the extra AND-gate: a nationally shared company name // cannot ride it (ADR-0017's holding, carried forward). The stronger „Документ" rung above is not // gated — the register named this person in THIS company, which makes the name key moot. + // The declared-ЕИК leg is NOT name-gated. ADR-0028: the ЕИК is the identity, not the name, so it + // resolves the company deterministically even behind a nationally shared фирма — which is exactly the + // case ADR-0017 was written about. Gating it on name uniqueness would discard the strongest + // identifier we have precisely where it is most needed. + if (declaredEik) return verdict('confirmed', true, { matchedFact: 'eik' }); + + // The SEAT leg is name-gated, and only this one. ADR-0017's holding carried forward: a name shared by + // two ЕИК cannot support a name-derived identity claim. The seat still rescues a GENERIC name — that + // is the whole point of the rung — it just cannot rescue a NATIONALLY SHARED one. if (nameGloballyUnique) { - if (declaredEik) return verdict('confirmed', true, { matchedFact: 'eik' }); - const seat = registrySeat(deed); // Empty NEVER confirms — otherwise every link with no seat data on either side rubber-stamps. if (seat.settlement !== '') { diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs index 7c43b797..858db919 100644 --- a/scripts/tr/evidence.test.mjs +++ b/scripts/tr/evidence.test.mjs @@ -176,6 +176,38 @@ test('rung 3 — the weakest rung ALSO requires global name uniqueness (ADR-0017 assert.notEqual(v.kind, 'confirmed', 'a nationally shared name cannot ride the weakest rung'); }); +test('rung 3 — a declared ЕИК is NOT gated by name uniqueness (ADR-0028)', () => { + // The case ADR-0017 was written about — a фирма backing two ЕИК — is exactly where a declarant-supplied + // ЕИК is most valuable. Gating it on the name would discard the strongest identifier precisely when the + // name is useless. + const other = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК'))]); + const v = evidenceVerdict({ + ...base, + deed: other, + declaredEik: true, + nameGloballyUnique: false, + }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'eik'); +}); + +test('rung 3 — the seat rung DOES rescue a merely generic name; it is uniqueness that gates it', () => { + // The seat leg exists to rescue generic names (a bare one- or two-word фирма). Requiring the name to + // be distinctive would empty the rung of its entire purpose; only NATIONAL non-uniqueness blocks it. + const generic = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: generic, + declaredSeats: ['Пловдив'], + nameGloballyUnique: true, + }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'seat:ПЛОВДИВ'); +}); + test('rung 3 — name uniqueness does NOT gate the stronger „Документ" rung', () => { const v = evidenceVerdict({ ...base, nameGloballyUnique: false }); assert.equal( From 2a087443344dade66c2b2393fbdda43b8a41af1d Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 08:41:56 +0300 Subject: [PATCH 10/69] feat(web): the surface explains the registry fact each link rests on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read path, the DTO, the card and the audit (#279, ADR-0033). This is what makes „every shown link explains itself" true rather than a promise: a reader sees which act was read, which entry, and when — and can open the same act. THE PREDICATE HAS THREE COPIES and all three moved together. SURFACED_OWNERSHIP in related-persons.ts is the one people find; the other two are precompute.sql and refresh-slice.sql, and the second of those runs on the 6-hourly cron feeding the officials search index. Missing it would leave officials findable whose links no longer surface — a stale index that contradicts the page it links to. The card's wording is load-bearing and tested as such. „лицето е вписано като съдружник/собственик" reports what the act RECORDS; it does not say the official owns anything, because that claim comes from their own declaration and is rendered separately as „дялово участие". „Потвърдено" says the COMPANY was identified from something the official declared — nobody was found in the act — so its label must not imply anyone was, and a test asserts it never contains „вписан" or „собственик". audit.mjs's C axis was dead code the moment the loader stopped emitting B_distinctive: it re-derived name distinctiveness, a rule no longer in force. It now re-derives the rule that IS in force — a published link must carry a seal, the seal must be a publishing rung, and the tier must agree with it — plus the PII rail as an audited invariant rather than a convention: matched_fact is a closed vocabulary, so a name reaching it is a hard finding. Five new cases, including a positive control, without which all four negatives would still pass if the axes fired unconditionally. Eight test fixtures needed 0006 or evidence seals. That is not incidental: the read gate now requires a seal, so a fixture without one renders an EMPTY surface and every assertion below it passes vacuously. related-persons-sql.test.ts seals its links with a SELECT over interest_links, so a row added later is sealed automatically and cannot silently drop off the surface. Two things the browser check caught that no unit test could. The dev server was serving pre-change HTML because my local vite config pinned __SIGMA_DEPLOY_TAG__ to a constant — the worker prefixes that tag into its edge-cache key, so the key never rotated and stale HTML survived every restart. That is exactly the failure the real vite.config.ts warns about, reproduced by ignoring it. And the row's
is CSS-uppercased, so the assertion had to be case-insensitive. --- apps/web/app/components/ConflictCards.tsx | 16 ++++ apps/web/app/lib/conflicts.test.ts | 37 +++++++- apps/web/app/lib/conflicts.ts | 18 ++++ .../app/routes/conflict.pages.render.test.tsx | 8 +- apps/web/app/routes/conflicts.render.test.tsx | 30 ++++++ packages/api-contract/src/index.ts | 9 ++ .../db/src/contractor-identity-sql.test.ts | 7 +- .../etl-entity-canonicalization-sql.test.ts | 3 + packages/db/src/integrity-checks.test.ts | 3 + packages/db/src/migrations.test.ts | 1 + packages/db/src/queries/related-persons.ts | 31 +++++- packages/db/src/refresh-slice.test.ts | 8 ++ packages/db/src/related-persons-sql.test.ts | 24 ++++- scripts/cacbg/audit.mjs | 50 ++++++++-- scripts/cacbg/audit.test.mjs | 95 ++++++++++++++++++- scripts/precompute.sql | 7 ++ scripts/refresh-slice.sql | 7 ++ 17 files changed, 334 insertions(+), 20 deletions(-) diff --git a/apps/web/app/components/ConflictCards.tsx b/apps/web/app/components/ConflictCards.tsx index 4bbfeba4..28783efd 100644 --- a/apps/web/app/components/ConflictCards.tsx +++ b/apps/web/app/components/ConflictCards.tsx @@ -20,6 +20,7 @@ import { officialHref, partitionContracts, relationLabel, + registryEvidenceLabel, temporalLabel, } from '../lib/conflicts'; @@ -181,6 +182,21 @@ function ConflictCard({ )}
+ {/* The Trade Register fact the link's identity rests on (#279, ADR-0033). This is what makes + „every shown link explains itself" true rather than a promise: a reader can open the same act + we read and check it. The wording is careful — the register records a ROLE, it does not + certify the ownership claim, which comes from the official's own declaration. */} +
+
Регистър
+
+ + + {registryEvidenceLabel(l)} + {l.registryEntryDate ? ` · вписване ${l.registryEntryDate}` : ''} + {l.registryLookupDate ? ` · справка ${l.registryLookupDate}` : ''} + +
+
{l.contractCount > 0 && ( diff --git a/apps/web/app/lib/conflicts.test.ts b/apps/web/app/lib/conflicts.test.ts index a4bfd5f8..6cec490a 100644 --- a/apps/web/app/lib/conflicts.test.ts +++ b/apps/web/app/lib/conflicts.test.ts @@ -21,6 +21,7 @@ import { partitionContracts, relationLabel, temporalLabel, + registryEvidenceLabel, } from './conflicts'; function link(over: Partial = {}): ConflictLink { @@ -43,7 +44,13 @@ function link(over: Partial = {}): ConflictLink { contemporaneousValueEur: 40_000_000, firstContractYear: '2021', lastContractYear: '2024', - sourceUrl: 'https://register.cacbg.bg/2024/i.xml', + sourceUrl: 'https://register.cacbg.bg/2024/x.xml', + // #279: a link only reaches the DTO when its identity rests on a Trade Register fact. + evidenceKind: 'document', + registryRole: 'owner', + registryEntryNumber: '20110502101007', + registryEntryDate: '2011-05-02', + registryLookupDate: '2026-08-05', ...over, }; } @@ -523,3 +530,31 @@ describe('authorityShareDisplay', () => { }); }); }); + +describe('registryEvidenceLabel', () => { + // The wording is load-bearing. The register records a ROLE; it does not certify that the official owns + // anything — that claim comes from their own declaration and is rendered separately. A label that said + // „собственик според ТР" would assert something the evidence does not support (ADR-0033 decision 2). + it('reports what the act records, never an ownership conclusion', () => { + expect(registryEvidenceLabel({ evidenceKind: 'document', registryRole: 'owner' })).toBe( + 'лицето е вписано като съдружник/собственик', + ); + expect(registryEvidenceLabel({ evidenceKind: 'document', registryRole: 'manager' })).toBe( + 'лицето е вписано като управител', + ); + }); + + it('a seat/ЕИК confirmation claims identity, not a registry role', () => { + // „Потвърдено" means the COMPANY was identified from something the official declared — nobody was + // found in the act, so the label must not imply anyone was. + const label = registryEvidenceLabel({ evidenceKind: 'confirmed', registryRole: null }); + expect(label).toBe('самоличност, потвърдена по декларирани данни'); + expect(label).not.toMatch(/вписан/); + }); + + it('never renders the word „собственик" for a mere confirmation', () => { + expect(registryEvidenceLabel({ evidenceKind: 'confirmed', registryRole: 'owner' })).not.toMatch( + /собственик/, + ); + }); +}); diff --git a/apps/web/app/lib/conflicts.ts b/apps/web/app/lib/conflicts.ts index 50a6f16b..5b99180e 100644 --- a/apps/web/app/lib/conflicts.ts +++ b/apps/web/app/lib/conflicts.ts @@ -24,6 +24,24 @@ const RELATION_LABEL: Record = { }; /** Bulgarian label for a declared relation. Unknown values pass through — never invent a stronger claim. */ +/** + * How the company's identity was established, in the register's own terms (#279, ADR-0033). + * + * Deliberately does NOT say the official owns anything: „вписан съдружник/собственик" reports what the + * act RECORDS, while the ownership claim itself comes from the official's own declaration and is + * rendered separately as „дялово участие". „Потвърдено" means the company was identified by a fact the + * official declared — the seat or the ЕИК — not that anybody was found in the act. + */ +export function registryEvidenceLabel(l: { + evidenceKind: 'document' | 'confirmed'; + registryRole: 'owner' | 'manager' | null; +}): string { + if (l.evidenceKind === 'confirmed') return 'самоличност, потвърдена по декларирани данни'; + return l.registryRole === 'manager' + ? 'лицето е вписано като управител' + : 'лицето е вписано като съдружник/собственик'; +} + export function relationLabel(relation: string): string { return RELATION_LABEL[relation] ?? relation; } diff --git a/apps/web/app/routes/conflict.pages.render.test.tsx b/apps/web/app/routes/conflict.pages.render.test.tsx index c6f2498c..fbfc4fdb 100644 --- a/apps/web/app/routes/conflict.pages.render.test.tsx +++ b/apps/web/app/routes/conflict.pages.render.test.tsx @@ -34,7 +34,13 @@ function link(over: Partial = {}): ConflictLink { contemporaneousValueEur: 30_000_000, firstContractYear: '2020', lastContractYear: '2024', - sourceUrl: 'https://register.cacbg.bg/2024/i.xml', + sourceUrl: 'https://register.cacbg.bg/2024/x.xml', + // #279: a link only reaches the DTO when its identity rests on a Trade Register fact. + evidenceKind: 'document', + registryRole: 'owner', + registryEntryNumber: '20110502101007', + registryEntryDate: '2011-05-02', + registryLookupDate: '2026-08-05', ...over, }; } diff --git a/apps/web/app/routes/conflicts.render.test.tsx b/apps/web/app/routes/conflicts.render.test.tsx index 68dad5fb..ce2a78db 100644 --- a/apps/web/app/routes/conflicts.render.test.tsx +++ b/apps/web/app/routes/conflicts.render.test.tsx @@ -36,6 +36,12 @@ function link(over: Partial = {}): ConflictLink { firstContractYear: '2020', lastContractYear: '2024', sourceUrl: 'https://register.cacbg.bg/2024/i.xml', + // #279: a link only reaches the DTO when its identity rests on a Trade Register fact. + evidenceKind: 'document', + registryRole: 'owner', + registryEntryNumber: '20110502101007', + registryEntryDate: '2011-05-02', + registryLookupDate: '2026-08-05', ...over, }; } @@ -238,3 +244,27 @@ describe('/conflicts route — render', () => { ).not.toBeNull(); }); }); + +describe('Trade Register evidence on the card (#279, ADR-0033)', () => { + it('renders the registry fact the link rests on, so the card explains itself', async () => { + await renderConflicts([link({ evidenceKind: 'document', registryRole: 'owner' })]); + const text = container.textContent ?? ''; + expect(text).toContain('Регистър'); + expect(text).toContain('лицето е вписано като съдружник/собственик'); + expect(text).toContain('вписване 2011-05-02'); // WHICH entry + expect(text).toContain('справка 2026-08-05'); // and HOW FRESH it is + }); + + it('a seat/ЕИК confirmation never implies somebody was found in the act', async () => { + await renderConflicts([link({ evidenceKind: 'confirmed', registryRole: null })]); + const text = container.textContent ?? ''; + expect(text).toContain('самоличност, потвърдена по декларирани данни'); + expect(text).not.toContain('вписано като'); + }); + + it('links out to the register so a reader can check the same act we read', async () => { + await renderConflicts([link({ eik: '201122335' })]); + const hrefs = [...container.querySelectorAll('a')].map((a) => a.getAttribute('href') ?? ''); + expect(hrefs.some((h) => h.includes('201122335'))).toBe(true); + }); +}); diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index d1d9f709..9f6fe164 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -705,6 +705,15 @@ export interface ConflictLink { firstContractYear: string | null; lastContractYear: string | null; sourceUrl: string | null; // a representative declaration URL — provenance, never a fabricated value + // Trade Register evidence (#279, ADR-0033). A link only reaches this DTO when its identity rests on a + // checkable registry fact, so these describe WHICH fact — the surface's whole point is that every shown + // link can explain itself. `registryRole` is the role the register records, NOT a claim about who owns + // what: the ownership claim comes from the official's own declaration. + evidenceKind: 'document' | 'confirmed'; // the only two rungs that publish + registryRole: 'owner' | 'manager' | null; // set only for evidenceKind='document' + registryEntryNumber: string | null; // TEXT — a fieldEntryNumber exceeds the exact-integer range + registryEntryDate: string | null; // the registry entry the evidence rests on + registryLookupDate: string; // when the deed was read — the freshness bound on the claim } /** One contract of a linked winner, marked by whether it was signed during the declared-stake window. diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index efd769fe..ea4de1ce 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -16,6 +16,11 @@ const migration3 = readFileSync( resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'), 'utf8', ); +// …and 0006, which those same blocks now join for the Trade Register evidence gate (#279, ADR-0033). +const migration6 = readFileSync( + resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'), + 'utf8', +); const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); const normalize = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); const precompute = readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8'); @@ -66,7 +71,7 @@ function build(path: 'normalize' | 'refresh'): DatabaseSync { const db = new DatabaseSync(':memory:'); db.exec(schema); db.exec(migration2); - db.exec(migration3); + db.exec(migration3 + migration6); db.exec(staging); db.exec(seed); if (path === 'normalize') { diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index df67b27e..9ffdea5b 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -11,6 +11,8 @@ const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // refresh-slice.sql's officials block reads interest_links (0003); build it so the script doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). +const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -40,6 +42,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index 52acac25..bf81ae93 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -28,6 +28,8 @@ const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bid const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // precompute.sql's officials block reads interest_links (0003); build it so precompute doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). +const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); const precomputePath = resolve(root, 'scripts/precompute.sql'); function sqlite(dbPath: string, sql: string): void { @@ -71,6 +73,7 @@ function freshDb(): string { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); sqlite(dbPath, CLEAN_FIXTURE); return dbPath; } diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index b92c174b..fed25dda 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -132,6 +132,7 @@ describe('served migrations', () => { ); readScript(dbPath, migration2); readScript(dbPath, migration3); + readScript(dbPath, migration6); readScript(dbPath, backfill); readScript(dbPath, precompute); diff --git a/packages/db/src/queries/related-persons.ts b/packages/db/src/queries/related-persons.ts index ce15ff2b..ff99e7f5 100644 --- a/packages/db/src/queries/related-persons.ts +++ b/packages/db/src/queries/related-persons.ts @@ -82,6 +82,11 @@ interface LinkRow { first_contract_year: string | null; last_contract_year: string | null; source_url: string | null; + evidence_kind: string | null; + registry_role: string | null; + entry_number: string | null; + entry_date: string | null; + lookup_date: string | null; } // The winner's contracts, joined exactly as the ETL aggregate does (contracts→tenders→authorities→bidders, @@ -119,8 +124,15 @@ export const NEXUS_ORDER = `(il.own_institution = 'exact') DESC, (contemporaneou // The two surfaced ownership classes (ADR-0032): the official's own stake (private_ownership) and a close // relative's (family_ownership). Anchored once here + in LINK_CONTRACTS_SQL so the read gate and the // drill-down never drift. +// …and the identity must rest on a Trade Register fact (#279, ADR-0033). `status='published'` already +// encodes the loader's decision, so this EXISTS is belt-and-braces: it makes the read path refuse a link +// whose seal is missing or whose evidence is a withholding rung, even if a future writer sets status +// wrongly. 'document' and 'confirmed' are the only two rungs that publish; bar_joint_stock, unknown, +// refuted and outside_tr never reach a reader. export const SURFACED_OWNERSHIP = `il.status = 'published' - AND il.interest_class IN ('private_ownership', 'family_ownership')`; + AND il.interest_class IN ('private_ownership', 'family_ownership') + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed'))`; // Redundant-family collapse (ADR-0032, per todorkolev review). A family link is DROPPED when the SAME official // already has a published OWN stake in the SAME winner. Rendering both a self row and a family row for one // (official, ЕИК) is a de-anonymisation vector: the office-holder is himself in that company's Търговски @@ -159,8 +171,13 @@ export const LINK_SELECT = `SELECT il.link_key, il.person_id, p.name AS official -- (name, institution), ADR-0026; same subquery the search projection uses). Correlated per row, but the -- leaderboard is ≤1000 rows and hourly-cached, so the extra scan is immaterial. (SELECT d.institution FROM declarations d WHERE d.person_id = il.person_id - ORDER BY d.declared_year DESC LIMIT 1) AS institution + ORDER BY d.declared_year DESC LIMIT 1) AS institution, + -- The evidence the link rests on, so the card can explain itself (ADR-0033 decision 7). LEFT JOIN + -- rather than an inner one: SURFACED_OWNERSHIP already requires a publishing seal, and an inner join + -- here would silently re-filter rather than surface a contradiction. + ev.evidence_kind, ev.registry_role, ev.entry_number, ev.entry_date, ev.lookup_date FROM interest_links il + LEFT JOIN interest_link_evidence ev ON ev.link_key = il.link_key JOIN persons p ON p.id = il.person_id JOIN bidders b ON b.id = il.bidder_id WHERE ${SURFACED_OWNERSHIP} @@ -196,6 +213,16 @@ function toLink(r: LinkRow): ConflictLink { firstContractYear: r.first_contract_year, lastContractYear: r.last_contract_year, sourceUrl: r.source_url, + // SURFACED_OWNERSHIP guarantees a publishing seal exists, so these are non-null in practice; the + // fallbacks keep a half-migrated environment rendering rather than throwing on a null read. + evidenceKind: (r.evidence_kind === 'confirmed' ? 'confirmed' : 'document') as + | 'document' + | 'confirmed', + registryRole: + r.registry_role === 'owner' || r.registry_role === 'manager' ? r.registry_role : null, + registryEntryNumber: r.entry_number, + registryEntryDate: r.entry_date, + registryLookupDate: r.lookup_date ?? '', }; } diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 261a2874..82cc2f51 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -13,6 +13,8 @@ const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bid const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // refresh-slice.sql / precompute.sql officials block reads interest_links (0003) — build it in every chain. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). +const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const deriveAmendmentsPath = resolve(root, 'scripts/derive-amendments.sql'); @@ -185,6 +187,7 @@ function initWorkDb(dbPath: string): void { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); } @@ -576,6 +579,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -658,6 +662,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -709,6 +714,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -760,6 +766,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -897,6 +904,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index af338041..1c88b94f 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -22,6 +22,8 @@ import { const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006: SURFACED_OWNERSHIP now requires a Trade Register evidence seal (#279, ADR-0033). +const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }).trim(); @@ -122,6 +124,13 @@ INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, co -- Rollup row for the awarding body — the per-authority capture-share denominator the read query LEFT JOINs. INSERT INTO authority_totals (authority_id, name, spent_eur, contracts, suppliers, avg_eur) VALUES ('a:1','ОБЩИНА ТЕСТ',50000000,10,4,5000000); + +-- Every link carries a Trade Register evidence seal (#279, ADR-0033). SURFACED_OWNERSHIP now requires +-- one, so a fixture without seals renders an EMPTY surface and every assertion below passes vacuously. +-- Derived from interest_links itself, so a row added to the fixture later is sealed automatically and +-- cannot silently drop off the surface. +INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links; `; describe('свързани-лица SQL (real SQLite)', () => { @@ -131,6 +140,7 @@ describe('свързани-лица SQL (real SQLite)', () => { try { readScript(dbPath, migration0); readScript(dbPath, migration2); + readScript(dbPath, migration6); sqlite(dbPath, FIXTURE); return fn(dbPath); } finally { @@ -193,7 +203,9 @@ describe('свързани-лица SQL (real SQLite)', () => { INSERT INTO persons (id, name) VALUES ('person:zero','Нула Тестов'); INSERT INTO interest_links (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES - ('il:zerofam','person:zero|666|family','person:zero','eik:666','666','НУЛА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2021',0,0,NULL,NULL,'published');`, + ('il:zerofam','person:zero|666|family','person:zero','eik:666','666','НУЛА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2021',0,0,NULL,NULL,'published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence);`, ); const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); expect(board.some((r) => r.official === 'Нула Тестов')).toBe(false); // no live contracts → gated out @@ -217,6 +229,8 @@ describe('свързани-лица SQL (real SQLite)', () => { (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES ('il:alen','person:alen|701','person:alen','eik:701','701','АЛЕН КО ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'none',1,'2020','2021',1,1000000,'2021','2021','published'), ('il:boyan','person:boyan|702','person:boyan','eik:702','702','БОЯН КО ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'none',1,'2020','2021',1,10000000,'2021','2021','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES ('t:71','unp71','Обект А','a:1','открита процедура'),('t:72','unp72','Обект Б','a:1','открита процедура'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES @@ -322,6 +336,8 @@ describe('свързани-лица SQL (real SQLite)', () => { (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES ('il:dubl-self','person:dubl|800','person:dubl','eik:800','800','ДУБЪЛ ЕООД','exact_name_key','v1','B_distinctive','owns','private_ownership',0,'none',1,'2020','2022',2,60000,'2021','2022','published'), ('il:dubl-fam','person:dubl|800|family','person:dubl','eik:800b','800','ДУБЪЛ ЕООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2022',2,60000,'2021','2022','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES ('t:80','unp80','Обект Дубъл','a:1','открита процедура'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES ('c:80','t:80','eik:800',60000,'EUR','2021-05-01','Д-80',60000);`, ); @@ -350,6 +366,8 @@ describe('свързани-лица SQL (real SQLite)', () => { (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES ('il:asim-self-held','person:asim|850','person:asim','eik:850','850','АСИМ ЕООД','exact_name_key','v1','C_hold','owns','private_ownership',0,'none',1,'2020','2022',2,40000,'2021','2022','held'), ('il:asim-fam','person:asim|850|family','person:asim','eik:850','850','АСИМ ЕООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2022',2,40000,'2021','2022','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES ('t:85','unp85','Обект Асим','a:1','открита процедура'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES ('c:85','t:85','eik:850',40000,'EUR','2021-05-01','Д-85',40000);`, ); @@ -475,7 +493,9 @@ describe('свързани-лица SQL (real SQLite)', () => { INSERT INTO persons (id, name) VALUES ('person:praz','Празен Тестов'); INSERT INTO interest_links (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES - ('il:praz','person:praz|900','person:praz','eik:900','900','ПРАЗЕН ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'exact',1,'2020','2022',7,7000000,'2021','2022','published');`, + ('il:praz','person:praz|900','person:praz','eik:900','900','ПРАЗЕН ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'exact',1,'2020','2022',7,7000000,'2021','2022','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence);`, ); const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); expect(board.some((r) => r.official === 'Празен Тестов')).toBe(false); // gated out — no live contracts diff --git a/scripts/cacbg/audit.mjs b/scripts/cacbg/audit.mjs index 50ff2933..a072a6d0 100644 --- a/scripts/cacbg/audit.mjs +++ b/scripts/cacbg/audit.mjs @@ -7,7 +7,6 @@ import { DatabaseSync } from 'node:sqlite'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { nameDistinctiveness } from './classify.mjs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); @@ -31,12 +30,21 @@ const published = db ` SELECT il.id, il.link_key, il.person_id, il.eik, il.entity_key, il.match_method, il.publish_tier, il.bidder_id, il.relation, il.contemporaneous, il.contract_value_eur, - b.name AS bidder_name, b.eik_normalized AS bidder_eik, b.eik_valid AS bidder_eik_valid + b.name AS bidder_name, b.eik_normalized AS bidder_eik, b.eik_valid AS bidder_eik_valid, + -- LEFT JOIN, deliberately: a published link with NO seal is the finding, so it must reach the + -- loop rather than be filtered out of it. + e.evidence_kind, e.registry_role, e.matched_fact FROM interest_links il JOIN bidders b ON b.id = il.bidder_id + LEFT JOIN interest_link_evidence e ON e.link_key = il.link_key WHERE il.status = 'published'`, ) .all(); +// The only two evidence rungs that publish (ADR-0033 decision 1). Everything else withholds. +const PUBLISHING_EVIDENCE = new Set(['document', 'confirmed']); +// The closed vocabulary for a sealed matched_fact. Anything else is a potential name leak. +const MATCHED_FACT = /^(?:seat:[\p{Lu} -]+|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + const findings = []; const flag = (link, axis, detail) => findings.push({ axis, link_key: link.link_key, eik: link.eik, detail }); @@ -50,12 +58,16 @@ for (const l of published) { // l.eik is a valid winner BEARING this name key (a stray/mis-attached ЕИК is still caught). The ЕИК+name // double-lock itself is re-proven in the provenance pass below (A_eik_no_provenance). if (!rec) flag(l, 'A_key_missing', `entity_key ${l.entity_key} not found in live bidder set`); - else if (l.publish_tier === 'A_eik') { + else if (l.match_method === 'declared_eik') { + // Identity here is the declarant-provided ЕИК, not the name (ADR-0028), so a name shared by more + // than one winner is legitimate — the ЕИК picks exactly one. Require instead that l.eik is a valid + // winner BEARING this name key, which still catches a stray or mis-attached ЕИК. The ЕИК+name + // double-lock is re-proven independently in the provenance pass below. if (!rec.valid.has(l.eik)) flag( l, 'A_eik_not_winner', - `A_eik published ${l.eik}, not among key ${l.entity_key}'s valid winners {${[...rec.valid].join(',')}}`, + `declared_eik published ${l.eik}, not among key ${l.entity_key}'s valid winners {${[...rec.valid].join(',')}}`, ); } else if (rec.valid.size !== 1) flag( @@ -72,12 +84,34 @@ for (const l of published) { if (!l.bidder_eik_valid) flag(l, 'B_eik_invalid', `published on eik_valid=0 bidder ${l.bidder_name}`); - // C. Tier honesty: B_distinctive must actually be distinctive by the same classifier that gated it. - if (l.publish_tier === 'B_distinctive' && nameDistinctiveness(l.entity_key) !== 'distinctive') + // C. Evidence honesty (#279, ADR-0033). The tier IS the evidence kind now, so the axis that used to + // re-derive name distinctiveness re-derives the publishing rule instead: a published link must + // carry a seal, and that seal must be one of the two rungs that publish. `nameDistinctiveness` + // no longer gates publication at all — it survives only as a withholding filter inside the + // loader — so re-checking it here would assert a rule that is no longer in force. + if (!l.evidence_kind) + flag(l, 'C_no_evidence', `published with no Trade Register evidence seal (${l.bidder_name})`); + else if (!PUBLISHING_EVIDENCE.has(l.evidence_kind)) + flag( + l, + 'C_withholding_evidence', + `published on evidence_kind='${l.evidence_kind}', which withholds (${l.bidder_name})`, + ); + else if (l.evidence_kind !== l.publish_tier) + flag( + l, + 'C_tier_evidence_mismatch', + `publish_tier='${l.publish_tier}' but sealed evidence_kind='${l.evidence_kind}'`, + ); + + // C2. The PII rail, audited rather than assumed: matched_fact is a CLOSED vocabulary and can never + // carry a name. The registry deed's names are read only to produce a boolean and must never reach + // a served column (#279 §9, ADR-0033 decision 5). A schema cannot enforce this; this does. + if (l.matched_fact != null && !MATCHED_FACT.test(l.matched_fact)) flag( l, - 'C_not_distinctive', - `tier B_distinctive but nameDistinctiveness=${nameDistinctiveness(l.entity_key)} (${l.bidder_name})`, + 'C_matched_fact_shape', + `matched_fact='${l.matched_fact}' is outside the closed vocabulary — a name may have leaked`, ); } diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index df36baa8..46d42121 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -26,7 +26,7 @@ const dirs = []; // Build a fixture DB (bidders + declarations + declared_interests + interest_links), run audit.mjs against // it as a subprocess, and return { threw, out } — threw=true iff the audit exited non-zero (a hard finding). -function buildAndAudit({ bidders, decls = [], dis = [], links }) { +function buildAndAudit({ bidders, decls = [], dis = [], links, seals = [] }) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-audit-')); dirs.push(dir); const DB = path.join(dir, 'fixture.sqlite'); @@ -38,10 +38,16 @@ function buildAndAudit({ bidders, decls = [], dis = [], links }) { CREATE TABLE interest_links( id TEXT PRIMARY KEY, link_key TEXT, person_id TEXT, eik TEXT, entity_key TEXT, match_method TEXT, publish_tier TEXT, bidder_id TEXT, relation TEXT, contemporaneous INT, contract_value_eur REAL, status TEXT); + -- The evidence seal (#279, migration 0006). The audit LEFT JOINs it, so it must exist even when a + -- case deliberately leaves a link unsealed — an unsealed published link is itself a finding. + CREATE TABLE interest_link_evidence( + link_key TEXT PRIMARY KEY, evidence_kind TEXT, registry_role TEXT, matched_fact TEXT, + entry_number TEXT, entry_date TEXT, lookup_date TEXT, rules_version TEXT, live_status TEXT); ${bidders.map((b) => `INSERT INTO bidders VALUES (${b});`).join('\n')} ${decls.map((d) => `INSERT INTO declarations VALUES (${d});`).join('\n')} ${dis.map((d) => `INSERT INTO declared_interests(declaration_id, entity_raw) VALUES (${d});`).join('\n')} ${links.map((l) => `INSERT INTO interest_links VALUES (${l});`).join('\n')} + ${seals.map((e) => `INSERT INTO interest_link_evidence(link_key,evidence_kind,registry_role,matched_fact,lookup_date,rules_version,live_status) VALUES (${e});`).join('\n')} `); db.close(); @@ -73,8 +79,9 @@ test('A_eik behind a colliding name, backed by a real ЕИК+name double-lock, P // the declarant wrote BOTH the ЕИК and the фирма → the double-lock the loader required dis: [`'d1','„ОБЩ" ЕООД, ЕИК 100000001'`], links: [ - `'il1','p1|100000001','p1','100000001','${KEY}','declared_eik','A_eik','b1','owns',0,1000,'published'`, + `'il1','p1|100000001','p1','100000001','${KEY}','declared_eik','confirmed','b1','owns',0,1000,'published'`, ], + seals: [`'p1|100000001','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], }); assert.equal( threw, @@ -93,8 +100,9 @@ test('A_eik published on a ЕИК that is NOT a winner bearing the name key → dis: [`'d1','„ОБЩ" ЕООД, ЕИК 300000003'`], // entity_key claims the colliding name, but the published eik/bidder is the unrelated 300000003 links: [ - `'il1','p1|300000003','p1','300000003','${KEY}','declared_eik','A_eik','b3','owns',0,1000,'published'`, + `'il1','p1|300000003','p1','300000003','${KEY}','declared_eik','confirmed','b3','owns',0,1000,'published'`, ], + seals: [`'p1|300000003','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], }); assert.equal(threw, true, 'a stray-ЕИК A_eik link must fail the gate'); assert.equal( @@ -110,8 +118,9 @@ test('A_eik with no declaration carrying its ЕИК+name → A_eik_no_provenance decls: [`'d1','p1'`], dis: [`'d1','нещо съвсем друго без ЕИК'`], // no ЕИК, no фирма → the double-lock cannot be re-proven links: [ - `'il1','p1|100000001','p1','100000001','${K('„ОБЩ" ЕООД')}','declared_eik','A_eik','b1','owns',0,1000,'published'`, + `'il1','p1|100000001','p1','100000001','${K('„ОБЩ" ЕООД')}','declared_eik','confirmed','b1','owns',0,1000,'published'`, ], + seals: [`'p1|100000001','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], }); assert.equal( threw, @@ -130,9 +139,85 @@ test('a NON-A_eik (name-based) colliding link STILL fails A_multi_eik — the na bidders: COLLIDING_BIDDERS, // exact_name_key published on a name that maps to 2 ЕИК — exactly what the gate must keep rejecting links: [ - `'il1','p1|100000001','p1','100000001','${KEY}','exact_name_key','B_distinctive','b1','owns',0,1000,'published'`, + `'il1','p1|100000001','p1','100000001','${KEY}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'`, ], }); assert.equal(threw, true, 'a name-based colliding published link must still fail'); assert.equal(/A_multi_eik/.test(out), true, 'invariant A still fires for non-A_eik links'); }); + +// ── C: evidence honesty (#279, ADR-0033) ───────────────────────────────────────────────────────── +// The axis that used to re-derive name distinctiveness now re-derives the PUBLISHING rule, because the +// publishing rule changed: identity rests on a registry fact, and nameDistinctiveness survives only as +// a withholding filter inside the loader. Re-checking distinctiveness here would assert a rule that is +// no longer in force — a test that passes while guarding nothing. +test('a published link with NO evidence seal is a hard finding', () => { + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [], // the whole point: nothing sealed it + }); + assert.equal(threw, true, 'publishing without evidence must fail the gate'); + assert.equal(/C_no_evidence/.test(out), true, out); +}); + +test('a link published on a WITHHOLDING rung is a hard finding', () => { + // bar_joint_stock / unknown / outside_tr never publish. If one ever reaches status='published', the + // loader and the seal disagree and a claim is on the surface that no evidence supports. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','unknown','b1','owns',0,1000,'published'`, + ], + seals: [`'p1|100000001','unknown',NULL,NULL,'2026-08-05','tr-rules-1','live'`], + }); + assert.equal(threw, true); + assert.equal(/C_withholding_evidence/.test(out), true, out); +}); + +test('a tier that disagrees with its own seal is a hard finding', () => { + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [`'p1|100000001','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], + }); + assert.equal(threw, true); + assert.equal(/C_tier_evidence_mismatch/.test(out), true, out); +}); + +test('a matched_fact outside the closed vocabulary is a hard finding — the name-leak rail', () => { + // #279 §9: the deed's names are read only to produce a boolean and must never reach a served column. + // A schema cannot enforce a vocabulary, so the audit does. This is the shape a leak would take. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','ИВАН ПЕТРОВ ТЕСТОВ','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'a name in matched_fact must fail the gate'); + assert.equal(/C_matched_fact_shape/.test(out), true, out); +}); + +test('a well-formed seal passes every C axis (positive control)', () => { + // Without this, all four negatives above would still pass if the axes fired unconditionally. + const { threw } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, false, 'a correctly sealed link must not fail the gate'); +}); diff --git a/scripts/precompute.sql b/scripts/precompute.sql index c79a26ef..0c10e737 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -245,6 +245,13 @@ FROM interest_links il JOIN persons p ON p.id = il.person_id -- rendering both re-identifies the relative via a ТР owner lookup, and the company is already surfaced -- by the self row. WHERE il.status = 'published' AND il.interest_class IN ('private_ownership', 'family_ownership') + -- …and the identity rests on a Trade Register fact (#279, ADR-0033). This predicate is the THIRD copy + -- of the surface gate — the other two are SURFACED_OWNERSHIP in packages/db/src/queries/related-persons.ts + -- and the sibling block in the other of precompute.sql / refresh-slice.sql. All three must move + -- together: this one feeds the officials search index, so omitting it would keep officials findable + -- whose links no longer surface. + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed')) AND EXISTS (SELECT 1 FROM contracts cc JOIN bidders bb ON bb.id = cc.bidder_id WHERE bb.eik_normalized = il.eik) AND NOT (il.interest_class = 'family_ownership' AND EXISTS ( diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 3f4544be..c3ff6fdb 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -1863,6 +1863,13 @@ FROM interest_links il JOIN persons p ON p.id = il.person_id -- rendering both re-identifies the relative via a ТР owner lookup, and the company is already surfaced -- by the self row. WHERE il.status = 'published' AND il.interest_class IN ('private_ownership', 'family_ownership') + -- …and the identity rests on a Trade Register fact (#279, ADR-0033). This predicate is the THIRD copy + -- of the surface gate — the other two are SURFACED_OWNERSHIP in packages/db/src/queries/related-persons.ts + -- and the sibling block in the other of precompute.sql / refresh-slice.sql. All three must move + -- together: this one feeds the officials search index, so omitting it would keep officials findable + -- whose links no longer surface. + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed')) AND EXISTS (SELECT 1 FROM contracts cc JOIN bidders bb ON bb.id = cc.bidder_id WHERE bb.eik_normalized = il.eik) AND NOT (il.interest_class = 'family_ownership' AND EXISTS ( From 796903ae4646abc97c48e3a82dd1b12ce02f173a Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 09:20:04 +0300 Subject: [PATCH 11/69] docs/ci: publish the rule, and put the decisions on a cadence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #279 §9 asks for the publishing decisions to run on a schedule and for the rule to be public. ADR-0021 E10 makes the second one a libel-defence obligation, not a nicety: a reader must be able to re-derive any link. The methodology page now carries the evidence ladder VERBATIM — all six rungs, including the two that hide a link — and, more importantly, the disclosures that make it honest rather than merely stated: • the register carries NO ЕГН, so a three-name match is not proof of identity and a namesake is possible. Said in those words, not implied. • which is why the match must be all three names inside ONE registry entry; initials and Latin-script names are refused. • seats move, so a registered seat only confirms a period it predates. • erased entries are skipped everywhere. • what the evidence proves and what it does not: the register confirms the IDENTITY OF THE COMPANY; it does not certify that the official owns it — that claim is their own declaration. This is why the card says „вписано като съдружник/собственик" and never „собственик според ТР". • the monthly cadence, the entry number and lookup date on every link, and the 35-day retention with no third-party name kept. The divestment promise is corrected too: withdrawal on a later silent filing is an INFERENCE FROM SILENCE whose commonest cause is a finished mandate, so it is now reconciled against the live act before it takes effect. Cadence, deliberately on the EXISTING workflow rather than a second one. A duplicate would carry its own copy of the credential guards, the D1-target guard and the ship floor — and the copy is the one nobody exercises. A scheduled run takes the same path, targeting STAGING (production stays manual, so a prod write remains a deliberate act) and with full_crawl false, so it never touches the register: decisions are a pure function of declarations, cached deeds and contracts, which is exactly what §9 means. Registry lookups get their own monthly workflow because they are the only step with a network budget, and separating them is what keeps the decision run zero-network by construction. It ships nothing, caches the deeds between runs so a refresh stays ~400 requests rather than a full crawl, and treats exit 2 (the rate limiter) as a WARNING, not a failure — the run stopped politely and the cache is resumable, and a red X there would train everyone to ignore the one signal that means „the register asked us to stop". The uploaded artifact is the cache INDEX only; the raw deeds carry third-party names and never leave the runner. Spec §3.3's Phase-0 is closed with what was measured, including the three facts that contradicted the issue: the sustained 429, „not in the register" being an HTTP 200 with an empty body, and fetch failing where node:https succeeds. The LIA gains an addendum for the new source and the runbook two new takedown grounds — a court-annulled entry (чл. 29 ЗТРРЮЛНЦ) and a namesake match. Neither looks like a bug: the pipeline works correctly and the claim is still wrong, so neither is fixed by a rules bump. --- .github/workflows/related-persons-data.yml | 23 +++- .../workflows/related-persons-tr-refresh.yml | 114 ++++++++++++++++++ apps/web/app/routes/conflict.methodology.tsx | 64 +++++++++- docs/runbooks/related-persons-suppression.md | 15 +++ docs/spec/related-persons-foundation.md | 8 ++ docs/spec/related-persons-lia.md | 34 ++++++ 6 files changed, 252 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/related-persons-tr-refresh.yml diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index 16f35c7e..c09bed4d 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -16,6 +16,19 @@ name: Related-persons data foundation (build + ship) # XML (fetch.mjs skips files already on disk) — it fills gaps, not a from-scratch rebuild unless the cache is cleared. on: + # DAILY, and deliberately on the EXISTING workflow rather than a second one. #279 §9 asks for the + # publishing decisions to run on a cadence; duplicating the job would create a second ship path with + # its own copy of the credential guards, the D1-target guard and the ship floor — and the copy is the + # one nobody exercises. A scheduled run takes the same path as a manual one, with two differences: + # • it targets STAGING (the environment default below). Production stays manual, so the + # assertD1TargetAuthorized guard keeps a prod write a deliberate act. + # • `full_crawl` is empty and therefore false, so it never crawls the register — it re-runs the + # decisions over the cached corpus, which is exactly what #279 §9 means by a pure, zero-network + # decision. Registry lookups have their own workflow (related-persons-tr-refresh.yml). + # If the corpus cache has been evicted, extract yields nothing, the ship floor refuses, and the run + # fails loudly rather than wiping the surface. + schedule: + - cron: '0 3 * * *' workflow_dispatch: inputs: environment: @@ -34,7 +47,7 @@ permissions: concurrency: # Never let two data-foundation writes to the same environment overlap; do not cancel one in flight. - group: related-persons-data-${{ inputs.environment }} + group: related-persons-data-${{ inputs.environment || 'staging' }} cancel-in-progress: false jobs: @@ -46,7 +59,7 @@ jobs: # (stalled gov-server I/O) — fail rather than burn the slot. The raw corpus is cached (below) so a # re-run resumes instead of re-crawling from scratch. timeout-minutes: 300 - environment: ${{ inputs.environment }} + environment: ${{ inputs.environment || 'staging' }} env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} @@ -54,7 +67,7 @@ jobs: SIGMA_D1_NAME: ${{ vars.SIGMA_D1_NAME }} # The DECLARED intended environment — the independent anchor the ship guard checks the (name, id) pair # against (T48), so a consistent-but-wrong pair can't wipe the wrong DB. See assertD1TargetAuthorized. - SIGMA_SHIP_ENV: ${{ inputs.environment }} + SIGMA_SHIP_ENV: ${{ inputs.environment || 'staging' }} # Suppression (takedown) list keying — load.mjs fingerprints link-suppressions.jsonl under this salt and # refuses to build if the list is non-empty but the salt is unset (fail-closed, ADR-0031). Without this # the first suppression entry would break every ship; the key version guards a coordinated salt rotation. @@ -93,14 +106,14 @@ jobs: sigma-blue|sigma-green) IS_PROD_SLOT=1 ;; *) IS_PROD_SLOT=0 ;; esac - if [ "${{ inputs.environment }}" = "production" ]; then + if [ "${{ inputs.environment || 'staging' }}" = "production" ]; then if [ "$IS_PROD_SLOT" != "1" ]; then echo "::error::production requires SIGMA_D1_NAME to be a prod slot (sigma-blue|sigma-green), got '${SIGMA_D1_NAME}' — refusing to ship." exit 1 fi else if [ "$IS_PROD_SLOT" = "1" ]; then - echo "::error::SIGMA_D1_NAME='${SIGMA_D1_NAME}' (a PRODUCTION slot) on a '${{ inputs.environment }}' run — a rebuild would WIPE prod. Set a non-production SIGMA_D1_NAME variable for this Environment." + echo "::error::SIGMA_D1_NAME='${SIGMA_D1_NAME}' (a PRODUCTION slot) on a '${{ inputs.environment || 'staging' }}' run — a rebuild would WIPE prod. Set a non-production SIGMA_D1_NAME variable for this Environment." exit 1 fi fi diff --git a/.github/workflows/related-persons-tr-refresh.yml b/.github/workflows/related-persons-tr-refresh.yml new file mode 100644 index 00000000..807c103c --- /dev/null +++ b/.github/workflows/related-persons-tr-refresh.yml @@ -0,0 +1,114 @@ +name: Related-persons — Trade Register cache refresh (monthly) + +# Refreshes the Търговски регистър deed cache the publishing decision rests on (#279, ADR-0033). +# +# WHY THIS IS ITS OWN WORKFLOW, separate from the decision run: this is the only step in the project +# that touches a public register at volume, and it is the only one with a network budget. The decisions +# themselves are a pure function of declarations, cached deeds and contracts — they must be able to run +# daily without ever calling the register. Splitting them keeps that true by construction. +# +# It SHIPS NOTHING. It writes only the cache under scratch/tr/, which is git-ignored and never leaves +# the runner except as an artifact. A run that fails leaves the previous cache intact. +# +# Pace and refusals live in scripts/tr/fetch-deeds.mjs and are not configurable here on purpose: +# 1 request / 3 s, sequential, a closed candidate set, and a 429 ends the run (exit 2) without marking +# anything. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the limiter is the +# operator's only way to state a rate preference, so we do not tune around it. + +on: + schedule: + # 02:00 UTC on the 1st. Monthly, per #279 §9 — a deed changes rarely, and every extra pass is + # ~400 requests against somebody else's service for almost no new information. + - cron: '0 2 1 * *' + workflow_dispatch: + inputs: + max_age_days: + description: Re-fetch deeds older than this many days (default 30). Use a large value to force a full refresh. + type: string + default: '30' + limit: + description: Stop after this many lookups (blank = all pending). Bound a first run while watching for a 429. + type: string + default: '' + +concurrency: + # Never two crawls at once: they would double the request rate against the register, which is the one + # thing the pacing exists to prevent. Queue rather than cancel — a cancelled crawl wastes its progress. + group: tr-cache-refresh + cancel-in-progress: false + +permissions: + contents: read + +jobs: + refresh: + runs-on: ubuntu-latest + # ~400 candidates at 1 per 3 s is ~20 minutes; the ceiling allows retries and a slower day without + # letting a hung run sit for the 6-hour default. + timeout-minutes: 60 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24 + cache: pnpm + + # The cache is the whole point of the workflow: without restoring it, every run would re-fetch + # every deed and turn a 400-request refresh into a 400-request full crawl each month. + - name: Restore the deed cache + id: cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.2 + with: + path: scratch/tr + key: tr-deed-cache-${{ github.run_id }} + restore-keys: | + tr-deed-cache- + + - run: pnpm install --frozen-lockfile + + # The closed candidate set. It is produced by the decision run (load.mjs writes it), so a cache + # refresh can never invent an ЕИК to look up — it can only refresh what the corpus already + # resolved. Absent on a first run, in which case there is nothing to refresh yet. + - name: Check for a candidate list + id: candidates + run: | + if [ -s scratch/cacbg/staging/candidate-eiks.txt ]; then + echo "count=$(wc -l < scratch/cacbg/staging/candidate-eiks.txt)" >> "$GITHUB_OUTPUT" + echo "ok=true" >> "$GITHUB_OUTPUT" + else + echo "No candidate-eiks.txt — run the decision workflow first. Nothing to refresh." + echo "ok=false" >> "$GITHUB_OUTPUT" + fi + + - name: Refresh deeds + if: steps.candidates.outputs.ok == 'true' + env: + MAX_AGE: ${{ inputs.max_age_days || '30' }} + LIMIT: ${{ inputs.limit }} + run: | + set -euo pipefail + ARGS=(--eiks-file scratch/cacbg/staging/candidate-eiks.txt --max-age-days "$MAX_AGE") + [ -n "$LIMIT" ] && ARGS+=(--limit "$LIMIT") + set +e + node scripts/tr/fetch-deeds.mjs "${ARGS[@]}" + code=$? + set -e + # Exit 2 is the rate limiter, and it is NOT a build failure: the run stopped politely, marked + # nothing, and the partial cache is valid and resumable. Failing the workflow here would train + # everyone to ignore a red X on the one signal that means "the register asked us to stop". + if [ "$code" -eq 2 ]; then + echo "::warning::The register rate-limited us; the run stopped and the cache is resumable." + exit 0 + fi + exit "$code" + + - name: Publish the cache index as an artifact (no raw deeds — they carry third-party data) + if: always() && steps.candidates.outputs.ok == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: tr-cache-index + # The SQLite index only. scratch/tr/deeds/*.json holds owner and manager names and company + # addresses (ADR-0033 decision 5) and must not leave the runner. + path: scratch/tr/tr-cache.sqlite + retention-days: 7 diff --git a/apps/web/app/routes/conflict.methodology.tsx b/apps/web/app/routes/conflict.methodology.tsx index 32936a30..c9a432b0 100644 --- a/apps/web/app/routes/conflict.methodology.tsx +++ b/apps/web/app/routes/conflict.methodology.tsx @@ -182,6 +182,64 @@ export default function ConflictMethodology() { връзката е двусмислена и не се публикува — никога не отгатваме кой от субектите е имал предвид деклараторът.

+ +

Проверка в Търговския регистър

+

+ Съвпадението по име вече не е достатъчно, за да се публикува + връзка. За всяко дружество правим отделна справка в{' '} + Търговския регистър (публична, без регистрация) и връзката се + показва само ако самоличността на дружеството е потвърдена от{' '} + проверим факт в акта. Редът е следният — прилага се първото, което + е изпълнено: +

+
    +
  1. + Акционерна форма (АД, ЕАД, КДА) — не се показва никога. Книгата + на акционерите не е публична, тоест твърдението е непроверимо; освен това дялът + може да е миноритарен или борсов, тоест несъществен. +
    2. +
  2. + Документ. И трите имена на лицето се срещат в живо поле на акта — + управители, съдружници, едноличен собственик или физическо лице-търговец. Пазим + ролята (съдружник/собственик или само управител), защото тя не е едно и също. +
    3. +
  3. + Потвърдено. Лицето не е намерено в акта, но декларирано от него + седалище съвпада с регистрираното за този ЕИК, или самият + декларатор е изписал ЕИК. Тук се потвърждава{' '} + кое е дружеството, а не кой го притежава. +
    4. +
  4. + Оборена. Само за собствен дял: лицето не фигурира никъде в акта, + а най-късното вписване за собственост е преди декларирания период + — тоест записът покрива целия период и не назовава лицето. Връзката отпада. +
    5. +
  5. + Неизвестна — всичко останало. Остава скрита. +
    6. +
+

+ Какво доказва това и какво не. Регистърът потвърждава{' '} + самоличността на дружеството; той не удостоверява, че лицето притежава дял + — това твърдение идва от неговата собствена декларация. Затова на картата пише + „лицето е вписано като съдружник/собственик", а не „лицето е собственик според ТР". +

+

+ Ограниченията, честно. Търговският регистър{' '} + не съдържа ЕГН, затова съвпадението по трите имена не е абсолютно + доказателство за самоличност — възможен е съименник. Затова се + изисква пълно съвпадение и на трите имена в{' '} + един и същ запис на акта (две от три не се приема; имена с инициали + или с латински букви не се приемат). Седалищата се променят, затова регистрираното + седалище се приема само ако е вписано преди декларирания период. Заличените записи + се пропускат навсякъде. +

+

+ Справките се опресняват месечно, а датата на справката и номерът на + вписването се показват на всяка връзка — така всяко твърдение носи своя източник и + своята давност. Актовете се пазят само служебно, за срок до 35 дни; имена на трети + лица от тях не се съхраняват и не влизат в никакъв публичен запис. +

@@ -267,7 +325,11 @@ export default function ConflictMethodology() { Прекратен дял. Ако лице подаде по-късна декларация за собственост, която вече не изброява дадено дружество, връзката се смята за приключила и се оттегля от публичната част — така остарял запис - не внушава конфликт, който е приключил. + не внушава конфликт, който е приключил. Това обаче е извод от мълчание: + най-честата причина да спреш да декларираш е изтекъл мандат, а не продажба. Затова + преди оттеглянето сверяваме с живия акт в Търговския регистър — + ако лицето и днес е вписано като съдружник или собственик, дялът не е прекратен и + връзката остава.

diff --git a/docs/runbooks/related-persons-suppression.md b/docs/runbooks/related-persons-suppression.md index ac2b140c..2aaa2d97 100644 --- a/docs/runbooks/related-persons-suppression.md +++ b/docs/runbooks/related-persons-suppression.md @@ -4,6 +4,21 @@ A contested, corrected, or legally-challenged link on the свързани-ли the version-controlled suppression list. See [ADR-0031](../adr/0031-suppressions-version-controlled-fingerprinted.md) for why this is a git list keyed on an HMAC fingerprint, not a DB row. +## Grounds that require a takedown + +Two of these are specific to the Trade Register evidence (#279, ADR-0033) and are easy to miss, because +neither shows up as a bug: the pipeline is working correctly and the claim is still wrong. + +- **A court-annulled entry (чл. 29 ЗТРРЮЛНЦ).** A registry entry set aside by a court is void, but the + deed we cached may still carry it, and the rules that read it are unchanged — so a rules-version bump + will NOT remove the link. This is an evidence-invalidation ground, and suppression is the mechanism. +- **A match against the wrong person (a namesake).** The register carries no ЕГН, so a three-name match + inside one entity is strong evidence of identity but not proof. If the named official is not the person + in the act, the link is a false public claim about them and must come down immediately — do not wait for + a rules change or a re-crawl. +- The ordinary grounds are unchanged: a contested link under review, a correction the source has since + published, or a legal challenge. + ## What you need - The exact `link_key`. Self links are `pid|eik`; a close-relative link is `pid|eik|family`. Get it from the diff --git a/docs/spec/related-persons-foundation.md b/docs/spec/related-persons-foundation.md index 8c50fa94..61d399f5 100644 --- a/docs/spec/related-persons-foundation.md +++ b/docs/spec/related-persons-foundation.md @@ -70,6 +70,13 @@ Of #128's five checkboxes: **1** (shared owners) contingently unblocked via §3. **Per-ЕИК public lookup** (`portal.registryagency.bg`) is free, no fee. Recoverable per ЕИК: **управители, съдружници/собственици на капитала + stakes, капитал, legal form, seat.** **ЕГН masked** → owner identity name-based. Run **only** for the bounded set (bidders + resolved declared/donor companies) — thousands, not ~900k. **Hard constraints (DPA-verified):** **NEVER bulk-scrape TR** (КЗЛД ruled bulk provision unlawful; CJEU C-200/23 climate). **Never store scraped ЕГН**, even if leaked in a document image. Phase-0 confirms endpoint, fields, and rate limits. +**Phase-0 is CLOSED — measured 2026-08-04/05 (#279, ADR-0033).** Endpoint: `GET portal.registryagency.bg/CR/api/Deeds/{ЕИК}`, public JSON, no authentication. Facts worth recording because three of them contradict what the issue predicted: +- **The register rate-limits, and the block is sustained.** HTTP 429 at ~50 cumulative requests ending in a burst, then 429 to everything — no `Retry-After`, no quota header. So a 429 is an instruction to stop, never a transient to retry through: the crawler ends the run, marks nothing, and resumes later. The implemented pace is 1 request / 3 s, sequential, over a **closed** candidate set the crawler cannot extend (it never follows a link out of a deed). +- **„Not in the register" is an HTTP 200 with a ZERO-BYTE body**, not a 404 and not HTML. Verified on Община София (`000696327`): empty twice while a real company returned its full deed in the same window. Permanence therefore keys on the **status**, not the empty body — empty under 200 is an answer and may be cached; empty under 5xx is a failure and stays transient. +- **`fetch` does not work against this host.** The identical request returns 500/empty via undici and 200 with the full deed via `node:https`. +- Field values are HTML fragments inside the JSON envelope, and one field routinely holds several people separated by `

`; erasure is marked structurally (`erasure-text-inline`), not textually. Matching must happen **inside one entity**, or one person's given name combines with another's surname. +- **ЕГН is absent** from every payload examined, and is never stored either way. Raw deeds live only under git-ignored `scratch/tr/` with a 35-day retention; the cache index holds no name at all. + ## 4. Data model New domain tables, built by `normalize-raw.sql` from persistent staging; `interest_links` written by a **JS resolver pass** (§5), NOT by fuzzy SQL. @@ -182,6 +189,7 @@ The public site MUST carry a plain-language methodology page. It is part of the 1. **Sources + legal basis** — CACBG declarations (ЗСП чл. 75, Сметна палата); ЕРИК (Изборен кодекс); targeted TR per-ЕИК (public company facts). Each with a direct source link. 2. **What is shown and what is not** — officials' own declared holdings; **family holdings are NOT published** (v1); addresses/ЕГН never shown. 3. **The matching rule, verbatim** — declared company **full name** (incl. legal form) is normalized and matched **exactly** against the winner's registered name, which is paired 1:1 with its ЕИК in the procurement source; because Bulgarian trade names are **nationally unique** (ЗТРРЮЛНЦ/ТЗ), an exact match is the same legal entity. Ambiguous cases (truncated names, no exact match) are **excluded or human-confirmed, never guessed**. Every link shows its full evidence chain. + **Superseded in part by #279 / ADR-0033:** a name match is no longer *sufficient* to publish. Every link additionally rests on a checkable **Търговски регистър** fact, and the methodology page carries the six-rung evidence ladder verbatim, together with the disclosures that make it honest: the register carries **no ЕГН**, so a three-name match is not proof of identity and a namesake is possible; the match must therefore be a **full three-token subset inside a single registry entity**; seats move, so a registered seat only confirms a period it predates; erased entries are skipped; and every published link shows its **entry number and lookup date**. 4. **Certainty & framing** — only deterministic exact matches (and human-confirmed cases) appear; each is a „**модел за проверка, не обвинение**"; the tool shows *declared* links only and **does not claim to find hidden ownership** (it is a floor, not the full picture). 5. **Temporal meaning** — a declaration is a point-in-time snapshot; how overlap with a contract date is (and isn't) interpreted. 6. **Known gaps** — the recall holes (§5), self-reporting limits, sources it cannot see. diff --git a/docs/spec/related-persons-lia.md b/docs/spec/related-persons-lia.md index 1309c42a..07523940 100644 --- a/docs/spec/related-persons-lia.md +++ b/docs/spec/related-persons-lia.md @@ -128,6 +128,40 @@ v Poland* (2428/05) приема онлайн публикуване на дек регистър. Приемаме този оспорим елемент съзнателно. **Резервен план при оспорване:** един филтър (сваляне на семейните връзки към еднолични дружества), не спиране на повърхността. +## 9а. Допълнение: справки в Търговския регистър (#279, ADR-0033) + +Нов **източник** и нова **обработка**, затова оценката се допълва, а не само се препотвърждава. + +**Каква е обработката.** За всяко дружество-кандидат правим една справка в публичния Търговски регистър и +четем живия акт. Актът съдържа **лични данни на трети лица** — имена на съдружници, собственици и +управители, които не заемат публична длъжност, както и адрес на дружеството. + +**Цел и основание.** Целта е тясна и проверима: да се установи **самоличността на дружеството** зад +декларирано име, преди срещу конкретно лице да бъде публикувано твърдение. Това е мярка, която *намалява* +риска за субектите на данни — досега самоличността се решаваше по евристика за отличителност на името, а +сега — по документ. Основанието е същият надделяващ обществен интерес по ЗДОИ чл. 41и, ал. 4, приложен към +по-точна, а не към по-широка публикация. + +**Минимизиране — какво НЕ правим.** +- Имената от акта се ползват **само в булеви сравнения** и не влизат в никаква публична таблица, отговор или + лог. Индексът на кеша не съдържа нито едно име — само ЕИК, дати, кодове и хеш на тялото. +- Адресът **не се извлича**: от полето за седалище се чете единствено сегментът „Населено място". +- Полетата за **действителен собственик** (`CR_F_550_L`) и за контрол (`537/538`) **не се ползват** — това + би било претенция за действителна собственост, оставена настрана след CJEU C-37/20 (ADR-0007, т. 1). +- Суровите актове се пазят само служебно, в git-игнорирана директория, със **срок 35 дни**. +- Обхватът е **затворен**: справка се прави само за ЕИК, които вече са резултат от съпоставянето; обхождащият + никога не следва връзка навън от акт. Това не е масово изтегляне (спец. §3.3). + +**Остатъчен риск — съименник.** Регистърът **не съдържа ЕГН**, затова съвпадението по трите имена е силно, но +не абсолютно доказателство за самоличност. Смекчаване: изисква се пълно съвпадение и на трите имена в **един и +същ запис** на акта; имена с инициали или с латински букви не се приемат; фамилни съвпадения не се ползват +никъде. Грешката тук не измисля твърдение за собственост — тя **закача длъжностното лице за друго дружество**, +което е също толкова невярно и се сваля по реда на ADR-0031 (виж runbook-а: „съвпадение с грешно лице"). + +**Отложено съзнателно.** Етикетите „и към днешна дата" по т. 7 на #279 **не се пускат** в тази фаза: те +твърдят сегашно време за поименно лице, а давността им е ограничена от цикъла на опресняване. Влизат само +след отделен преглед на настоящата оценка. + ## 10. Статус и преглед Draft за правен преглед; изпраща се паралелно с пускането, не като блокер (ADR-0032, т. 7). Ревизира се при промяна From d50a723b00d03a7abef00670c23e326e8689d008 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 09:39:12 +0300 Subject: [PATCH 12/69] fix(test): two fixtures that only passed because of local state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI caught both; local runs could not, and the reasons are worth recording because they will recur. load-ambiguous.test.mjs ran load.mjs WITHOUT setting TR_CACHE_DB, so it fell back to the repo's real scratch/tr/tr-cache.sqlite — a file that exists on my machine only because the live smoke run created it, and that exists nowhere in CI. The test was silently running against a developer artifact. It now builds its own empty cache and passes the path explicitly, like every other loader test. refresh-officials.test.ts executes the real refresh-slice.sql, which now joins interest_link_evidence, but built its schema from 0000/0001/0003 only. Locally `pnpm test` reported 6/6 because turbo REPLAYED A CACHED RESULT for @sigma/ingest — my change was to scripts/refresh-slice.sql, which is not among that package's declared inputs, so the cache key did not move. `turbo run test --force` fails the same way CI does. Fixture now applies 0006 and seals its links. The seal is derived with a SELECT over interest_links rather than written row by row, so a link added to the fixture later is sealed automatically. That matters more than it looks: the officials batch requires a publishing rung, so an unsealed fixture indexes NOBODY and every assertion below it passes vacuously. Coverage baseline ratcheted up, as check-coverage nudged: apps/web +1.33pp (the new jsdom card tests and the evidence-label tests), packages/db +0.34pp, packages/ingest +0.57pp, packages/shared +0.83pp branches. --- coverage-baseline.json | 16 ++++++------- packages/ingest/src/refresh-officials.test.ts | 8 +++++++ scripts/cacbg/load-ambiguous.test.mjs | 23 ++++++++++++++++++- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/coverage-baseline.json b/coverage-baseline.json index 5abe85c9..064f448f 100644 --- a/coverage-baseline.json +++ b/coverage-baseline.json @@ -6,24 +6,24 @@ "branches": 58.2 }, "apps/web": { - "lines": 89.7, - "branches": 81.8 + "lines": 91, + "branches": 82.4 }, "packages/config": { "lines": 92.8, "branches": 72.2 }, "packages/db": { - "lines": 94.2, - "branches": 79 + "lines": 94.5, + "branches": 79.3 }, "packages/ingest": { - "lines": 85.8, - "branches": 80 + "lines": 86.3, + "branches": 80.4 }, "packages/shared": { - "lines": 95.4, - "branches": 80 + "lines": 95.5, + "branches": 80.8 } } } diff --git a/packages/ingest/src/refresh-officials.test.ts b/packages/ingest/src/refresh-officials.test.ts index 55170ef2..6f3eb4c6 100644 --- a/packages/ingest/src/refresh-officials.test.ts +++ b/packages/ingest/src/refresh-officials.test.ts @@ -20,6 +20,9 @@ const migrations = [ '0000_init.sql', '0001_flow_pairs_bidder_index.sql', '0003_related_persons_foundation.sql', + // The officials batch now joins the Trade Register evidence seal (#279, ADR-0033) — without 0006 the + // real refresh-slice.sql this test executes cannot parse. + '0006_interest_link_evidence.sql', ].map((f) => resolve(root, 'packages/db/migrations', f)); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); @@ -67,6 +70,11 @@ INSERT INTO interest_links -- her — dropping the s.status='published' guard would collapse it behind an invisible self stake (false neg). ('il:ys','person:ЯНА|444','person:ЯНА','eik:444','444','ДЕЛТА ООД','exact_name_key','v1','C_hold','owns','private_ownership',0,'none',1,'2020','2023',1,20000,'2021','2021','held'), ('il:yf','person:ЯНА|444|family','person:ЯНА','eik:444','444','ДЕЛТА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',1,'none',1,'2020','2023',1,20000,'2021','2021','published'); +-- Every link carries an evidence seal (#279, ADR-0033): the officials batch requires a publishing rung, +-- so a fixture without seals indexes NOBODY and every assertion below passes vacuously. Derived from +-- interest_links, so a row added later is sealed automatically. +INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links; `; describe('ETL refresh-slice officials batch (the live parse-then-execute path)', () => { diff --git a/scripts/cacbg/load-ambiguous.test.mjs b/scripts/cacbg/load-ambiguous.test.mjs index 336cfb30..01cf8f7a 100644 --- a/scripts/cacbg/load-ambiguous.test.mjs +++ b/scripts/cacbg/load-ambiguous.test.mjs @@ -45,12 +45,33 @@ function buildAndLoad(bidderRows) { fs.writeFileSync(path.join(STAGING, 'holdings.jsonl'), ''); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + // An EMPTY Trade Register cache, pointed at explicitly (#279, ADR-0033). This fixture declares no + // holdings, so it resolves no links and the coverage gate is satisfied by an empty candidate set — + // but the cache FILE must still exist, because a missing one refuses the whole load. + // + // Explicit rather than defaulted, and that matters: load.mjs falls back to the repo's real + // scratch/tr/tr-cache.sqlite, so a test that omits TR_CACHE_DB silently runs against whatever the + // developer's last live crawl left behind. That is exactly how this test passed locally and failed + // in CI, where no such file exists. + const trDb = path.join(dir, 'tr-cache.sqlite'); + new DatabaseSync(trDb).close(); + let threw = false; try { execFileSync( 'node', ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], - { cwd: ROOT, env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, stdio: 'pipe' }, + { + cwd: ROOT, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: trDb, + TR_RAW_DIR: path.join(dir, 'tr-deeds'), + }, + stdio: 'pipe', + }, ); } catch { threw = true; // execFileSync throws on a non-zero exit code (a fired hard-gate) From 7a424bda21620fa3da71a1050636e932e5722aea Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:29:59 +0300 Subject: [PATCH 13/69] fix(tr): accept double-quoted attributes in the deed parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The entity split and the erasure strip both matched class='...' only. The live register emits single quotes today and every fixture is verbatim from a real deed, so nothing is broken now — but the failure mode if that ever changes is the worst one this module has: the split stops firing, all owners in a field collapse into one token pool, and a name assembled from two different people starts matching. That is a named public claim about someone who is not there. deed.mjs's own doctrine is "refuse loudly, never guess". Silently mis-splitting is neither, so match either quote style instead. The new split test strips the
separators first. With them present it passes on the
rule alone and proves nothing about the quote handling — verified by watching it pass before the fix. --- scripts/tr/deed.mjs | 4 ++-- scripts/tr/deed.test.mjs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/scripts/tr/deed.mjs b/scripts/tr/deed.mjs index 7709e31c..fdb3c1bd 100644 --- a/scripts/tr/deed.mjs +++ b/scripts/tr/deed.mjs @@ -64,7 +64,7 @@ export function entityBlocks(html, { strict = false } = {}) { const decoded = decodeEntities(html); // decode BEFORE splitting — see the order above const chunks = decoded .split(/]*>/i) - .flatMap((part) => part.split(/(?=]*class='[^']*record-container)/i)) + .flatMap((part) => part.split(/(?=]*class=['"][^'"]*record-container)/i)) .map((c) => c.trim()) .filter((c) => c !== ''); @@ -74,7 +74,7 @@ export function entityBlocks(html, { strict = false } = {}) { // Read the visible text WITHOUT the erasure notice, so „Заличено обстоятелство." never counts as // content and an erased block reads as empty. const withoutNotice = chunk.replace( - /]*class='[^']*erasure-text-inline[^']*'[^>]*>.*?<\/div>/gis, + /]*class=['"][^'"]*erasure-text-inline[^'"]*['"][^>]*>.*?<\/div>/gis, ' ', ); const text = stripTags(withoutNotice); diff --git a/scripts/tr/deed.test.mjs b/scripts/tr/deed.test.mjs index 6ec634ef..9a7f4c97 100644 --- a/scripts/tr/deed.test.mjs +++ b/scripts/tr/deed.test.mjs @@ -75,6 +75,36 @@ test('T1 — tokens from two DIFFERENT people never combine into a match', () => assert.equal(fullSubsetMatch(frankenstein, F19_THREE), true, 'naive whole-field match fires'); }); +// The live register emits single-quoted attributes today, and every fixture above is verbatim from a +// real deed. But the entity split is the ONLY thing standing between three people and one merged +// token pool, and it must not be quiet about a markup change: if the register ever switches to +// class="…", a quote-specific pattern stops splitting, all three owners collapse into one block, and +// the frankenstein match above starts firing — a named public claim about a person who is not there. +// R7's doctrine for this module is „refuse loudly, never guess"; silently mis-splitting is neither. +const dq = (s) => s.replace(/class='([^']*)'/g, 'class="$1"'); + +test('T1 — the entity split survives double-quoted attributes (markup-drift hardening)', () => { + // Strip the
separators first. entityBlocks splits on BOTH
and record-container precisely + // because either may be absent; with the
present this test would pass on the
rule alone + // and prove nothing about the quote handling it is here to pin. + const blocks = entityBlocks(dq(F19_THREE).replace(/]*>/gi, '')); + assert.equal(blocks.length, 3, 'a quote style change must not merge three owners into one block'); + assert.equal( + blocks.some((b) => fullSubsetMatch('ПЕНКО КОСТАДИНОВ ФИЛИПОВ', b.text)), + false, + 'cross-entity match under double quotes — the libel bug via markup drift', + ); + assert.ok(blocks.some((b) => fullSubsetMatch('ПЕНКО НЕСТОРОВ НЕСТОРОВ', b.text))); +}); + +test('T1 — erasure is still detected and stripped under double-quoted attributes', () => { + const [block] = entityBlocks(dq(F23_ERASED)); + assert.equal(block.erased, true); + assert.equal(block.text, '', 'the erasure notice must not survive as content'); + // The strict contradiction check must not fire merely because the quote style changed. + assert.doesNotThrow(() => entityBlocks(dq(F23_ERASED), { strict: true })); +}); + test('T1 positive control — the CORRECT declarant does match', () => { // Without this, a matcher that always returns false passes every negative test above (ADR-0027). const blocks = entityBlocks(F19_THREE); From 299400e61dfe0136b58498b18525546faccca600 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:30:29 +0300 Subject: [PATCH 14/69] =?UTF-8?q?fix(tr):=20anchor=20the=20=D0=95=D0=93?= =?UTF-8?q?=D0=9D=20guard=20so=20a=2013-digit=20=D0=95=D0=98=D0=9A=20canno?= =?UTF-8?q?t=20abort=20the=20crawl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assertNoEgnShape refused any value matching /\d{10}/, justified in three places by "an ЕИК is 9 or 13 digits, never 10 — so it cannot reject a legitimate identifier". That reasoning only holds if the run is matched as a WHOLE. A 13-digit ЕИК (клон) CONTAINS ten-digit substrings, and on the fetched path rawPath is `.json`, derived from that very ЕИК: /\d{10}/.test("1155361790001.json") === true upsertDeed sits past fetch-deeds' JSON.parse/assertUicEcho try-catch, so the throw propagated and killed the whole run on the first branch office that returned a deed — after its paced request was spent. Fail-closed for correctness, but the crawl is the data source for the entire feature. The existing test asserted this exact case was safe and passed anyway: markOutsideTr sets no rawPath, and the fetched fixture used a 9-digit ЕИК, so the one path that throws was never exercised. Anchor the pattern, and screen by EXCLUSION rather than a hand-maintained list of four field names — upsertDeed binds thirteen values and the next one added would have bypassed the rail silently. Two fields stay exempt for structural reasons, not convenience. `eik` is already shape-validated by safeEik. `bodySha256` is 64 chars of [0-9a-f]: measured over 200k digests, a standalone ten-digit run occurs in 7.2% of them (18.1% unanchored), so screening it would refuse roughly one deed in fourteen for no privacy gain. A digest is not personal data — it exists precisely so that no deed content reaches the index. --- scripts/tr/cache.mjs | 35 ++++++++++++++++++++++++----------- scripts/tr/cache.test.mjs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/scripts/tr/cache.mjs b/scripts/tr/cache.mjs index 673c6b47..846c4eb9 100644 --- a/scripts/tr/cache.mjs +++ b/scripts/tr/cache.mjs @@ -7,9 +7,11 @@ // exist only in the raw JSON under git-ignored scratch/, are read only to produce a boolean, and // never enter a public table, a response or a log. A hash rather than an excerpt, because an // excerpt of a deed IS third-party personal data. -// 2. Nothing may carry a ten-digit run. That is the ЕГН shape, and the check is sound precisely -// because an ЕИК is 9 or 13 digits — never 10 — so it cannot reject a legitimate identifier. -// ЕГН was absent from every payload examined; this is the rail for the day one leaks. +// 2. Nothing may carry a STANDALONE ten-digit run. That is the ЕГН shape, and the check is sound +// precisely because an ЕИК is 9 or 13 digits — never 10 — so it cannot reject a legitimate +// identifier. „Standalone" is load-bearing: a 13-digit ЕИК contains ten-digit substrings, so an +// unanchored match would refuse every клон. ЕГН was absent from every payload examined; this is +// the rail for the day one leaks. // // Resumability is the other job: the register throttles hard and a 429 ends the run (client.mjs), so a // crawl must be able to pick up exactly where it stopped without re-requesting what it already has. @@ -49,21 +51,33 @@ export function openCache(file) { } // ── the ЕГН rail ────────────────────────────────────────────────────────────── -const TEN_DIGITS = /\d{10}/; +// ANCHORED, and that is the whole correctness of the rail. An ЕИК is 9 or 13 digits — never 10 — so +// a ten-digit run cannot be a legitimate identifier here. But that reasoning only holds when the run +// is matched as a WHOLE: an unanchored /\d{10}/ matches INSIDE the 13-digit ЕИК of a клон, and +// rawPath on the fetched path is `.json`, so every branch office would be refused and the crawl +// would abort on the first one. +const EGN_SHAPE = /(? assert.doesNotThrow(() => markOutsideTr(db, '1155361790001', 'клон')); })); +// A 13-digit ЕИК (клон/подразделение) CONTAINS ten-digit substrings, so an unanchored /\d{10}/ +// rejects it — and rawPath on the fetched path is `.json`, derived from that very ЕИК. This is +// the exact shape fetch-deeds.mjs writes (path.relative(rawDir, deedPath(eik))), and upsertDeed sits +// past its JSON.parse/assertUicEcho try-catch, so a throw here aborts the whole crawl on the first +// branch office that returns a deed. The guard's own stated soundness ("an ЕИК is 9 or 13 digits, +// never 10") only holds if the run is matched as a WHOLE, which is why the pattern is anchored. +test('a 13-digit ЕИК does not trip the ЕГН guard through its own derived rawPath', () => + withCache((db) => { + assert.doesNotThrow(() => + upsertDeed(db, deed({ eik: '1155361790001', rawPath: '1155361790001.json' })), + ); + assert.equal(readDeed(db, '1155361790001').eik, '1155361790001'); + })); + +// The rail must not be a hand-maintained allowlist of four field names: upsertDeed binds thirteen +// values, and the next one added would bypass the check silently. Every bound value is screened. +test('the ЕГН guard screens EVERY bound value, not a hand-picked subset', () => + withCache((db) => { + for (const field of ['seatEntryDate', 'latestOwnEntryDate', 'fetchedAt']) + assert.throws(() => upsertDeed(db, deed({ [field]: '8001014567' })), /ten-digit|ЕГН/i, field); + })); + +// ...with one exemption, and it is measured rather than assumed: a sha256 hex digest is 64 chars of +// [0-9a-f], so a standalone ten-digit run occurs in ~7% of hashes (18% unanchored). Screening it +// would refuse roughly one deed in fourteen for no privacy gain — a hash is not an ЕГН, and it is a +// hash precisely so that no deed content reaches the index. +test('the body hash is exempt from the ЕГН guard — a digit run there is arithmetic, not an ЕГН', () => + withCache((db) => { + assert.doesNotThrow(() => upsertDeed(db, deed({ bodySha256: `8001014567${'a'.repeat(54)}` }))); + })); + test('the schema exposes no column that could hold a person name', () => withCache((db) => { const cols = db From 7f2e174e1a731cc83ce7c1f274f833c16b584db0 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:30:44 +0300 Subject: [PATCH 15/69] feat(tr): enforce the 35-day deed retention with a purge step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0033 decision 5 promises the deed cache is kept "with a 35-day retention (one refresh cycle plus slack) and a purge step in the same job". Neither existed. --max-age-days defaulted to null and only made a cached row pending again — it re-REQUESTS, it never deletes — so nothing ever unlinked the raw JSON that holds co-owner and manager names and company addresses. Third-party personal data accumulated on disk unbounded, with a documented TTL that deleted nothing. Freshness and retention are now separate flags because they are separate obligations: --max-age-days decides what gets re-requested, --retention-days decides what stops being kept. Retention defaults to the ADR's 35 days rather than to "off", so the rail holds for an operator who passes neither. purgeExpired runs in `finally`, and that placement is the point: retention is an obligation about other people's data, not a reward for a clean run, so it must also happen on the paths that leave early — a 429 (exit 2), a tripped breaker, an unresolved candidate. A failed purge is loud but never masks the run's own exit code, least of all the 429 that tells the operator to back off. It is a privacy rail, not cache eviction. Under normal operation it removes nothing, because the monthly refresh rewrites each row well inside the window; what it actually catches is residue — a company that dropped out of the candidate set, or a refresh that failed — plus orphaned raw files that no read path can reach and that are therefore pure retained personal data. purgeExpired was written before its tests, so RED was established afterwards by mutation: stubbing the expiry query kills 4 tests, disabling the orphan sweep kills 2, and moving the call out of `finally` kills the 429 case. --- .../workflows/related-persons-tr-refresh.yml | 5 ++ scripts/tr/cache.mjs | 72 +++++++++++++++++- scripts/tr/cache.test.mjs | 73 ++++++++++++++++++- scripts/tr/fetch-deeds.mjs | 35 ++++++++- scripts/tr/fetch-deeds.test.mjs | 54 ++++++++++++++ 5 files changed, 234 insertions(+), 5 deletions(-) diff --git a/.github/workflows/related-persons-tr-refresh.yml b/.github/workflows/related-persons-tr-refresh.yml index 807c103c..9cc09add 100644 --- a/.github/workflows/related-persons-tr-refresh.yml +++ b/.github/workflows/related-persons-tr-refresh.yml @@ -10,6 +10,11 @@ name: Related-persons — Trade Register cache refresh (monthly) # It SHIPS NOTHING. It writes only the cache under scratch/tr/, which is git-ignored and never leaves # the runner except as an artifact. A run that fails leaves the previous cache intact. # +# It also PURGES: raw deeds past the 35-day retention (ADR-0033 decision 5) are deleted at the end of +# every run, including one a 429 cut short. Retention is not the same knob as freshness — --max-age-days +# decides what gets re-requested, --retention-days decides what stops being kept — and the default is +# the ADR's 35 days, so the rail holds even when the workflow passes neither. +# # Pace and refusals live in scripts/tr/fetch-deeds.mjs and are not configurable here on purpose: # 1 request / 3 s, sequential, a closed candidate set, and a 429 ends the run (exit 2) without marking # anything. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the limiter is the diff --git a/scripts/tr/cache.mjs b/scripts/tr/cache.mjs index 846c4eb9..20668718 100644 --- a/scripts/tr/cache.mjs +++ b/scripts/tr/cache.mjs @@ -153,10 +153,16 @@ export function readDeed(db, eik) { }; } +/** ADR-0033 decision 5: one monthly refresh cycle plus slack. See purgeExpired. */ +export const RETENTION_DAYS = 35; + /** * Which of `wanted` still need a request — the resumability primitive. - * A cached row past `maxAgeDays` becomes pending again (the 35-day retention of ADR-0033 decision 5); - * pass no `maxAgeDays` to treat any cached row as current. + * + * `maxAgeDays` is a FRESHNESS knob, not the retention rail: past it a cached row becomes pending and + * is re-requested. Retention — actually deleting the personal data — is purgeExpired below. Refreshing + * a deed rewrites it; only purging removes it, and conflating the two is how a documented TTL ends up + * never deleting anything. Pass no `maxAgeDays` to treat any cached row as current. */ export function pendingEiks(db, wanted, { maxAgeDays = null, now = new Date() } = {}) { const cutoff = maxAgeDays == null ? null : now.getTime() - maxAgeDays * 86_400_000; @@ -198,3 +204,65 @@ export function coverage(db, wanted) { ratio: wanted.length === 0 ? 1 : covered / wanted.length, }; } + +/** + * Delete deed data past its retention — ADR-0033 decision 5's purge step. + * + * This is a PRIVACY rail, not a cache-eviction policy, and the distinction decides the design. The raw + * JSON under scratch/tr/deeds/ is the only place third-party names live: co-owners and managers who + * hold no public office, and the company's street address. We keep it because re-deriving a boolean is + * cheaper than re-requesting somebody else's register — not because we are entitled to hold it. + * + * Retention is 35 days: one monthly refresh cycle plus slack. Under normal operation nothing is ever + * purged, because the refresh at ~30 days rewrites the row first. What this actually catches is the + * residue — a company that dropped out of the candidate set, or a refresh that failed — which is + * exactly the data with no remaining reason to exist. + * + * Orphaned files are removed too: a raw deed with no index row is unreachable by every read path here, + * so it is pure retained personal data. + * + * @returns {{rows:number, files:number, orphans:number}} what was removed + */ +export function purgeExpired( + db, + rawDir, + { retentionDays = RETENTION_DAYS, now = new Date() } = {}, +) { + const cutoff = new Date(now.getTime() - retentionDays * 86_400_000).toISOString(); + const expired = db.prepare('SELECT eik, raw_path FROM deeds WHERE fetched_at < ?').all(cutoff); + + let files = 0; + for (const row of expired) { + if (!row.raw_path) continue; + // Resolve through the same safeEik path-traversal rail the writer used, never the stored string. + try { + fs.unlinkSync(path.join(rawDir, `${safeEik(row.eik)}.json`)); + files++; + } catch (e) { + // Already gone is the goal state, not a failure. Anything else must surface — a purge that + // silently fails to delete is worse than no purge, because it reports success. + if (e.code !== 'ENOENT') throw e; + } + } + db.prepare('DELETE FROM deeds WHERE fetched_at < ?').run(cutoff); + + let orphans = 0; + let names = []; + try { + names = fs.readdirSync(rawDir); + } catch (e) { + if (e.code !== 'ENOENT') throw e; // no raw dir yet — nothing to orphan + } + const known = new Set( + db + .prepare('SELECT eik FROM deeds') + .all() + .map((r) => `${r.eik}.json`), + ); + for (const name of names) { + if (!name.endsWith('.json') || known.has(name)) continue; + fs.unlinkSync(path.join(rawDir, name)); + orphans++; + } + return { rows: expired.length, files, orphans }; +} diff --git a/scripts/tr/cache.test.mjs b/scripts/tr/cache.test.mjs index 4b675e87..8046c1ad 100644 --- a/scripts/tr/cache.test.mjs +++ b/scripts/tr/cache.test.mjs @@ -9,7 +9,16 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { openCache, upsertDeed, markOutsideTr, pendingEiks, readDeed, coverage } from './cache.mjs'; +import { + openCache, + upsertDeed, + markOutsideTr, + pendingEiks, + readDeed, + coverage, + purgeExpired, + RETENTION_DAYS, +} from './cache.mjs'; function tmpDb() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-cache-')); @@ -158,3 +167,65 @@ test('markOutsideTr is permanent-by-intent and records WHY', () => assert.equal(row.status, 'outside_tr'); assert.match(row.outsideReason, /ДЗЗД/); })); + +// ── retention (ADR-0033 decision 5) ─────────────────────────────────────────── +// The ADR promises „a 35-day retention and a purge step in the same job". Freshness (pendingEiks' +// maxAgeDays) only makes a row pending again — it re-REQUESTS, it never deletes. Only this purge +// removes the third-party names in the raw deed, so these tests are the difference between a stated +// TTL and an enforced one. +function withRaw(fn) { + const { dir, file } = tmpDb(); + const rawDir = path.join(dir, 'deeds'); + fs.mkdirSync(rawDir, { recursive: true }); + const db = openCache(file); + try { + return fn(db, rawDir); + } finally { + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } +} +const writeRaw = (rawDir, eik) => + fs.writeFileSync(path.join(rawDir, `${eik}.json`), '{"names":"third-party PII"}'); + +test('purgeExpired deletes the raw deed AND its row past the retention window', () => + withRaw((db, rawDir) => { + const now = new Date('2026-08-05T00:00:00Z'); + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-05-01T00:00:00Z' })); // ~96 days old + upsertDeed(db, deed({ eik: '201122335', fetchedAt: '2026-08-01T00:00:00Z' })); // 4 days old + writeRaw(rawDir, '115536179'); + writeRaw(rawDir, '201122335'); + + const res = purgeExpired(db, rawDir, { retentionDays: 35, now }); + assert.equal(res.rows, 1); + assert.equal(res.files, 1); + assert.equal(fs.existsSync(path.join(rawDir, '115536179.json')), false, 'PII must be gone'); + assert.equal(readDeed(db, '115536179'), null); + // The in-window deed is untouched — a purge that also evicted live cache would force a re-crawl, + // which is the one thing the pacing exists to avoid. + assert.equal(fs.existsSync(path.join(rawDir, '201122335.json')), true); + assert.ok(readDeed(db, '201122335')); + })); + +test('purgeExpired removes orphaned raw deeds — unreachable data is pure retained PII', () => + withRaw((db, rawDir) => { + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-08-01T00:00:00Z' })); + writeRaw(rawDir, '115536179'); + writeRaw(rawDir, '204556676'); // no index row: no read path can ever reach it + const res = purgeExpired(db, rawDir, { now: new Date('2026-08-05T00:00:00Z') }); + assert.equal(res.orphans, 1); + assert.equal(fs.existsSync(path.join(rawDir, '204556676.json')), false); + assert.equal(fs.existsSync(path.join(rawDir, '115536179.json')), true); + })); + +test('purgeExpired defaults to the 35-day window and tolerates an already-missing file', () => + withRaw((db, rawDir) => { + assert.equal(RETENTION_DAYS, 35); + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-06-25T00:00:00Z' })); // 41 days + upsertDeed(db, deed({ eik: '201122335', fetchedAt: '2026-07-15T00:00:00Z' })); // 21 days + // No raw file on disk for the expired row: already gone is the goal state, not an error. + const res = purgeExpired(db, rawDir, { now: new Date('2026-08-05T00:00:00Z') }); + assert.equal(res.rows, 1); + assert.equal(res.files, 0); + assert.ok(readDeed(db, '201122335'), 'the 21-day-old deed is inside the window'); + })); diff --git a/scripts/tr/fetch-deeds.mjs b/scripts/tr/fetch-deeds.mjs index 0e14915d..78c63192 100644 --- a/scripts/tr/fetch-deeds.mjs +++ b/scripts/tr/fetch-deeds.mjs @@ -24,7 +24,14 @@ import crypto from 'node:crypto'; import { assertTrScratchIgnored, TR_DB, TR_RAW, safeEik, deedPath } from './paths.mjs'; import { eikChecksumValid } from './eik.mjs'; import { deedUrl, politeTrGet, RateLimitError, httpsGet } from './client.mjs'; -import { openCache, upsertDeed, markOutsideTr, pendingEiks } from './cache.mjs'; +import { + openCache, + upsertDeed, + markOutsideTr, + pendingEiks, + purgeExpired, + RETENTION_DAYS, +} from './cache.mjs'; import { assertUicEcho, registryLegalForm, @@ -74,11 +81,17 @@ export function parseTrOptions(argv) { ); } const maxAgeRaw = get('max-age-days', ''); + // --max-age-days is FRESHNESS (re-request past it); --retention-days is RETENTION (delete past it). + // They are separate flags because they are separate obligations: refreshing rewrites the personal + // data, only purging removes it. Retention defaults to the ADR's 35 days rather than to „off", so + // the rail holds for an operator who passes neither. + const retentionRaw = get('retention-days', ''); return { eiksFile, limit: limitRaw ? posInt(limitRaw, 'limit') : Infinity, minIntervalMs, maxAgeDays: maxAgeRaw ? posInt(maxAgeRaw, 'max-age-days') : null, + retentionDays: retentionRaw ? posInt(retentionRaw, 'retention-days') : RETENTION_DAYS, }; } @@ -117,7 +130,7 @@ export async function run({ argv = process.argv, } = {}) { guard(); - const { eiksFile, limit, minIntervalMs, maxAgeDays } = parseTrOptions(argv); + const { eiksFile, limit, minIntervalMs, maxAgeDays, retentionDays } = parseTrOptions(argv); const requested = readEiksFile(eiksFile); // A shape- or checksum-invalid code is dropped BEFORE any request: it cannot name a real company, @@ -241,6 +254,24 @@ export async function run({ } return 0; } finally { + // The purge step ADR-0033 decision 5 puts „in the same job" — in `finally`, and that placement is + // the point. Retention is an obligation about other people's data, not a reward for a clean run, + // so it must also happen on the paths that leave early: a 429 (exit 2), a tripped breaker, an + // unresolved candidate. Under normal operation it removes nothing, because the monthly refresh + // rewrites each row well inside the window; anything it does delete is residue — a company that + // left the candidate set, or a refresh that never landed. + try { + const purged = purgeExpired(db, rawDir, { retentionDays, now: now() }); + if (purged.rows || purged.files || purged.orphans) { + console.log( + `purged ${purged.rows} row(s), ${purged.files} raw deed(s), ${purged.orphans} orphan(s) past ${retentionDays}d retention`, + ); + } + } catch (e) { + // A failed purge must be loud but must not mask the run's own outcome — especially not a 429, + // whose exit code is what tells the operator to back off. + console.error(`purge failed: ${e.message}`); + } db.close(); } } diff --git a/scripts/tr/fetch-deeds.test.mjs b/scripts/tr/fetch-deeds.test.mjs index 8b8296d7..3a4b1dc0 100644 --- a/scripts/tr/fetch-deeds.test.mjs +++ b/scripts/tr/fetch-deeds.test.mjs @@ -371,3 +371,57 @@ test('the circuit breaker aborts a sustained wall of failures', async () => { c.cleanup(); } }); + +// ── retention (ADR-0033 decision 5: „a purge step in the same job") ─────────── +test('the job purges past-retention deeds, and does so even when a 429 ends the run', async () => { + // Retention is an obligation about other people's data, not a reward for a clean run: the paths that + // leave early — a 429, a tripped breaker — are exactly the ones where a naive placement after the + // fetch loop would skip it and let third-party names sit on disk indefinitely. + for (const [label, route] of [ + ['clean run', ok(A)], + ['429 stops the run', status(429)], + ]) { + const c = ctx(); + try { + // An old deed with its raw file — well past the 35-day window at the harness's fixed clock. + fs.mkdirSync(c.rawDir, { recursive: true }); + fs.writeFileSync(path.join(c.rawDir, `${C}.json`), '{"owner":"ТРЕТО ЛИЦЕ"}'); + const db = new DatabaseSync(c.dbFile); + db.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT); + INSERT INTO deeds (eik,status,fetched_at,raw_path) VALUES ('${C}','fetched','2026-01-01T00:00:00Z','${C}.json');`); + db.close(); + + await harness(c, [A], { [A]: route }).promise; + + assert.equal( + fs.existsSync(path.join(c.rawDir, `${C}.json`)), + false, + `${label}: the past-retention raw deed must be deleted`, + ); + assert.equal( + rows(c.dbFile).some((r) => r.eik === C), + false, + `${label}: its index row must go with it`, + ); + } finally { + c.cleanup(); + } + } +}); + +test('the purge leaves in-window deeds alone — it is a privacy rail, not a cache eviction', async () => { + // If it evicted live cache, every run would re-request deeds it already holds, which is precisely + // the volume against the register the pacing exists to avoid. + const c = ctx(); + try { + await harness(c, [A], { [A]: ok(A) }).promise; + assert.equal(rows(c.dbFile).length, 1, 'the deed just fetched must survive its own job'); + assert.equal(fs.existsSync(path.join(c.rawDir, `${A}.json`)), true); + } finally { + c.cleanup(); + } +}); From 2327e6265cf5ea909ba7c8b028210c5aa3c926e3 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:31:03 +0300 Subject: [PATCH 16/69] feat(related-persons): implement the monotonicity gate ADR-0033 specifies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rules_version was only ever written (load.mjs) and never read for a comparison. Every "§8's monotonicity gate keys on this" comment — in load.mjs, evidence.mjs and migration 0006 — pointed at code that did not exist: no pre-wipe export, no snapshot compare, no finding. ship-related-persons.mjs's count floor cannot stand in for it. It compares a COUNT, so a one-for-one swap — one true published link silently dropped, one gained — leaves it perfectly quiet. Two halves: load.mjs exports the currently published set with each link's rules_version immediately before it drops the CACBG tables. The rebuild is total, so that set exists only in this instant. On a first run the tables are absent and the export is an empty array — written rather than skipped, so a missing file keeps meaning "the loader never ran", not "nothing was published". Any error other than "no such table" propagates; swallowing it would turn a broken export into a permanently silent gate. audit.mjs compares against it. A link published last run and not published now under an UNCHANGED rules_version is a hard finding: nothing licensed the removal. Under a changed version it is an intentional event and degrades to a printed diff, so a rules bump still shows a human which named claims it withdrew. Held and withdrawn links are excluded from the export — they were never a public claim, so their absence is not a regression. The failure direction this protects against is recall, not libel: a missing gate risks withholding a true link, never fabricating a false one. But the surface it guards is about to triple, so a rules-neutral change must not be able to shrink it silently. Also drops the dead nameDistinctiveness import from load.mjs (referenced only in a comment since the ladder replaced it) and replaces it with a note on why it stays in classify.mjs. --- scripts/cacbg/audit.mjs | 55 +++++++++++++++++++++++++++ scripts/cacbg/audit.test.mjs | 73 +++++++++++++++++++++++++++++++++++- scripts/cacbg/load.mjs | 43 ++++++++++++++++++--- scripts/cacbg/load.test.mjs | 63 +++++++++++++++++++++++++++++++ 4 files changed, 226 insertions(+), 8 deletions(-) diff --git a/scripts/cacbg/audit.mjs b/scripts/cacbg/audit.mjs index a072a6d0..2a20db6a 100644 --- a/scripts/cacbg/audit.mjs +++ b/scripts/cacbg/audit.mjs @@ -7,10 +7,14 @@ import { DatabaseSync } from 'node:sqlite'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import fs from 'node:fs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; +import { RULES_VERSION } from '../tr/evidence.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); +const STAGING = process.env.CACBG_STAGING || path.join(ROOT, 'scratch/cacbg/staging'); +const SNAPSHOT = path.join(STAGING, 'published-snapshot.json'); const { companyNameKey } = await import('../../packages/shared/src/company-name-key.ts'); const db = new DatabaseSync(DB, { readOnly: true }); @@ -160,6 +164,40 @@ for (const l of nonExact) { db.close(); +// D. Monotonicity — ADR-0033 decision 6, the correction of #279 §8. +// +// §8 asked for a seal kept „forever" and strictly-additive recomputation. That is unachievable: labels +// flip, the deed cache expires, and a court can annul an entry (чл. 29 ЗТРРЮЛНЦ) with no rules change +// at all. So the seal is NOT a store — it is re-derived every run — and monotonicity is enforced HERE, +// as a gate, against the export load.mjs writes immediately before it drops the CACBG tables. +// +// The rule: a link that was published last run and is not published now is a REGRESSION unless the +// rules themselves changed. Under an unchanged rules_version nothing licensed the removal, so it is a +// hard finding. Under a changed one it is an intentional event and degrades to a printed diff. +// +// ship-related-persons.mjs's count floor cannot do this job: it compares a COUNT, so a one-for-one +// swap — one true link silently dropped, one gained — leaves it perfectly quiet. +const publishedNow = new Set(published.map((l) => l.link_key)); +let priorPublished = null; +try { + priorPublished = JSON.parse(fs.readFileSync(SNAPSHOT, 'utf8')); +} catch (e) { + // ENOENT is the legitimate first run — there is no prior surface to regress from. Anything else + // (unreadable, malformed) must not be swallowed into a silent pass of the gate. + if (e.code !== 'ENOENT') throw e; +} +const vanished = (priorPublished ?? []).filter((p) => !publishedNow.has(p.link_key)); +const regressions = vanished.filter((p) => p.rules_version === RULES_VERSION); +for (const p of regressions) + findings.push({ + axis: 'D_monotonicity', + link_key: p.link_key, + eik: p.link_key.split('|')[1] ?? '', + // The link_key is named explicitly: it is the only handle a human has to go and look at which + // claim disappeared, and the shared axis report prints the ЕИК alone. + detail: `${p.link_key} published last run under rules_version ${p.rules_version} (unchanged) and is not published now — nothing licensed this removal`, + }); + // Report const byAxis = {}; for (const f of findings) (byAxis[f.axis] ??= []).push(f); @@ -171,6 +209,23 @@ for (const [axis, fs] of Object.entries(byAxis)) { for (const f of fs.slice(0, 20)) console.log(` - [${f.eik}] ${f.detail}`); console.log(''); } +if (priorPublished === null) { + console.log(`## monotonicity — no prior export at ${SNAPSHOT}; treating this as a first run\n`); +} else { + const declared = vanished.filter((p) => p.rules_version !== RULES_VERSION); + console.log( + `## monotonicity — ${priorPublished.length} published last run, ${publishedNow.size} now; ` + + `${regressions.length} regression(s), ${declared.length} declared removal(s)\n`, + ); + // Declared removals are not findings, but they are never silent: a rules bump is exactly when a + // human should read which named claims it withdrew. + for (const p of declared) + console.log( + ` - ${p.link_key} removed under a rules change (${p.rules_version} → ${RULES_VERSION})`, + ); + if (declared.length) console.log(''); +} + console.log(`## non-exact provenance (${provenance.length}) — verify each cross-check by eye`); for (const p of provenance) { console.log(` - ${p.method} [${p.eik}] winner="${p.winner}"`); diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index 46d42121..38f4c346 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -26,10 +26,15 @@ const dirs = []; // Build a fixture DB (bidders + declarations + declared_interests + interest_links), run audit.mjs against // it as a subprocess, and return { threw, out } — threw=true iff the audit exited non-zero (a hard finding). -function buildAndAudit({ bidders, decls = [], dis = [], links, seals = [] }) { +function buildAndAudit({ bidders, decls = [], dis = [], links, seals = [], snapshot = null }) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-audit-')); dirs.push(dir); const DB = path.join(dir, 'fixture.sqlite'); + const staging = path.join(dir, 'staging'); + fs.mkdirSync(staging, { recursive: true }); + // The pre-wipe export load.mjs writes before it drops the CACBG tables. Absent === a first run. + if (snapshot) + fs.writeFileSync(path.join(staging, 'published-snapshot.json'), JSON.stringify(snapshot)); const db = new DatabaseSync(DB); db.exec(` CREATE TABLE bidders(id TEXT PRIMARY KEY, name TEXT, eik_normalized TEXT, eik_valid INT); @@ -57,7 +62,12 @@ function buildAndAudit({ bidders, decls = [], dis = [], links, seals = [] }) { out = execFileSync( 'node', ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'audit.mjs')], - { cwd: ROOT, env: { ...process.env, CACBG_DB: DB }, encoding: 'utf8', stdio: 'pipe' }, + { + cwd: ROOT, + env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: staging }, + encoding: 'utf8', + stdio: 'pipe', + }, ); } catch (e) { threw = true; // execFileSync throws on non-zero exit (a hard finding fired) @@ -221,3 +231,62 @@ test('a well-formed seal passes every C axis (positive control)', () => { }); assert.equal(threw, false, 'a correctly sealed link must not fail the gate'); }); + +// ── D. Monotonicity (ADR-0033 decision 6) ──────────────────────────────────────────────────────── +// #279 §8 asked for a seal kept „forever" and strictly-additive recomputation. ADR-0033 showed that is +// false (labels flip, the cache expires, a court can annul an entry) and replaced the STORE with a +// GATE: seals are re-derived every run, and the audit compares the current published set against the +// pre-wipe export. A link that vanishes under an UNCHANGED rules_version is a hard finding — nothing +// about the rules changed, so its disappearance is a regression, not a decision. +// +// Why this needs its own gate and ship's count floor cannot serve: assertShipFloor checks a COUNT. +// A one-for-one swap — one link lost, one gained — leaves the count identical and the floor silent +// while a true published link is dropped. Only a per-key comparison sees it. +const SEALED = [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'`, +]; +const ONE_LINK = { + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`, `'b2','ВТОРА ФИРМА ЕООД','200000002',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: SEALED, +}; + +test('D — a previously published link that vanished under an UNCHANGED rules_version is a hard finding', () => { + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, // published last run, gone now + ], + }); + assert.equal(threw, true, 'a silent recall regression must fail the build'); + assert.match(out, /D_monotonicity/); + assert.match(out, /p9\|200000002/); +}); + +test('D — the same disappearance under a CHANGED rules_version is a printed diff, not a finding', () => { + // Removal stays an intentional event: bumping the rules version is how you declare one. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [{ link_key: 'p9|200000002', rules_version: 'tr-rules-0' }], + }); + assert.equal(threw, false, 'a declared rules change must not fail the build'); + assert.match(out, /p9\|200000002/, 'but it must still be reported'); +}); + +test('D — no snapshot (a first run) neither fires nor crashes', () => { + const { threw } = buildAndAudit(ONE_LINK); + assert.equal(threw, false); +}); + +test('D positive control — an unchanged published set produces no monotonicity finding', () => { + // Without this, a gate that never fires would pass both negatives above. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [{ link_key: 'p1|100000001', rules_version: 'tr-rules-1' }], + }); + assert.equal(threw, false); + assert.doesNotMatch(out, /D_monotonicity/); +}); diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 5a37ae91..3fe9d294 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -14,12 +14,9 @@ import { DatabaseSync } from 'node:sqlite'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { - nameDistinctiveness, - temporalStatus, - localityToken, - closelyHeldForm, -} from './classify.mjs'; +// nameDistinctiveness is deliberately NOT imported: the evidence ladder replaced it in the publish +// path (ADR-0033). It survives in classify.mjs for the review queue, not for a publishing decision. +import { temporalStatus, localityToken, closelyHeldForm } from './classify.mjs'; import { openCache, readDeed, coverage } from '../tr/cache.mjs'; import { TR_DB, TR_RAW } from '../tr/paths.mjs'; import { evidenceVerdict, reconcileTermination, RULES_VERSION } from '../tr/evidence.mjs'; @@ -84,6 +81,40 @@ const suppressedFp = new Set(suppEntries.map((e) => e.fp)); // B3 unused-suppression gate: every listed fingerprint MUST match exactly one built link. Track which get // used; a fingerprint that matched nothing (a stale/mis-keyed takedown) fails the build after the load loop. const usedSuppressions = new Set(); +// Export the CURRENT published surface before anything is dropped — ADR-0033 decision 6. +// +// The rebuild below is total, so the previous run's published set exists only in this instant. The +// audit compares against this file and hard-fails on a link that was published last run and is not +// published now under an UNCHANGED rules_version: nothing licensed that removal, so it is a silent +// recall regression. rules_version travels per key, because a link that vanished under a rules BUMP is +// an intentional removal and must degrade to a printed diff instead. +// +// Held and withdrawn links are deliberately excluded: they were never a public claim, so their absence +// next run is not a regression. On a first run the tables do not exist yet and the export is an empty +// set — written, not skipped, so a missing file means „the loader never ran", not „nothing published". +const priorPublished = (() => { + try { + return db + .prepare( + `SELECT il.link_key AS link_key, e.rules_version AS rules_version + FROM interest_links il + LEFT JOIN interest_link_evidence e ON e.link_key = il.link_key + WHERE il.status = 'published'`, + ) + .all() + .map((r) => ({ link_key: r.link_key, rules_version: r.rules_version ?? RULES_VERSION })); + } catch (e) { + // A first run: interest_links does not exist yet. Any OTHER failure must surface — swallowing it + // would turn a broken export into a permanently silent gate. + if (!/no such table/i.test(e.message)) throw e; + return []; + } +})(); +fs.writeFileSync( + path.join(STAGING, 'published-snapshot.json'), + JSON.stringify(priorPublished, null, 2) + '\n', +); + // Full idempotent rebuild that also picks up schema changes: drop the CACBG tables (children first — // FK-safe) and re-apply the migration. Nothing to preserve — suppressions are external now. for (const t of [ diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index d90b44a4..bd5e5bc8 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -1157,3 +1157,66 @@ test('a family stake publishes ONLY when the official confirmed the company them assert.equal(seal.matched_fact, 'seat:РУСЕ'); db.close(); }); + +test('the published surface is exported BEFORE the wipe, so the audit can gate monotonicity', () => { + // ADR-0033 decision 6. The loader rebuilds the CACBG tables from scratch every run, so the previous + // published set exists only in the instant before the DROP. Without this export the audit has + // nothing to compare against and the monotonicity gate can never fire — which is precisely the state + // review found: rules_version was written and never read. + // A FIRST run, on its own database: interest_links does not exist yet, so the export must be an + // empty set that is WRITTEN rather than skipped — a missing file has to keep meaning „the loader + // never ran", not „nothing was published". + const fresh = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-load-first-')); + fs.mkdirSync(path.join(fresh, 'staging'), { recursive: true }); + for (const f of fs.readdirSync(STAGING)) + fs.copyFileSync(path.join(STAGING, f), path.join(fresh, 'staging', f)); + // The base DB (bidders, contracts) comes from the main pipeline and is a precondition of the load — + // a genuinely empty file is not a first run, it is a broken one. So: copy the base and drop the + // CACBG tables, which is exactly the state before the loader has ever run against it. + const firstDb = path.join(fresh, 'first.sqlite'); + fs.copyFileSync(DB, firstDb); + const fdb = new DatabaseSync(firstDb); + for (const t of [ + 'interest_link_evidence', + 'interest_link_authorities', + 'interest_links', + 'declared_interests', + 'related_persons_internal', + 'declarations', + 'persons', + ]) + fdb.exec(`DROP TABLE IF EXISTS ${t}`); + fdb.close(); + runLoad({ CACBG_DB: firstDb, CACBG_STAGING: path.join(fresh, 'staging') }); + assert.deepEqual( + JSON.parse(fs.readFileSync(path.join(fresh, 'staging', 'published-snapshot.json'), 'utf8')), + [], + ); + fs.rmSync(fresh, { recursive: true, force: true }); + + const SNAP = path.join(STAGING, 'published-snapshot.json'); + runLoad(); + const db = open(); + const published = db + .prepare("SELECT link_key FROM interest_links WHERE status='published'") + .all() + .map((r) => r.link_key); + db.close(); + assert.ok(published.length > 0, 'fixture must publish something for this test to mean anything'); + + runLoad(); + const snap = JSON.parse(fs.readFileSync(SNAP, 'utf8')); + assert.deepEqual( + snap.map((s) => s.link_key).sort(), + [...published].sort(), + 'the export must hold exactly what was published before the wipe', + ); + // rules_version travels WITH each key: the gate distinguishes „vanished under unchanged rules" + // (a regression) from „vanished under a rules bump" (an intentional removal), and it cannot do that + // from the current version alone. + for (const s of snap) + assert.ok(s.rules_version?.length > 0, 'each exported key carries its rules'); + // Held and withdrawn links must NOT be exported — they were never a public claim, so their absence + // next run is not a regression to gate on. + assert.equal(snap.length, published.length); +}); From 76ed3b2bf16514ff99c80a3be7a74941bdd57b0a Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:31:03 +0300 Subject: [PATCH 17/69] docs(adr): record ADR-0033's amendments on the ADRs it amends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Superseded ADRs gain a "Superseded by" back-pointer; amended ones gained nothing. A reader of ADR-0007 — the foundational certainty bar — had no way to discover that ADR-0033 carved an explicit exception to its decisions 2 and 3. Adds an "Amended by" line to 0007, 0010, 0021 and 0028, each naming which decisions moved. A header breadcrumb, so the convention that accepted ADRs are not rewritten still holds. --- docs/adr/0007-scope-and-certainty-bar.md | 1 + docs/adr/0010-pii-posture.md | 1 + docs/adr/0021-methodology-page-and-temporal-freshness.md | 1 + docs/adr/0028-declared-eik-is-a-determining-identifier.md | 1 + 4 files changed, 4 insertions(+) diff --git a/docs/adr/0007-scope-and-certainty-bar.md b/docs/adr/0007-scope-and-certainty-bar.md index 18fcb446..c41f4255 100644 --- a/docs/adr/0007-scope-and-certainty-bar.md +++ b/docs/adr/0007-scope-and-certainty-bar.md @@ -1,6 +1,7 @@ # ADR-0007: Scope and certainty bar - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — решения 2 и 3 (изричното изключение за „евристика, която твърди") - Date: 2026-07-05 - Deciders: lb (Head of AI, ИО), Claude - Related: spec §1–2, §8 diff --git a/docs/adr/0010-pii-posture.md b/docs/adr/0010-pii-posture.md index 8f09ead8..cf9898a0 100644 --- a/docs/adr/0010-pii-posture.md +++ b/docs/adr/0010-pii-posture.md @@ -1,6 +1,7 @@ # ADR-0010: PII posture - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — решения 3 и 6 (кешът от актове и срокът на съхранение) - Date: 2026-07-05 - Deciders: lb, Claude - Related: spec §8; КЗЛД guidance; GDPR Art. 6(1)(c)/(f), Art. 85 diff --git a/docs/adr/0021-methodology-page-and-temporal-freshness.md b/docs/adr/0021-methodology-page-and-temporal-freshness.md index 810a74b2..d2f0d9cd 100644 --- a/docs/adr/0021-methodology-page-and-temporal-freshness.md +++ b/docs/adr/0021-methodology-page-and-temporal-freshness.md @@ -1,6 +1,7 @@ # ADR-0021: Public methodology/corrections page (E10) + temporal freshness & divestment expiry (E11) - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — E11 (етикетите за настояще време се отлагат за фаза 2) - Date: 2026-07-06 - Deciders: lb, Claude - Related: [ADR-0007](0007-scope-and-certainty-bar.md), [ADR-0019](0019-private-interest-vs-ex-officio-classification.md), [ADR-0020](0020-conflict-explorer-surface-posture.md); spec §5/§8/§9; `scripts/cacbg/load.mjs`, `apps/web/app/routes/conflict.methodology.tsx`, `packages/db/src/queries/related-persons.ts` diff --git a/docs/adr/0028-declared-eik-is-a-determining-identifier.md b/docs/adr/0028-declared-eik-is-a-determining-identifier.md index 9f123482..9ad76597 100644 --- a/docs/adr/0028-declared-eik-is-a-determining-identifier.md +++ b/docs/adr/0028-declared-eik-is-a-determining-identifier.md @@ -1,6 +1,7 @@ # ADR-0028: Declared ЕИК is a determining identifier — tier `A_eik`, exempt from the TR census - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — декларираният ЕИК остава определящ, но вече иска и регистърно доказателство - Date: 2026-07-11 - Deciders: lb, Claude - Related: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md), [ADR-0015](0015-tr-name-uniqueness-census.md), [ADR-0016](0016-free-text-entity-resolution.md); `scripts/cacbg/load.mjs`, `scripts/cacbg/classify.mjs` From 935ce1c1a2475a7f8afe8a89a059f8d4876244e1 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:31:03 +0300 Subject: [PATCH 18/69] test(web): scope the missing-source assertion to the card that has none MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit expect(text()).toContain('—') ran against the whole page, where the value and date cells render em-dashes of their own. It would have survived the source branch being deleted outright. Scope it to the card whose sourceUrl is null and assert the declaration anchor is absent from it. --- apps/web/app/routes/conflicts.render.test.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/web/app/routes/conflicts.render.test.tsx b/apps/web/app/routes/conflicts.render.test.tsx index ce2a78db..6be5ea6e 100644 --- a/apps/web/app/routes/conflicts.render.test.tsx +++ b/apps/web/app/routes/conflicts.render.test.tsx @@ -194,8 +194,16 @@ describe('/conflicts route — render', () => { (a) => a.textContent === 'декларация', ); expect(sourceAnchor?.getAttribute('href')).toBe('https://register.cacbg.bg/2024/i.xml'); - // the zero-contract link has no toggle (contractCount === 0) and a muted „—" source - expect(text()).toContain('—'); + // Scoped to the card that actually has sourceUrl: null. A page-wide toContain('—') passes on any + // em-dash anywhere — including the ones the value and date cells render — so it would survive the + // source branch being deleted outright. + const cards = container.querySelectorAll('.conflict-card'); + const noSourceCard = [...cards].find((c) => c.textContent?.includes('ПРАЗЕН ООД'))!; + expect(noSourceCard).toBeTruthy(); + expect( + [...noSourceCard.querySelectorAll('a')].some((a) => a.textContent === 'декларация'), + ).toBe(false); + expect(noSourceCard.textContent).toContain('—'); }); it('a zero-contract link renders no „Виж договорите" toggle', async () => { From f4d73dcdb592bac882ab44be648440335c80125a Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:39:16 +0300 Subject: [PATCH 19/69] =?UTF-8?q?docs(adr):=20resolve=20the=20=C2=A75/?= =?UTF-8?q?=C2=A710=20control-number=20gap=20in=20ADR-0033?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It was parked as "resolve with the author". It does not need the author — the issue's own numbers determine it. §5 gives each rung a control count: 4 barred + 281 document + 102 seat + 3 ЕИК + 21 refuted + 156 unknown = 567, plus the 4 links whose ЕИК is not in the register at all (§5 scopes the ladder to links "с изтеглен акт", so those reach no rung) = 571. §10's identity row is 281/102/156/21/4/4 = 568. The difference is exactly the "3 по ЕИК". The labels give the cause away. The identity row calls its second bucket "седалище" — one leg of rung 3 — while the evidence row two lines below calls the same rung "потвърдено". Rung 3 has two legs; the row tallied one and presented the result as a partition of the resolved set. A histogram that dropped a category, not a disputed measurement. The stated total 568 equals that row's sum exactly, so it is derived from the histogram rather than measured independently. Both therefore move together: the bucket becomes "потвърдено: 105", the resolved total becomes 571. F8 measures against those and reports disagreement rather than adopting whichever reading makes the reconciliation pass. The identified-vs-surfaced gap moves with it — 57 links on the corrected reading, 54 on the row as written — so that figure independently distinguishes the two, which is worth more than the correction itself. Also records what is NOT a discrepancy, so nobody "fixes" it: "извън ТР" is 4 in §10 and 3 in §3/§11 because the units differ — 3 ЕИК not in the register, carrying 4 links between them. §10's row counts links. --- ...-evidence-replaces-name-distinctiveness.md | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md index b3315c99..e75c28c3 100644 --- a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -261,11 +261,28 @@ None of these are follow-ups. - **Precision is proven by a hand-labelled sample, not by the control totals.** A human verifies against the portal: every reconciled link, every refuted link, and random samples of „Документ" and „Потвърдено". **Pass mark: zero wrong-company and zero wrong-person errors.** The control numbers of #279 §10 are a - reproducibility check — and they are usable as one only after the §5/§10 discrepancy is resolved with the - author (§5's „102 по седалище, 3 по ЕИК" leaves those three ЕИК confirmations in no bucket of the §10 - row, which sums to exactly 568 without them), and re-measured on top of the §12 phantom-row fix, which - changed the corpus — that fix has landed as #281 (87 phantom rows across 15 sets; 256,286 announced − - 87 = 256,199, covered exactly by 255,582 fetched + 617 missing at source). + reproducibility check, and two things had to be settled before they could serve as one. + + **The §5/§10 discrepancy is a dropped histogram category, not a disputed measurement.** §5's rungs carry + their own control counts: 4 barred + 281 document + 102 seat + 3 ЕИК + 21 refuted + 156 unknown = 567, + plus the 4 links whose ЕИК is not in the register at all (§5 scopes the ladder to links „с изтеглен акт", + so those reach no rung) = **571**. §10's identity row is 281 / 102 / 156 / 21 / 4 / 4 = **568**. The + difference is exactly the „3 по ЕИК". + + The cause is visible in the labels. The identity row names its second bucket **„седалище"** — one leg of + rung 3 — while the evidence row two lines below names the same rung **„потвърдено"** (251 / 78). Rung 3 + has two legs; the row tallied one and presented the result as a partition of the resolved set. And the + stated total 568 equals that row's sum exactly, which makes it a figure derived from the histogram rather + than measured independently. So both move together: the bucket becomes **„потвърдено: 105"** and the + resolved total becomes **571**. F8 measures against those and reports if its own count disagrees — the + reconciliation is not permitted to adopt whichever reading makes it pass. + + **Not a discrepancy, and not to be „fixed":** „извън ТР" is 4 in §10 and 3 in §3/§11 because the units + differ — 3 ЕИК that are not in the register, carrying 4 links between them. §10's row counts links. + + The numbers must also be re-measured on top of the §12 phantom-row fix, which changed the corpus — that + fix has landed as #281 (87 phantom rows across 15 sets; 256,286 announced − 87 = 256,199, covered exactly + by 255,582 fetched + 617 missing at source). - **Every anti-false-zero control is a positive control.** „Sofia seat confirmations: 0" is indistinguishable from a broken normalizer without one; so is a joint-stock bar with no marginal effect over `closelyHeldForm`, and a matcher that always returns false (ADR-0027's lesson). @@ -304,7 +321,8 @@ None of these are follow-ups. as-of evidence via `?entryDate=`, and the §7 „и към днешна дата" labels with their post-period contracts. - The seat rung is structurally capped and we accept the cap: a declared seat is captured **only** from the ООД/ЕООД holdings table of an *asset* declaration (`parse.mjs`), so it can never rescue a link declared - solely in an interests declaration. This is probably consistent with the measured 102 — rung 1 bars the + solely in an interests declaration. This is probably consistent with the 102 measured for the seat leg + specifically (not the 105 of the whole rung, whose other three come from the ЕИК leg) — rung 1 bars the joint-stock cases anyway — but it is confirmed against the corpus before launch, not assumed. ### Measured outcome — recorded on completion @@ -313,8 +331,10 @@ This ADR is accepted on the design. Exactly one amendment is permitted afterward filled in with the measured result, and nothing else in the file is rewritten (repo convention — an accepted ADR is superseded, not edited). To be recorded: -- the realized ladder split against the #279 §10 control row, with each number naming the SQL that produced - it, and the 54-link gap between identified (281 + 102) and surfaced (329) attributed bucket by bucket; +- the realized ladder split against the #279 §10 control row **as corrected above** (281 / 105 / 156 / 21 / + 4 / 4 = 571), with each number naming the SQL that produced it, and the gap between identified (281 + 105) + and surfaced (329) attributed bucket by bucket — 57 links on the corrected reading, 54 on the row as + written, so the figure itself distinguishes the two; - the true candidate-ЕИК count (#279's „~400" is an assumption; the figure is `COUNT(DISTINCT eik)` over all links, not just published ones); - the hand-labelled sample result; From 2b06b7638e754c6cb331b2452b4a1c9f078f4ecc Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 13:50:13 +0300 Subject: [PATCH 20/69] fix(tr): survive a malformed numeric entity instead of killing the crawl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit String.fromCodePoint throws RangeError above U+10FFFF, and the argument comes straight off the wire: `�` in any deed field threw out of decodeEntities, out of entityBlocks and registrySeat, past the crawl loop's refuse-and-continue block — which covered only JSON.parse and assertUicEcho — and out of run(). One malformed escape ended a paced crawl that had already spent its request budget, and did the same to load.mjs at decision time. An out-of-range code point is not a character and cannot be part of a name, so it now decodes to nothing and the rest of the entity still parses. The in-range path is unchanged and tested, so the guard bounds the decoder without disabling it. Widen the two refuse-and-continue blocks to match what they already claimed: the crawl loop now wraps the HTML parsing as well as the JSON, and load.mjs holds the single link whose deed it cannot read rather than failing the whole run — one bad payload out of ~400 must not decide every other link's fate and then trip the ship floor. --- scripts/cacbg/load.mjs | 37 +++++++++++++++++++++---------- scripts/tr/deed.mjs | 15 +++++++++++-- scripts/tr/deed.test.mjs | 35 +++++++++++++++++++++++++++++ scripts/tr/fetch-deeds.mjs | 45 ++++++++++++++++++++++---------------- 4 files changed, 100 insertions(+), 32 deletions(-) diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 3fe9d294..7900a17e 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -614,19 +614,23 @@ for (const rec of agg.values()) { // person in THIS company). nameDistinctiveness is deliberately NOT part of this gate: the seat rung // exists precisely to rescue a generic name, so requiring distinctiveness would empty it. const nameUnique = nameGloballyUnique(rec.key); + // „Неизвестна" — the withholding verdict, used for every way of ending up with no usable evidence. + const noEvidence = () => ({ + kind: 'unknown', + publishable: false, + registryRole: null, + matchedFact: null, + entryNumber: null, + entryDate: null, + rulesVersion: RULES_VERSION, + }); // With --allow-partial-tr the operator has accepted an incomplete cache. An uncached ЕИК then yields // no evidence at all, which is „Неизвестна" — held. It must never be read as a reason to publish. - const verdict = missing - ? { - kind: 'unknown', - publishable: false, - registryRole: null, - matchedFact: null, - entryNumber: null, - entryDate: null, - rulesVersion: RULES_VERSION, - } - : evidenceVerdict({ + let verdict; + if (missing) verdict = noEvidence(); + else { + try { + verdict = evidenceVerdict({ deed, outsideTr, declarantName: rec.person, @@ -636,6 +640,17 @@ for (const rec of agg.values()) { scope: rec.scope, nameGloballyUnique: nameUnique, }); + } catch (err) { + // A deed we cannot parse is a deed we cannot reason about — so this link withholds, exactly as an + // uncached one does, and the run continues. Failing the whole load instead would let one malformed + // deed out of ~400 decide the fate of every other link, and the ship floor would then refuse the + // reduced surface — turning a single bad payload into a total outage. Loud, per link, fail-closed. + console.error( + ` ${rec.eik}: evidence UNREADABLE — ${err instanceof Error ? err.message : err} (link held)`, + ); + verdict = noEvidence(); + } + } const tier = verdict.kind; const contemporaneous = [...years].some( (cy) => temporalStatus(declYears, cy) === 'contemporaneous', diff --git a/scripts/tr/deed.mjs b/scripts/tr/deed.mjs index fdb3c1bd..71686ed8 100644 --- a/scripts/tr/deed.mjs +++ b/scripts/tr/deed.mjs @@ -29,12 +29,23 @@ const ENTITIES = { '>': '>', ' ': ' ', }; +// `String.fromCodePoint` THROWS RangeError above U+10FFFF (and on a surrogate half), and the input is +// whatever the register put on the wire. An unguarded throw here does not stay local: it escapes +// entityBlocks and registrySeat, past the crawl loop's refuse-and-continue block (which covers only +// JSON.parse + assertUicEcho) and out of run() — one malformed escape in one deed ends a paced crawl +// that has already spent its request budget, and does the same to load.mjs at decision time. +// Out of range is not a character and cannot be part of a name, so it decodes to nothing: the rest of +// the entity still parses, which is the difference between losing a glyph and losing the run. +const MAX_CODE_POINT = 0x10ffff; +const codePoint = (n) => + Number.isInteger(n) && n >= 0 && n <= MAX_CODE_POINT ? String.fromCodePoint(n) : ''; + /** Decode the entity set the register actually emits, plus numeric escapes. FIRST step, always. */ function decodeEntities(s) { return String(s) .replace(/&(?:quot|apos|amp|lt|gt|nbsp);/g, (m) => ENTITIES[m]) - .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n))) - .replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCodePoint(parseInt(n, 16))); + .replace(/&#(\d+);/g, (_, n) => codePoint(Number(n))) + .replace(/&#x([0-9a-f]+);/gi, (_, n) => codePoint(parseInt(n, 16))); } /** Strip tags and collapse whitespace, INSIDE one already-isolated entity. */ diff --git a/scripts/tr/deed.test.mjs b/scripts/tr/deed.test.mjs index 9a7f4c97..c45611c8 100644 --- a/scripts/tr/deed.test.mjs +++ b/scripts/tr/deed.test.mjs @@ -172,6 +172,41 @@ test('an empty htmlData yields no entities and does not throw', () => { assert.deepEqual(entityBlocks(null), []); }); +// A numeric entity is attacker-shaped input in the only sense that matters here: it comes off the +// wire, and `String.fromCodePoint` throws RangeError above U+10FFFF. That throw escapes decodeEntities, +// escapes entityBlocks and registrySeat, and — because the crawl loop's try/catch covers only +// JSON.parse + assertUicEcho — escapes run() and kills the process. One malformed entity in one deed +// would end a paced crawl that has already spent its request budget. Out of range is not a person, so +// the only defensible reading is „no character": drop it and keep parsing the rest of the entity. +test('an out-of-range numeric entity is dropped, never thrown out of the parser', () => { + const overflow = F19_THREE.replace('ПЕНКО', '�ПЕНКО'); + const blocks = entityBlocks(overflow); + assert.equal(blocks.length, 3, 'the deed still parses into its three entities'); + assert.match(blocks[1].text, /ПЕНКО НЕСТОРОВ НЕСТОРОВ/, 'the surrounding name survives intact'); + assert.ok(!blocks[1].text.includes('&#'), 'the escape itself does not survive as literal text'); +}); + +test('the hex numeric form is guarded too — both decode lines, not just the decimal one', () => { + const overflow = F19_THREE.replace('ИЛИЯН', '�ИЛИЯН'); + const blocks = entityBlocks(overflow); + assert.equal(blocks.length, 3); + assert.match(blocks[2].text, /ИЛИЯН КОСТАДИНОВ ФИЛИПОВ/); +}); + +test('an in-range numeric entity still decodes — the guard bounds, it does not disable', () => { + // П is „П". A guard that dropped every numeric escape would silently mangle real names. + const blocks = entityBlocks(F19_THREE.replace('ПЕНКО', 'ПЕНКО')); + assert.match(blocks[1].text, /ПЕНКО НЕСТОРОВ НЕСТОРОВ/); +}); + +test('registrySeat survives the same malformed entity rather than aborting the load', () => { + // registrySeat sits OUTSIDE the crawl loop's refuse-and-continue block (fetch-deeds.mjs), and + // load.mjs calls it again at decision time — so an unguarded throw here takes down both legs. + const d = deedOf([field('CR_F_5_L', F5_SEAT.replace('с. Марково', '�с. Марково'))]); + const seat = registrySeat(d); + assert.equal(seat.settlement, 'МАРКОВО'); +}); + test('liveFields keeps only the requested codes and reports entry date/number', () => { const d = deedOf([ field('CR_F_7_L', F7_MANAGER, { nameCode: 'CR_F_7_L', fieldEntryDate: '2017-09-15T00:00:00' }), diff --git a/scripts/tr/fetch-deeds.mjs b/scripts/tr/fetch-deeds.mjs index 78c63192..cdea4f73 100644 --- a/scripts/tr/fetch-deeds.mjs +++ b/scripts/tr/fetch-deeds.mjs @@ -216,35 +216,42 @@ export async function run({ continue; } - let deed; + // ONE refuse-and-continue block around EVERYTHING derived from the response — the JSON, the UIC + // echo, and the HTML parsing alike. The parsing used to sit outside it, which made the block's + // own promise false: a throw from deed.mjs escaped the loop, escaped run(), and killed the + // process, so one malformed deed ended a crawl that had already spent its paced request budget. + // The decode guard in deed.mjs removes the one throw we know of; this is the rail that holds when + // the next one appears, and the cost of being wrong here is measured in hours of pacing. try { - deed = JSON.parse(res.body.toString('utf8')); + const deed = JSON.parse(res.body.toString('utf8')); // The deed we got back must be the deed we asked for, or every claim derived from it names // the wrong company (R8). assertUicEcho(deed, eik); + + // The raw response is the ONLY place names live; it stays under git-ignored scratch. Written + // only after the echo check, so a deed for the wrong company never lands on disk. + atomicWrite(deedPath(eik, rawDir), res.body); + const seat = registrySeat(deed); + const form = registryLegalForm(deed); + upsertDeed(db, { + eik, + status: 'fetched', + httpStatus: 200, + fetchedAt: now().toISOString(), + rawPath: path.relative(rawDir, deedPath(eik, rawDir)), + bodySha256: crypto.createHash('sha256').update(res.body).digest('hex'), + legalFormCode: form.code, + legalFormVerdict: form.verdict, + seatNormalized: seat.settlement || null, + seatEntryDate: seat.entryDate, + latestOwnEntryDate: latestOwnershipEntryDate(deed), + }); } catch (err) { console.error(` ${eik}: REFUSED — ${err instanceof Error ? err.message : err}`); unresolved++; consecutive++; continue; } - - // The raw response is the ONLY place names live; it stays under git-ignored scratch. - atomicWrite(deedPath(eik, rawDir), res.body); - const seat = registrySeat(deed); - upsertDeed(db, { - eik, - status: 'fetched', - httpStatus: 200, - fetchedAt: now().toISOString(), - rawPath: path.relative(rawDir, deedPath(eik, rawDir)), - bodySha256: crypto.createHash('sha256').update(res.body).digest('hex'), - legalFormCode: registryLegalForm(deed).code, - legalFormVerdict: registryLegalForm(deed).verdict, - seatNormalized: seat.settlement || null, - seatEntryDate: seat.entryDate, - latestOwnEntryDate: latestOwnershipEntryDate(deed), - }); consecutive = 0; } From 95d8c513444e499c570bd8793931b89961de8be5 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 13:52:10 +0300 Subject: [PATCH 21/69] test(db): put the evidence seal gate under actual test pressure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deleting the whole `EXISTS (… interest_link_evidence …)` clause out of SURFACED_OWNERSHIP left the db suite green at 361/361. Every fixture sealed every link 'document', so the one piece of SQL standing between a withheld link and a named public claim — that a specific official owns a specific company — had no test on it at all. Seed each real withholding rung evidence.mjs can emit (refuted, bar_joint_stock, unknown, outside_tr) plus a link with no seal row, all on the same winner and otherwise identical, and assert absence across all four public queries: the leaderboard, the official page, the company page and the contract drill-down. The 'confirmed' positive control is the load-bearing part. Without it an absent row could be absent for an unrelated reason and every assertion would pass vacuously — which is the failure mode being fixed, not a shape to reproduce. Verified by mutation: with the gate removed, six of these fail and the positive control still passes. --- packages/db/src/related-persons-sql.test.ts | 101 ++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index 1c88b94f..380949a0 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -524,4 +524,105 @@ describe('свързани-лица SQL (real SQLite)', () => { ); }); }); + + // ── the evidence seal gate (#279, ADR-0033 decision 1) ────────────────────── + // + // This block exists because the gate had NO test pressure at all: deleting the entire + // `EXISTS (… interest_link_evidence …)` clause out of SURFACED_OWNERSHIP left the whole db suite + // green (361/361), because every fixture above seals every link 'document'. That clause is the only + // SQL between a WITHHELD link and a named public claim that a specific official owns a specific + // company, so an untested one is the most expensive kind of dead rail. + // + // Each rung below is a REAL withholding outcome of evidence.mjs, not an invented value: + // refuted — the register contradicts the declared stake + // bar_joint_stock — an АД/ЕАД, where a declared parcel of shares is not a material conflict + // unknown — no rung reached; the deed proves nothing either way + // outside_tr — ДЗЗД/BULSTAT, not in the Trade Register at all + // …plus a link with NO seal row whatsoever, which is what a half-loaded run produces. + describe('a link withheld by its evidence seal never reaches any public query', () => { + // All six share the SAME winner (eik 111, which has live contracts) and the same shape, so the + // ONLY thing that differs is the seal. Without that, an absent row could be absent for an unrelated + // reason and every assertion here would pass vacuously — which is exactly the bug being fixed. + const WITHHELD = [ + ['refuted', 'person:ref', 'Оборен Тестов'], + ['bar_joint_stock', 'person:bar', 'Акционер Тестов'], + ['unknown', 'person:unk', 'Неясен Тестов'], + ['outside_tr', 'person:out', 'Извън Тестов'], + ] as const; + + function seedRungs(dbPath: string): void { + const people = [...WITHHELD.map(([, id, name]) => [id, name] as const)]; + people.push(['person:none', 'Безпечатен Тестов'], ['person:ok', 'Потвърден Тестов']); + const links = [...WITHHELD.map(([, id]) => id), 'person:none', 'person:ok']; + sqlite( + dbPath, + `INSERT INTO persons (id, name) VALUES ${people.map(([id, n]) => `('${id}','${n}')`).join(',')}; + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES + ${links + .map( + (p) => + `('il:${p.split(':')[1]}','${p}|111','${p}','eik:111','111','ТРЕЙС ГРУП ХОЛД АД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'none',1,'2019','2023',3,1000,'2020','2021','published')`, + ) + .join(',')}; + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) VALUES + ${WITHHELD.map(([kind, id]) => `('${id}|111','${kind}',NULL,NULL,'2026-08-05','tr-rules-1','live')`).join(',')}, + ('person:ok|111','confirmed',NULL,'seat:СОФИЯ','2026-08-05','tr-rules-1','live');`, + ); + // person:none deliberately gets NO evidence row at all. + } + + it('the POSITIVE CONTROL surfaces — so every absence below is caused by the seal, not the fixture', () => { + withDb((dbPath) => { + seedRungs(dbPath); + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => r.official === 'Потвърден Тестов')).toBe(true); + expect(rows(dbPath, lit(OFFICIAL_SQL, 'person:ok'))).toHaveLength(1); + }); + }); + + for (const [kind, personId, name] of WITHHELD) { + it(`'${kind}' is withheld from the leaderboard, the official page, the company page and the drill-down`, () => { + withDb((dbPath) => { + seedRungs(dbPath); + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => r.official === name)).toBe(false); + expect(board.some((r) => r.link_key === `${personId}|111`)).toBe(false); + // The official's own page must 404 rather than render a withheld claim under their name… + expect(rows(dbPath, lit(OFFICIAL_SQL, personId))).toHaveLength(0); + // …the company page must not list them among that winner's office-holders… + expect(rows(dbPath, lit(COMPANY_SQL, '111')).some((r) => r.official === name)).toBe( + false, + ); + // …and the drill-down must not enumerate the contracts of a link it may not name. + expect(rows(dbPath, lit(LINK_CONTRACTS_SQL, `${personId}|111`))).toHaveLength(0); + }); + }); + } + + it('a link with NO seal row at all is withheld too — a half-loaded run must not publish', () => { + // The LEFT JOIN in LINK_SELECT would happily return this row with NULL evidence columns; only the + // EXISTS gate keeps it off the surface. This is the case a partial load actually produces. + withDb((dbPath) => { + seedRungs(dbPath); + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => r.official === 'Безпечатен Тестов')).toBe(false); + expect(rows(dbPath, lit(OFFICIAL_SQL, 'person:none'))).toHaveLength(0); + expect( + rows(dbPath, lit(COMPANY_SQL, '111')).some((r) => r.official === 'Безпечатен Тестов'), + ).toBe(false); + expect(rows(dbPath, lit(LINK_CONTRACTS_SQL, 'person:none|111'))).toHaveLength(0); + }); + }); + + it('the company page shows ONLY the sealed office-holders of that winner', () => { + // Four withheld links and one confirmed link all point at eik 111, alongside the base fixture's + // Иван. A gate that let any withheld rung through would show up here as an extra name. + withDb((dbPath) => { + seedRungs(dbPath); + const names = rows(dbPath, lit(COMPANY_SQL, '111')).map((r) => r.official); + expect(names.sort()).toEqual(['Иван Минев', 'Потвърден Тестов']); + }); + }); + }); }); From 0a6131795978e56bde7882ce5663f48787ceabec Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 13:54:46 +0300 Subject: [PATCH 22/69] fix(db): withhold an unrecognised evidence seal instead of upgrading it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit toLink read `evidence_kind === 'confirmed' ? 'confirmed' : 'document'`, so NULL, 'refuted', 'bar_joint_stock', 'outside_tr', a rung added by a later rules_version, or a typo all became 'document' — the strongest claim the card can make, rendering „лицето е вписано като съдружник/собственик": that the register names this specific person in this specific company. The SQL gate makes that unreachable today, but the failure direction was backwards at the one mapping in the codebase where a wrong default is a defamatory statement about a named human being rather than a rendering glitch. The LEFT JOIN in LINK_SELECT was deliberately left non-inner so a contradiction would surface here; the fallback then converted exactly that contradiction into the strongest available label. Rows are now filtered to the two publishing rungs before mapping, and filtered before the emptiness check so an official whose every link is withheld 404s rather than rendering an empty page under their name. One withheld row no longer affects its sealed siblings. --- .../db/src/queries/related-persons.test.ts | 51 +++++++++++++++++++ packages/db/src/queries/related-persons.ts | 35 ++++++++++--- 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/packages/db/src/queries/related-persons.test.ts b/packages/db/src/queries/related-persons.test.ts index 51985651..8ba09404 100644 --- a/packages/db/src/queries/related-persons.test.ts +++ b/packages/db/src/queries/related-persons.test.ts @@ -33,6 +33,13 @@ function row(over: Record = {}) { first_contract_year: '2021', last_contract_year: '2024', source_url: 'https://register.cacbg.bg/2024/i.xml', + // SURFACED_OWNERSHIP only returns rows carrying a publishing seal, so the fixture carries one too — + // a fixture without it would exercise a row shape the SQL cannot produce. + evidence_kind: 'document', + registry_role: 'owner', + entry_number: '20110502101007', + entry_date: '2011-05-02', + lookup_date: '2026-08-05', ...over, }; } @@ -197,3 +204,47 @@ describe('conflict reads soft-fail on an un-migrated env (no 500)', () => { } }); }); + +// The evidence seal is what licenses the card's registry sentence, and the two rungs assert very +// different things: 'document' renders „лицето е вписано като съдружник/собственик" — that the register +// names THIS person in THIS company — while 'confirmed' claims only that the declared data matched. +// The mapper used to read `kind === 'confirmed' ? 'confirmed' : 'document'`, so NULL, 'refuted', a rung +// added later, or a typo all became the STRONGEST claim we can make about a named human being. The SQL +// gate makes that unreachable today; the failure DIRECTION is still wrong, and this is the one place in +// the codebase where being wrong by default is libel rather than a rendering glitch. +describe('an unrecognised evidence seal withholds the link instead of upgrading it', () => { + for (const kind of ['refuted', 'unknown', 'bar_joint_stock', 'outside_tr', 'future_rung_v9']) + it(`'${kind}' never renders as a registry claim`, async () => { + const db = fakeDb({ '10': [row({ evidence_kind: kind })] }); + expect(await getConflictLeaderboard(db, 10)).toEqual([]); + }); + + it('a NULL seal — the half-migrated read the old comment invited — withholds too', async () => { + const db = fakeDb({ '10': [row({ evidence_kind: null })] }); + expect(await getConflictLeaderboard(db, 10)).toEqual([]); + }); + + it('both publishing rungs still map, and to DIFFERENT kinds — the guard bounds, it does not flatten', async () => { + for (const kind of ['document', 'confirmed']) { + const db = fakeDb({ '10': [row({ evidence_kind: kind })] }); + const links = await getConflictLeaderboard(db, 10); + expect(links).toHaveLength(1); + expect(links[0]!.evidenceKind).toBe(kind); + } + }); + + it('the official page 404s rather than render a page under a name with nothing left to show', async () => { + const db = fakeDb({ 'person:ivan': [row({ evidence_kind: 'refuted' })] }); + expect(await getOfficialConflicts(db, 'person:ivan')).toBeNull(); + expect( + await getCompanyConflicts(fakeDb({ '111': [row({ evidence_kind: null })] }), '111'), + ).toBeNull(); + }); + + it('one withheld row does not take its sealed siblings down with it', async () => { + const db = fakeDb({ + '10': [row({ link_key: 'ok|111' }), row({ link_key: 'bad|111', evidence_kind: 'refuted' })], + }); + expect((await getConflictLeaderboard(db, 10)).map((l) => l.linkKey)).toEqual(['ok|111']); + }); +}); diff --git a/packages/db/src/queries/related-persons.ts b/packages/db/src/queries/related-persons.ts index ff99e7f5..8451b01a 100644 --- a/packages/db/src/queries/related-persons.ts +++ b/packages/db/src/queries/related-persons.ts @@ -190,6 +190,25 @@ export const LINK_SELECT = `SELECT il.link_key, il.person_id, p.name AS official -- …and drop a family link redundant with the official's own stake in the same winner (de-anon guard). AND ${NOT_REDUNDANT_FAMILY}`; +// The two rungs that license a public claim (ADR-0033 decision 1), and the ONLY two the card knows how +// to render. Anything else — 'refuted', 'unknown', 'bar_joint_stock', 'outside_tr', a rung added by a +// later rules_version, a typo, or NULL from a row with no seal — is withheld, never mapped. +// +// The mapper used to read `kind === 'confirmed' ? 'confirmed' : 'document'`, which turned every one of +// those into 'document' — the STRONGEST claim on the surface, rendering „лицето е вписано като +// съдружник/собственик": that the register names this specific person in this specific company. The SQL +// gate makes it unreachable today, but the direction was wrong, and this is the one mapping in the +// codebase where a default is a defamatory statement about a named human being rather than a glitch. +// The LEFT JOIN in LINK_SELECT is deliberately not an inner one so a contradiction SURFACES here; the +// old fallback then converted exactly that contradiction into the strongest possible label. +const PUBLISHING_EVIDENCE = new Set(['document', 'confirmed']); + +/** Rows whose seal licenses a public claim. Withholding is silent by design — the SQL already filters + * these out, so anything reaching here is a contradiction to drop, not a condition to report per row. */ +function sealed(rows: LinkRow[]): LinkRow[] { + return rows.filter((r) => PUBLISHING_EVIDENCE.has(String(r.evidence_kind))); +} + // own_institution is a 4-value verdict; only the deterministic 'exact' surfaces as true (the // name_contains/locality heuristics are disclosed elsewhere, never asserted as fact). function toLink(r: LinkRow): ConflictLink { @@ -213,11 +232,9 @@ function toLink(r: LinkRow): ConflictLink { firstContractYear: r.first_contract_year, lastContractYear: r.last_contract_year, sourceUrl: r.source_url, - // SURFACED_OWNERSHIP guarantees a publishing seal exists, so these are non-null in practice; the - // fallbacks keep a half-migrated environment rendering rather than throwing on a null read. - evidenceKind: (r.evidence_kind === 'confirmed' ? 'confirmed' : 'document') as - | 'document' - | 'confirmed', + // Narrowed, not defaulted — `sealed()` above has already dropped every other value, so this asserts + // what the filter guarantees instead of inventing a rung the row never carried. + evidenceKind: r.evidence_kind as 'document' | 'confirmed', registryRole: r.registry_role === 'owner' || r.registry_role === 'manager' ? r.registry_role : null, registryEntryNumber: r.entry_number, @@ -233,7 +250,7 @@ export const LEADERBOARD_SQL = `${LINK_SELECT} * relative's) in a procurement winner, ranked NEXUS-first (own-institution → contemporaneous → value). */ export async function getConflictLeaderboard(db: D1Database, limit = 100): Promise { try { - const rows = (await db.prepare(LEADERBOARD_SQL).bind(limit).all()).results; + const rows = sealed((await db.prepare(LEADERBOARD_SQL).bind(limit).all()).results); return rows.map(toLink); } catch (e) { if (conflictSchemaAbsent(e, 'leaderboard')) return []; // un-migrated env → empty surface, not a 500 @@ -251,7 +268,9 @@ export async function getOfficialConflicts( personId: string, ): Promise { try { - const rows = (await db.prepare(OFFICIAL_SQL).bind(personId).all()).results; + // Filtered BEFORE the emptiness check, so a person whose every link is withheld 404s rather than + // rendering an empty page under their name. + const rows = sealed((await db.prepare(OFFICIAL_SQL).bind(personId).all()).results); if (rows.length === 0) return null; const links = rows.map(toLink); return { official: links[0]!.official, links }; @@ -270,7 +289,7 @@ export async function getCompanyConflicts( eik: string, ): Promise { try { - const rows = (await db.prepare(COMPANY_SQL).bind(eik).all()).results; + const rows = sealed((await db.prepare(COMPANY_SQL).bind(eik).all()).results); if (rows.length === 0) return null; return { company: rows[0]!.company, eik, links: rows.map(toLink) }; } catch (e) { From df76ae742d61de6ff077c7683879ea1a96daf631 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 13:57:11 +0300 Subject: [PATCH 23/69] fix(cacbg): bound the seat matched_fact so a name cannot ride the seal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The closed-vocabulary rail admitted exactly what it exists to catch. `seat:` is a legitimate prefix and the settlement pattern was unbounded, so `seat:ИВАН ПЕТРОВ ГЕОРГИЕВ` — a full three-part Bulgarian name (ЗГР чл. 9) wearing an allowed prefix — passed the audit. That is the value a mis-split of the seat field would produce, which is the leak the rail was written for. A Bulgarian settlement is one or two tokens („СОФИЯ", „ВЕЛИКО ТЪРНОВО", „ГЕНЕРАЛ ТОШЕВО"); a three-part name is exactly three. Bounding at two separates them, and a rarer three-token seat stops the run for a human rather than publishing — the right direction for a rail whose failure mode is putting somebody's name on a served column. Moved to evidence.mjs next to the code that writes matched_fact, so the gate and the writer cannot drift apart, and enforced at write time as well: the audit runs after the whole domain is built, by which point the name is already in a table. --- scripts/cacbg/audit.mjs | 10 ++++++---- scripts/cacbg/audit.test.mjs | 35 +++++++++++++++++++++++++++++++++++ scripts/cacbg/load.mjs | 16 +++++++++++++++- scripts/tr/evidence.mjs | 24 ++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 5 deletions(-) diff --git a/scripts/cacbg/audit.mjs b/scripts/cacbg/audit.mjs index 2a20db6a..9c8355f6 100644 --- a/scripts/cacbg/audit.mjs +++ b/scripts/cacbg/audit.mjs @@ -9,7 +9,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import fs from 'node:fs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; -import { RULES_VERSION } from '../tr/evidence.mjs'; +import { RULES_VERSION, isSealedFact } from '../tr/evidence.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); @@ -46,8 +46,10 @@ const published = db // The only two evidence rungs that publish (ADR-0033 decision 1). Everything else withholds. const PUBLISHING_EVIDENCE = new Set(['document', 'confirmed']); -// The closed vocabulary for a sealed matched_fact. Anything else is a potential name leak. -const MATCHED_FACT = /^(?:seat:[\p{Lu} -]+|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; +// The closed vocabulary for a sealed matched_fact lives in evidence.mjs, next to the code that WRITES +// it — one definition, so the gate can never permit a shape the writer has stopped emitting (or, worse, +// the other way round). `isSealedFact` bounds the seat leg to a settlement's one or two tokens, which is +// what stops a three-part Bulgarian name riding through behind the legitimate `seat:` prefix. const findings = []; const flag = (link, axis, detail) => @@ -111,7 +113,7 @@ for (const l of published) { // C2. The PII rail, audited rather than assumed: matched_fact is a CLOSED vocabulary and can never // carry a name. The registry deed's names are read only to produce a boolean and must never reach // a served column (#279 §9, ADR-0033 decision 5). A schema cannot enforce this; this does. - if (l.matched_fact != null && !MATCHED_FACT.test(l.matched_fact)) + if (!isSealedFact(l.matched_fact)) flag( l, 'C_matched_fact_shape', diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index 38f4c346..712178c3 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -218,6 +218,41 @@ test('a matched_fact outside the closed vocabulary is a hard finding — the nam assert.equal(/C_matched_fact_shape/.test(out), true, out); }); +// …and the rail must catch the shape a leak would ACTUALLY take. `seat:` is a legitimate member +// of the vocabulary, so an unbounded `seat:[\p{Lu} -]+` admits `seat:ИВАН ПЕТРОВ ГЕОРГИЕВ` — a full +// three-part Bulgarian name (ЗГР чл. 9) wearing the prefix of a fact we allow. That is precisely the +// value the rail exists to reject, and it is the one a mis-split of the seat field would produce. +// A Bulgarian settlement is one or two tokens („СОФИЯ", „ВЕЛИКО ТЪРНОВО", „ГЕНЕРАЛ ТОШЕВО"); the +// three-part name is exactly three. The bound is deliberately tight: a rarer 3-token seat trips the +// audit and a human adjudicates, which is the correct direction for a rail whose failure mode is +// publishing somebody's name. +test('a THREE-TOKEN seat is a name shape, not a settlement — the rail must catch it', () => { + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','confirmed','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','confirmed',NULL,'seat:ИВАН ПЕТРОВ ГЕОРГИЕВ','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'a three-part name behind seat: must fail the gate'); + assert.equal(/C_matched_fact_shape/.test(out), true, out); +}); + +test('a real two-token settlement still passes — the bound must not empty the seat rung', () => { + const { threw } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','confirmed','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','confirmed',NULL,'seat:ВЕЛИКО ТЪРНОВО','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, false, 'ВЕЛИКО ТЪРНОВО is a settlement and must survive the rail'); +}); + test('a well-formed seal passes every C axis (positive control)', () => { // Without this, all four negatives above would still pass if the axes fired unconditionally. const { threw } = buildAndAudit({ diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 7900a17e..df5845dc 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -19,7 +19,12 @@ import { fileURLToPath } from 'node:url'; import { temporalStatus, localityToken, closelyHeldForm } from './classify.mjs'; import { openCache, readDeed, coverage } from '../tr/cache.mjs'; import { TR_DB, TR_RAW } from '../tr/paths.mjs'; -import { evidenceVerdict, reconcileTermination, RULES_VERSION } from '../tr/evidence.mjs'; +import { + evidenceVerdict, + isSealedFact, + reconcileTermination, + RULES_VERSION, +} from '../tr/evidence.mjs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; import { fingerprint, loadSuppressions, SUPPRESSION_KEY_VERSION } from './suppressions.mjs'; import { canonicalInstitution } from './institutions.mjs'; @@ -773,6 +778,15 @@ for (const rec of agg.values()) { : recon.label === 'manager_today' ? 'terminated_manager_still' : 'terminated'; + // Refuse at WRITE time, not only in the post-hoc audit. The audit runs after the whole domain is + // built; by then the name is already in a table, and a run that dies on a suppressed-by-default axis + // could ship it. `matched_fact` is the one sealed column derived from a deed's text, so it is the one + // place a third-party name can reach a served row — the rail belongs where the value is produced. + if (!isSealedFact(verdict.matchedFact)) + throw new Error( + `REFUSE TO SEAL: matched_fact for ${linkKey} is outside the closed vocabulary — a name may have ` + + `leaked out of a deed (#279 §9, ADR-0033 decision 5)`, + ); insEvidence.run( linkKey, verdict.kind, diff --git a/scripts/tr/evidence.mjs b/scripts/tr/evidence.mjs index f6619dad..01fce0e8 100644 --- a/scripts/tr/evidence.mjs +++ b/scripts/tr/evidence.mjs @@ -40,6 +40,30 @@ export const RULES_VERSION = 'tr-rules-1'; /** Rung 2 needs a real three-part Bulgarian name (ЗГР чл. 9). Two tokens is the homonym risk itself. */ const MIN_NAME_TOKENS = 3; +/** + * The CLOSED vocabulary a sealed `matched_fact` may take: `seat:`, `role:owner:`, + * `role:manager:`, or `eik`. It must NEVER carry the matched NAME — the deed's names are read + * only to produce a boolean and never leave git-ignored scratch (#279 §9, ADR-0033 decision 5). + * + * The seat token bound is the whole rail. `seat:` is a legitimate prefix, so an unbounded settlement + * pattern admits `seat:ИВАН ПЕТРОВ ГЕОРГИЕВ` — a full three-part Bulgarian name (ЗГР чл. 9) wearing an + * allowed prefix, which is exactly the value a mis-split of the seat field would produce and exactly + * what the rail exists to reject. A settlement is one or two tokens („СОФИЯ", „ВЕЛИКО ТЪРНОВО", + * „ГЕНЕРАЛ ТОШЕВО"); a three-part name is exactly three. Bounding at two separates them cleanly, and a + * rarer 3-token seat stops the run for a human rather than publishing — the correct direction for a rail + * whose failure mode is putting somebody's name on a served column. + * + * Defined ONCE and consumed by both the writer (load.mjs) and the audit, so the two cannot drift into + * a state where the gate permits what the writer emits. + */ +export const MATCHED_FACT_RE = + /^(?:seat:\p{Lu}[\p{Lu}-]*(?: \p{Lu}[\p{Lu}-]*)?|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + +/** True when `fact` is a member of the closed vocabulary. `null` is legal — a rung may match no fact. */ +export function isSealedFact(fact) { + return fact == null || MATCHED_FACT_RE.test(String(fact)); +} + // Court-registered companies were re-registered into the Търговски регистър in a single administrative // push, which flattened their entry dates into this window. „Strictly before the declared period" // certifies nothing when the date is an artefact of the migration rather than of the ownership, so the From 2d3c06420a60122a2b3f87175bfb79333fb444c0 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 13:58:30 +0300 Subject: [PATCH 24/69] fix(seed): seal the dev fixture so /conflicts is not empty on a fresh DB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seed exists for one reason — without it `/conflicts` renders an empty surface and the feature is unverifiable in a browser. Then #279 added the evidence seal requirement to SURFACED_OWNERSHIP, the seed wrote no seals, and the fixture stopped surfacing: this PR's own dev fixture died under this PR's own gate. Seal each seeded link with the rung it represents — document for the official's own stake, confirmed for the relative's seat-proved link, bar_joint_stock for the withheld АД — so the local surface shows the same three outcomes as production and keeps its negative case. The new test runs the real migrations, the real seed and the real exported queries, because the seed is a promise about the read gate and nothing was checking it. A future change to the publishing rule that the seed does not keep up with now fails here instead of in a browser days later. --- packages/db/src/seed-surface.test.ts | 95 ++++++++++++++++++++++++++++ scripts/seed.sql | 21 ++++++ 2 files changed, 116 insertions(+) create mode 100644 packages/db/src/seed-surface.test.ts diff --git a/packages/db/src/seed-surface.test.ts b/packages/db/src/seed-surface.test.ts new file mode 100644 index 00000000..e9484d25 --- /dev/null +++ b/packages/db/src/seed-surface.test.ts @@ -0,0 +1,95 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { COMPANY_SQL, LEADERBOARD_SQL, OFFICIAL_SQL } from './queries/related-persons'; + +// The dev seed exists for ONE reason: `/conflicts` renders an empty surface without it, so the whole +// feature is unverifiable in a browser. That makes the seed a promise about the read gate, and a promise +// nothing checked — it was written against SURFACED_OWNERSHIP as it stood, then #279 added the evidence +// seal requirement to that same predicate and the fixture silently stopped surfacing. A fresh dev DB +// rendered empty, which is exactly the state the seed was created to prevent. +// +// So: run the REAL migrations, the REAL seed.sql, and the REAL exported queries. Any future change to +// the publishing gate that the seed does not keep up with now fails here rather than in someone's +// browser, days later, as „the local app looks broken". + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migrations = [ + 'packages/db/migrations/0000_init.sql', + 'packages/db/migrations/0003_related_persons_foundation.sql', + 'packages/db/migrations/0006_interest_link_evidence.sql', +].map((m) => resolve(root, m)); +const seed = resolve(root, 'scripts/seed.sql'); + +function seededDb(fn: (dbPath: string) => T): T { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-seed-')); + const dbPath = resolve(dir, 'seed.sqlite'); + try { + for (const file of [...migrations, seed]) + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${file}\n`, stdio: 'pipe' }); + return fn(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} +function rows(dbPath: string, sql: string): Record[] { + const out = execFileSync('sqlite3', ['-json', dbPath], { input: sql, encoding: 'utf8' }).trim(); + return out ? JSON.parse(out) : []; +} +function lit(sql: string, ...vals: (string | number)[]): string { + let i = 0; + return sql.replace(/\?/g, () => { + const v = vals[i++]; + return typeof v === 'number' ? String(v) : `'${String(v).replace(/'/g, "''")}'`; + }); +} + +describe('scripts/seed.sql produces a surface a developer can actually see', () => { + it('the leaderboard is NOT empty — the seed survives every gate in SURFACED_OWNERSHIP', () => { + seededDb((dbPath) => { + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.length).toBeGreaterThan(0); + // Both published outcomes the seed is built around: the official's own stake and a relative's. + expect(board.map((r) => r.relation).sort()).toEqual(['owns', 'related']); + }); + }); + + it('every surfaced seed link carries a real publishing seal, not a coincidence', () => { + seededDb((dbPath) => { + for (const r of rows(dbPath, lit(LEADERBOARD_SQL, 100))) + expect(['document', 'confirmed']).toContain(r.evidence_kind); + }); + }); + + it('the held joint-stock link stays off the surface — the seed keeps its negative case', () => { + // ГАМА ИНВЕСТ АД is seeded as a withheld link precisely so a change to the publishing rule shows up + // as a before/after rather than as an empty page either way. If it ever surfaces, the materiality + // bar (ADR-0022) or the seal gate has quietly stopped working. + seededDb((dbPath) => { + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => String(r.company).includes('ГАМА ИНВЕСТ'))).toBe(false); + expect(rows(dbPath, lit(COMPANY_SQL, '204556676'))).toHaveLength(0); + }); + }); + + it('the official and company pages render too — not just the leaderboard', () => { + seededDb((dbPath) => { + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + const own = board.find((r) => r.relation === 'owns')!; + expect(rows(dbPath, lit(OFFICIAL_SQL, String(own.person_id))).length).toBeGreaterThan(0); + expect(rows(dbPath, lit(COMPANY_SQL, String(own.eik))).length).toBeGreaterThan(0); + }); + }); + + it('re-running the seed is idempotent — INSERT OR IGNORE, no duplicated links', () => { + seededDb((dbPath) => { + const before = rows(dbPath, lit(LEADERBOARD_SQL, 100)).length; + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${seed}\n`, stdio: 'pipe' }); + expect(rows(dbPath, lit(LEADERBOARD_SQL, 100))).toHaveLength(before); + }); + }); +}); diff --git a/scripts/seed.sql b/scripts/seed.sql index 9da64854..3d26a0da 100644 --- a/scripts/seed.sql +++ b/scripts/seed.sql @@ -28,6 +28,7 @@ INSERT OR IGNORE INTO bidders (id, name, bulstat) VALUES -- • a LIVE contract for that ЕИК — the read-time zero-contract gate -- • signed_at inside [first_declared_year, last_declared_year] — IN_WINDOW ⇒ the „в периода" chip -- • no published SELF stake on the same (official, ЕИК) for a family row — NOT_REDUNDANT_FAMILY +-- • a Trade Register evidence seal of a PUBLISHING rung — the #279 seal gate -- One link per outcome, including the non-surfaced ones, so a change to the publishing rule is -- visible as a before/after rather than as an empty page either way. @@ -95,3 +96,23 @@ VALUES INSERT OR IGNORE INTO interest_link_authorities (link_key, authority_id, authority_name, contract_count, value_eur, own) VALUES ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'auth-plovdiv', 'Община Пример', 1, 429000, 'exact'), ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'auth-sofia', 'Община София', 1, 190000, 'exact'); + +-- Trade Register evidence seals (#279, ADR-0033, migration 0006). NOT optional decoration: since the +-- evidence ladder landed, SURFACED_OWNERSHIP requires a publishing seal, so a seeded link without one +-- renders NOTHING — which is the exact failure this fixture exists to prevent. A seal per link, one per +-- rung, so the dev surface shows the same three outcomes as production: +-- document — the register names this person in this company (the strongest rung) +-- confirmed — identity confirmed by declared data (seat), no person found in the act itself +-- bar_joint_stock — an АД: a declared parcel of shares is not a material ownership conflict +-- matched_fact stays inside the closed vocabulary (evidence.mjs isSealedFact) — never a name. +INSERT OR IGNORE INTO interest_link_evidence + (link_key, evidence_kind, registry_role, matched_fact, entry_number, entry_date, lookup_date, rules_version, live_status) +VALUES + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'document', 'owner', 'role:owner:CR_F_19_L', + '20220314150210', '2022-03-14', '2026-08-05', 'tr-rules-1', 'live'), + ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'confirmed', NULL, 'seat:СОФИЯ', + '20210902110455', '2021-09-02', '2026-08-05', 'tr-rules-1', 'live'), + -- The held link is sealed too — seals exist for held links so the review queue can explain itself, + -- and this one doubles as the fixture's negative case for the seal gate. + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|204556676', 'bar_joint_stock', NULL, NULL, + NULL, NULL, '2026-08-05', 'tr-rules-1', 'live'); From 26333d841ba50967fa8051820100207480290f49 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 14:00:55 +0300 Subject: [PATCH 25/69] fix(tr): bound both the response size and the erasure regex's backtracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two halves of the same exposure, on a paced crawl where a wasted request budget cannot be re-spent cheaply. The request timeout bounded how long a response may STALL, but nothing bounded how much it may SEND: the reader buffered whatever arrived, so the endpoint decided how much memory this process held. Cap a body at 8 MB — ~240× the 34 KB a measured deed occupies, so it bounds abuse and can never refuse a large-but-legitimate company — and destroy the request rather than reject and leave the socket draining. Split out as collectBody so it is testable without TLS or a live socket. The erasure-notice strip used an unbounded lazy `.*?`, which backtracks quadratically when the opening div is never closed: 34K→3.3ms, 68K→13.6ms, 136K→53.8ms, 272K→240ms, 1M→4.0s, ×4 per doubling. Under an 8 MB cap alone that is still minutes of remote-controlled CPU per deed, so the cap is not on its own a fix. Bounding the lazy run to 2000 characters — ~80× a real one-sentence notice — makes it linear: the same inputs measure 5.3 / 10.3 / 24.8 / 41.6 / 187ms. The bound cannot resurrect an owner: `erased` is decided independently by ERASED_MARKER, so an over-long notice still marks the block erased and liveFields still drops it. --- scripts/tr/client.mjs | 60 ++++++++++++++++++++++++++++++++------ scripts/tr/client.test.mjs | 45 ++++++++++++++++++++++++++++ scripts/tr/deed.mjs | 20 ++++++++++--- scripts/tr/deed.test.mjs | 30 +++++++++++++++++++ 4 files changed, 142 insertions(+), 13 deletions(-) diff --git a/scripts/tr/client.mjs b/scripts/tr/client.mjs index 23d3769e..473f5e30 100644 --- a/scripts/tr/client.mjs +++ b/scripts/tr/client.mjs @@ -70,19 +70,61 @@ const wait = (ms) => new Promise((r) => setTimeout(r, ms)); * module deliberately does NOT verify against system roots (it pins a leaf instead, ADR-0011), so a * reader comparing the two needs to see that this leg takes the ordinary, stronger path. */ -export function httpsGet(url, { timeoutMs = 20_000 } = {}) { +/** + * Ceiling on a single response body. A measured deed is ~34 KB, so 8 MB is ~240× the real thing — + * this bounds abuse, not the register, and can never refuse a large-but-legitimate company. + */ +export const MAX_BODY_BYTES = 8 * 1024 * 1024; + +/** + * Read one response into a Buffer, refusing past `maxBytes`. + * + * The timeout on the request bounds how long a response may STALL; nothing bounded how much it may + * SEND. Without a counter the crawler buffers whatever arrives, so a wedged or hostile endpoint decides + * how much memory this process holds. It also composes badly with the parser: the erasure regex in + * deed.mjs backtracks quadratically on unclosed markup (measured 34K→3.7ms, 68K→13.5ms, 136K→53.2ms, + * 272K→215.4ms — ×4 per doubling), and that path is reachable only through a body large enough to make + * it matter. One byte counter bounds both, and destroying the request is the load-bearing half: merely + * rejecting the promise would leave the socket draining the rest of the response. + * + * Separated from `httpsGet` so it is testable without TLS or a live socket — `res` needs only to be an + * emitter of 'data'/'end'/'error', which is the same injection posture as politeTrGet's `httpGet`. + */ +export function collectBody(res, { req, maxBytes = MAX_BODY_BYTES } = {}) { + return new Promise((resolve, reject) => { + const chunks = []; + let size = 0; + let done = false; + const fail = (err) => { + if (done) return; + done = true; + req?.destroy(err); // tear the socket down; do not sit and drain what we already refused + reject(err); + }; + res.on('data', (c) => { + if (done) return; + size += c.length; + if (size > maxBytes) { + fail(new Error(`response too large: over ${maxBytes} bytes — refusing to buffer it`)); + return; + } + chunks.push(c); + }); + res.on('end', () => { + if (done) return; + done = true; + resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) }); + }); + res.on('error', fail); + }); +} + +export function httpsGet(url, { timeoutMs = 20_000, maxBytes = MAX_BODY_BYTES } = {}) { return new Promise((resolve, reject) => { const req = https.get( url, { headers: { accept: 'application/json', 'user-agent': TR_USER_AGENT } }, - (res) => { - const chunks = []; - res.on('data', (c) => chunks.push(c)); - res.on('end', () => - resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) }), - ); - res.on('error', reject); - }, + (res) => collectBody(res, { req, maxBytes }).then(resolve, reject), ); req.setTimeout(timeoutMs, () => req.destroy(new Error(`timeout after ${timeoutMs}ms: ${url}`))); req.on('error', reject); diff --git a/scripts/tr/client.test.mjs b/scripts/tr/client.test.mjs index 6694076f..20142602 100644 --- a/scripts/tr/client.test.mjs +++ b/scripts/tr/client.test.mjs @@ -6,6 +6,7 @@ // So a 429 is an instruction to stop, not a transient to retry through. ADR-0033 decision 7. import { test } from 'node:test'; import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; import { TR_HOST, TR_USER_AGENT, @@ -14,6 +15,8 @@ import { politeTrGet, assertTrHost, httpsGet, + collectBody, + MAX_BODY_BYTES, } from './client.mjs'; const okRes = (body = '{}') => ({ status: 200, headers: {}, body: Buffer.from(body) }); @@ -150,3 +153,45 @@ test('RateLimitError carries the url and is distinguishable from a generic failu assert.ok(err instanceof Error); assert.match(err.message, /115536179/); }); + +// ── response size ───────────────────────────────────────────────────────────── +// The timeout bounds how long a response may STALL; nothing bounded how large it may GROW. A deed is +// ~34 KB measured; an unbounded reader buffers whatever arrives, so a wedged or hostile endpoint could +// have the crawler hold an arbitrary amount of memory. It also composes badly with the parser: the +// erasure regex in deed.mjs backtracks quadratically on unclosed markup (measured 34K→3.7ms, +// 68K→13.5ms, 136K→53.2ms, 272K→215.4ms — ×4 per doubling), and that path is only reachable through a +// body large enough to make it matter. One byte counter bounds both. +test('a response past the cap is refused and the request destroyed, not buffered', async () => { + const res = fakeRes(200); + const destroyed = []; + const p = collectBody(res, { req: { destroy: (e) => destroyed.push(e) }, maxBytes: 1024 }); + res.emit('data', Buffer.alloc(700)); + res.emit('data', Buffer.alloc(700)); // 1400 > 1024 + await assert.rejects(p, /too large|1024/i); + assert.equal(destroyed.length, 1, 'the socket must be torn down, not left draining'); +}); + +test('a response under the cap still resolves with the WHOLE body', async () => { + const res = fakeRes(200); + const p = collectBody(res, { req: { destroy: () => {} }, maxBytes: 1024 }); + res.emit('data', Buffer.from('{"uic":')); + res.emit('data', Buffer.from('"115536179"}')); + res.emit('end'); + const out = await p; + assert.equal(out.status, 200); + assert.equal(out.body.toString(), '{"uic":"115536179"}'); +}); + +test('the cap leaves real deeds far under it — it bounds abuse, not the register', () => { + // A measured deed is ~34 KB. The cap must sit well above that or it becomes a correctness bug that + // silently refuses large-but-legitimate companies. + assert.ok(MAX_BODY_BYTES >= 1_000_000, `cap ${MAX_BODY_BYTES} is too tight for a real deed`); +}); + +// A stand-in for an http.IncomingMessage: an emitter carrying a status code. +function fakeRes(status) { + const e = new EventEmitter(); + e.statusCode = status; + e.headers = {}; + return e; +} diff --git a/scripts/tr/deed.mjs b/scripts/tr/deed.mjs index 71686ed8..1bb4a9be 100644 --- a/scripts/tr/deed.mjs +++ b/scripts/tr/deed.mjs @@ -63,6 +63,21 @@ function stripTags(html) { // and honouring both costs nothing while assuming one costs a wrong publish. const ERASED_MARKER = /erasure-text-inline|ui-icon-erased|field-text--erased/; +// The erasure notice, stripped so „Заличено обстоятелство." never reads as content. +// +// The `{0,2000}` bound is not cosmetic. With a plain `.*?` this backtracks QUADRATICALLY on markup where +// the opening div is never closed: every opening restarts a scan to end-of-input, measured at 34K→3.3ms, +// 68K→13.6ms, 136K→53.8ms, 272K→240ms, 1M→4.0s — ×4 per doubling. Bounding the lazy run makes each start +// position scan a fixed window instead, which is linear: the same inputs measure 5.3 / 10.3 / 24.8 / 41.6 +// / 187ms. A real notice is one short sentence, so 2000 characters is ~80× the live shape. +// +// Failing to match is safe by construction: `erased` is decided independently by ERASED_MARKER above, so +// an over-long notice still marks the block erased and liveFields still drops it. The only visible effect +// is that its text survives into the block — which in strict mode raises the drift alarm, loudly, rather +// than passing anything through silently. +const ERASURE_NOTICE = + /]*class=['"][^'"]*erasure-text-inline[^'"]*['"][^>]*>.{0,2000}?<\/div>/gis; + /** * Split one field's htmlData into the separate registered entities it holds. * @@ -84,10 +99,7 @@ export function entityBlocks(html, { strict = false } = {}) { const erased = ERASED_MARKER.test(chunk); // Read the visible text WITHOUT the erasure notice, so „Заличено обстоятелство." never counts as // content and an erased block reads as empty. - const withoutNotice = chunk.replace( - /]*class=['"][^'"]*erasure-text-inline[^'"]*['"][^>]*>.*?<\/div>/gis, - ' ', - ); + const withoutNotice = chunk.replace(ERASURE_NOTICE, ' '); const text = stripTags(withoutNotice); if (erased && text !== '' && strict) { throw new Error( diff --git a/scripts/tr/deed.test.mjs b/scripts/tr/deed.test.mjs index c45611c8..4a31459f 100644 --- a/scripts/tr/deed.test.mjs +++ b/scripts/tr/deed.test.mjs @@ -336,3 +336,33 @@ test('latestOwnershipEntryDate is null when no live ownership field survives', ( const d = deedOf([field('CR_F_23_L', F23_ERASED, { fieldOperation: 2 })]); assert.equal(latestOwnershipEntryDate(d), null); }); + +// The erasure-notice strip used an unbounded lazy `.*?`, which backtracks quadratically when the opening +// div is never closed — each opening restarts a scan to end-of-input. Measured on that shape: 34K→3.3ms, +// 68K→13.6ms, 136K→53.8ms, 272K→240ms, 1M→4.0s (×4 per doubling). The parser runs against whatever the +// register returns, so that is remote-controlled CPU on a paced crawl with a per-request budget. +test('adversarial unclosed markup parses in linear time, not quadratically', () => { + // ~1 MB of unclosed erasure openings — the exact shape that triggers the backtracking. + const doc = '
z'.repeat(32_000); + const t = process.hrtime.bigint(); + entityBlocks(doc); + const ms = Number(process.hrtime.bigint() - t) / 1e6; + // Bounded measures ~190ms here and unbounded ~4000ms, so 1500ms separates them with ~8× headroom + // over the bounded path — wide enough not to flake on a loaded runner, tight enough to catch a + // reintroduced `.*?`. + assert.ok(ms < 1500, `entityBlocks took ${ms.toFixed(0)}ms on 1MB of unclosed markup`); +}); + +test('an over-long erasure notice still marks the block erased — the bound cannot leak a live owner', () => { + // If the notice exceeds the bound the regex simply does not strip it. `erased` is decided separately + // by ERASED_MARKER, so the block is still erased and liveFields still drops it: the failure mode of + // the bound is a noisier block, never a resurrected owner. + const long = `
${'Заличено. '.repeat(400)}
`; + const [block] = entityBlocks(long); + assert.equal(block.erased, true); + assert.deepEqual( + liveFields(deedOf([field('CR_F_19_L', long)]), ['CR_F_19_L']), + [], + 'an erased block contributes no live entity regardless of its notice length', + ); +}); From 436dbc230878823cc6108c7eec941b8c88cab368 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 14:02:53 +0300 Subject: [PATCH 26/69] fix(cacbg): re-derive the deed path, and pin the duplicated joint-stock rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit load.mjs joined the cache's stored raw_path straight onto the raw directory — the one read that did not go through the safeEik rail purgeExpired already uses. The cache index travels between runs, and once it does a stored path is attacker-influenced input on a filesystem root. Re-derive from the ЕИК through deedPath, which also throws on a malformed code rather than quietly reading another company's deed. JOINT_SUFFIX in the TR parser and JOINT_STOCK in the classifier are the same rule held in two places, because the TR parser cannot import out of scripts/cacbg/ without closing a cacbg↔tr cycle. That duplication has already drifted once — 5f64f5c added КДА to classify.mjs while deed.mjs's comment still asserted it was absent there, so the comment was simply false. Correct the comment and pin the two byte-identical, textually and behaviourally: an unenforced „keep these in step" note is how the drift happened in the first place. Verified by mutation — removing КДА from either side fails the test. --- scripts/cacbg/classify.mjs | 5 ++++- scripts/cacbg/load.mjs | 9 +++++++-- scripts/tr/deed.mjs | 11 +++++++++-- scripts/tr/deed.test.mjs | 23 +++++++++++++++++++++++ 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/scripts/cacbg/classify.mjs b/scripts/cacbg/classify.mjs index 8bc07bce..a5f9de25 100644 --- a/scripts/cacbg/classify.mjs +++ b/scripts/cacbg/classify.mjs @@ -77,7 +77,10 @@ const norm = (s) => // token bounded on the left by string edge, whitespace or quotes — NOT hyphens/dots, so „АД-ХОК ЕООД" (a // hyphenated ООД name) is not misread. Anchoring to the suffix is what stops „АД ГРУП ООД" (an ООД whose // NAME begins with the token „АД") being wrongly excluded as joint-stock — the form there is ООД. -const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; +// Twinned byte-for-byte by JOINT_SUFFIX in scripts/tr/deed.mjs — the TR parser cannot import from this +// directory without closing a cacbg↔tr cycle. A test in deed.test.mjs pins the two identical; change one +// and that test fails rather than the two silently diverging on a legal form. +export const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; /** * Materiality by legal form. The public ownership surface is CLOSELY-HELD companies only (ООД/ЕООД/ЕТ/ * КД/СД/ДЗЗД or a form-unspecified name from the closely-held table). Joint-stock forms (АД/ЕАД/АДСИЦ/КДА, diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index df5845dc..7e0c46c1 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -18,7 +18,7 @@ import { fileURLToPath } from 'node:url'; // path (ADR-0033). It survives in classify.mjs for the review queue, not for a publishing decision. import { temporalStatus, localityToken, closelyHeldForm } from './classify.mjs'; import { openCache, readDeed, coverage } from '../tr/cache.mjs'; -import { TR_DB, TR_RAW } from '../tr/paths.mjs'; +import { TR_DB, TR_RAW, deedPath } from '../tr/paths.mjs'; import { evidenceVerdict, isSealedFact, @@ -535,7 +535,12 @@ function deedFor(eik) { if (!row) entry = { deed: null, outsideTr: false, missing: true }; else if (row.status === 'outside_tr') entry = { deed: null, outsideTr: true, missing: false }; else { - const file = path.join(TR_RAW_DIR, row.rawPath ?? `${eik}.json`); + // RE-DERIVED from the ЕИК through safeEik, never the stored raw_path. The cache index is written by + // the crawler but travels between runs — and once it does, a stored path is attacker-influenced + // input joined straight onto a filesystem root, which is a traversal read. purgeExpired already + // re-derives for exactly this reason; this was the one read that did not. deedPath also throws on a + // malformed ЕИК rather than quietly reading some other company's deed (R8). + const file = deedPath(eik, TR_RAW_DIR); entry = fs.existsSync(file) ? { deed: JSON.parse(fs.readFileSync(file, 'utf8')), outsideTr: false, missing: false } : { deed: null, outsideTr: false, missing: true }; diff --git a/scripts/tr/deed.mjs b/scripts/tr/deed.mjs index 1bb4a9be..13837d7a 100644 --- a/scripts/tr/deed.mjs +++ b/scripts/tr/deed.mjs @@ -241,8 +241,15 @@ const FORM_CODES = new Map([ // The фирма's legal form is its SUFFIX under ЗТРРЮЛНЦ, and the deed envelope's `fullName` carries it // („ПИМК" ООД) while CR_F_2_L is bare („ПИМК"). So the bar has a second, independent signal at zero -// extra cost. КДА is included here and is absent from classify.mjs's JOINT_STOCK/FORM_TOKENS today. -const JOINT_SUFFIX = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; +// extra cost. +// +// DELIBERATELY a twin of classify.mjs's JOINT_STOCK rather than an import, for the same reason +// personTokens twins parse.mjs's holderTokens: the dependency direction here is cacbg → tr (load.mjs +// imports this module), so importing back out of scripts/cacbg/ would close a cycle across the two +// directories. The two are pinned byte-identical by a test in deed.test.mjs — this comment used to +// claim КДА was missing from classify.mjs, which stopped being true in 5f64f5c, and an unenforced +// „keep these in step" note is exactly how that happens. +export const JOINT_SUFFIX = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; const CLOSELY_SUFFIX = /(?:^|[\s"„“”«»])(ООД|ЕООД|ЕТ|ДЗЗД|КД|СД|КООПЕРАЦИЯ)[\s"„“”«»]*$/u; /** diff --git a/scripts/tr/deed.test.mjs b/scripts/tr/deed.test.mjs index 4a31459f..9c539c87 100644 --- a/scripts/tr/deed.test.mjs +++ b/scripts/tr/deed.test.mjs @@ -20,6 +20,7 @@ import { registryLegalForm, latestOwnershipEntryDate, assertUicEcho, + JOINT_SUFFIX, } from './deed.mjs'; // ── real markup ─────────────────────────────────────────────────────────────── @@ -366,3 +367,25 @@ test('an over-long erasure notice still marks the block erased — the bound can 'an erased block contributes no live entity regardless of its notice length', ); }); + +// JOINT_SUFFIX here and JOINT_STOCK in scripts/cacbg/classify.mjs are the SAME rule — which legal-form +// suffixes mark a share-issuing company — held in two places because the TR parser cannot import out of +// scripts/cacbg/ without closing a cacbg↔tr cycle. The drift this risks has already happened once: 5f64f5c +// added КДА to classify.mjs while deed.mjs's comment still asserted it was absent there. A prose „keep +// these in step" note does not keep anything in step; this does. +test('the joint-stock suffix rule is identical in the TR parser and the classifier', async () => { + const { JOINT_STOCK } = await import('../cacbg/classify.mjs'); + assert.equal(JOINT_SUFFIX.source, JOINT_STOCK.source, 'the two patterns have diverged'); + assert.equal(JOINT_SUFFIX.flags, JOINT_STOCK.flags, 'the two patterns have diverged in flags'); + // Behavioural pin as well as textual: identical sources with different behaviour is impossible, but a + // future refactor could legitimately change BOTH sources while breaking one. These are the forms the + // bar exists for — every one must be caught by both, or a joint-stock parcel publishes as ownership. + for (const name of ['ТРЕЙС ГРУП ХОЛД АД', 'НЕЩО ЕАД', 'ФОНД АДСИЦ', 'НЕЩО КДА']) { + assert.equal(JOINT_SUFFIX.test(name), true, name); + assert.equal(JOINT_STOCK.test(name), true, name); + } + for (const name of ['АЛФА СТРОЙ ООД', 'БЕТА ЕООД', 'АД ГРУП ООД']) { + assert.equal(JOINT_SUFFIX.test(name), false, name); + assert.equal(JOINT_STOCK.test(name), false, name); + } +}); From 024c26f0e791aff0161c84cbc10a9abf76a4bc4c Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 14:07:54 +0300 Subject: [PATCH 27/69] feat(cacbg): add --emit-candidates so one job can bootstrap its own crawl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The register crawl and the decision run have to share a job: the raw deeds hold third-party names and cannot travel between runners. But the crawl's input list is derived from the resolved corpus, so only load.mjs can produce it — and load.mjs refuses to run without the very cache that list is used to fill. --emit-candidates writes the list and exits 0. Deriving it by running the full load and ignoring a non-zero exit was the alternative, and it erases the difference between „no cache yet" and „this run is broken". It runs on a throwaway copy of the work DB, and that is correctness rather than tidiness. Reaching the candidate list means rebuilding the corpus tables, which drops interest_links, and the pass never publishes — so against the real DB it would leave the table empty. The next real run would then export an EMPTY prior-published set as its monotonicity snapshot, and the gate whose only job is to notice a published claim disappearing would pass unconditionally from then on. Verified by mutation: pointing the pass at the real DB fails both tests. --- scripts/cacbg/load.mjs | 55 ++++++++++++++++++++--- scripts/cacbg/load.test.mjs | 87 +++++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 5 deletions(-) diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 7e0c46c1..bcceeb88 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -50,6 +50,19 @@ const TR_RAW_DIR = process.env.TR_RAW_DIR || TR_RAW; // A deliberate, logged override for the coverage gate below. Without it a single permanently // unreachable ЕИК would deadlock the pipeline forever; with it, the operator states that they know. const ALLOW_PARTIAL_TR = process.argv.includes('--allow-partial-tr'); +// Bootstrap mode: write the crawl's input list and stop, successfully. The decision run and the register +// crawl now share one job (they must — the raw deeds hold third-party names and cannot travel between +// runners), and that job has to be able to start from nothing: the list is derived from the resolved +// corpus, so only this script can produce it, but the full run refuses without the very cache the list +// is used to fill. Ignoring the refusal's exit code instead would erase the difference between „no cache +// yet" and „this run is broken". +// +// POINT THIS AT A SCRATCH COPY OF THE WORK DB. It is not a read-only pass: reaching the candidate list +// means rebuilding the corpus tables, so it drops and repopulates persons/declarations/declared_interests +// and leaves interest_links EMPTY. Empty is the safe end state (the ship floor refuses it, and no link +// can be published without evidence it never gathered), but it is not the state a subsequent real run +// should inherit. The one thing it must never touch either way is the monotonicity snapshot — see below. +const EMIT_CANDIDATES_ONLY = process.argv.includes('--emit-candidates'); const { companyNameKey, isMatchableKey } = await import('../../packages/shared/src/company-name-key.ts'); @@ -73,7 +86,24 @@ const readJsonl = (f) => .map((l) => JSON.parse(l)) : []; -const db = new DatabaseSync(DB); +// Bootstrap mode works on a THROWAWAY COPY, and that is not a convenience — it is the correctness of the +// monotonicity gate. Reaching the candidate list means rebuilding the corpus tables, which drops +// interest_links; the pass itself never publishes, so it would leave the table EMPTY. The next real run +// would then read that empty table as its prior-published set, write an empty snapshot, and the gate — +// whose only job is to notice a published claim disappearing — would pass unconditionally, for ever. +// Copying here rather than asking the caller to do it keeps the flag safe wherever it is invoked from. +const WORK_DB = EMIT_CANDIDATES_ONLY ? `${DB}.bootstrap` : DB; +if (EMIT_CANDIDATES_ONLY) { + for (const suffix of ['', '-wal', '-shm']) { + // -wal/-shm may legitimately be absent (a cleanly closed DB has neither); anything else must surface. + try { + fs.copyFileSync(`${DB}${suffix}`, `${WORK_DB}${suffix}`); + } catch (e) { + if (e.code !== 'ENOENT') throw e; + } + } +} +const db = new DatabaseSync(WORK_DB); db.exec('PRAGMA foreign_keys=ON'); // Suppressions live in a VERSION-CONTROLLED, HMAC-fingerprinted list (ADR-0031), NOT a DB table — so a // takedown survives a fresh-CI-runner rebuild and never ships the „who was taken down" signal to prod. @@ -115,10 +145,13 @@ const priorPublished = (() => { return []; } })(); -fs.writeFileSync( - path.join(STAGING, 'published-snapshot.json'), - JSON.stringify(priorPublished, null, 2) + '\n', -); +// Not written by the bootstrap pass, which works on a throwaway copy and has no business restating what +// the real run is about to record. +if (!EMIT_CANDIDATES_ONLY) + fs.writeFileSync( + path.join(STAGING, 'published-snapshot.json'), + JSON.stringify(priorPublished, null, 2) + '\n', + ); // Full idempotent rebuild that also picks up schema changes: drop the CACBG tables (children first — // FK-safe) and re-apply the migration. Nothing to preserve — suppressions are external now. @@ -491,6 +524,18 @@ function authOwn(authorityName, instNorms, instNormsLong, locTokens) { const candidateEiks = [...new Set([...agg.values()].map((r) => r.eik))].sort(); fs.writeFileSync(path.join(STAGING, 'candidate-eiks.txt'), candidateEiks.join('\n') + '\n'); +if (EMIT_CANDIDATES_ONLY) { + // Stop BEFORE the TR gate and before anything is written to the domain. A bootstrap pass that built + // links would leave a surface resting on no evidence at all, and a failure between this pass and the + // real one would leave that surface sitting in the work DB, shippable. + console.log( + `${candidateEiks.length} candidate ЕИК written for the crawler; stopping (--emit-candidates)`, + ); + db.close(); + for (const suffix of ['', '-wal', '-shm']) fs.rmSync(`${WORK_DB}${suffix}`, { force: true }); // the copy has served its purpose + process.exit(0); +} + if (!fs.existsSync(TR_CACHE_DB)) { db.close(); throw new Error( diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index bd5e5bc8..b2a9dbe7 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -1220,3 +1220,90 @@ test('the published surface is exported BEFORE the wipe, so the audit can gate m // next run is not a regression to gate on. assert.equal(snap.length, published.length); }); + +// The crawl input and the crawl's consumer are the same script, which used to make the two workflows +// deadlock: the decision run refused without a deed cache, the refresh run refused without a candidate +// list, and each produced only what the other needed. Merged into one job, the job still has to be able +// to BOOTSTRAP — produce the candidate list on a runner that has no cache yet — and it cannot do that by +// running the full load and ignoring a non-zero exit, because that exit is also how a genuinely broken +// run reports itself. Hence an explicit mode that stops at the list and succeeds. +test('--emit-candidates writes the crawl list and exits 0 with NO Trade Register cache', () => { + const gone = path.join(dir, 'bootstrap-absent.sqlite'); + const listFile = path.join(STAGING, 'candidate-eiks.txt'); + fs.rmSync(listFile, { force: true }); + assert.doesNotThrow(() => + execFileSync( + 'node', + [ + '--import', + path.join(HERE, 'register-ts.mjs'), + path.join(HERE, 'load.mjs'), + '--emit-candidates', + ], + { + cwd: ROOT, + env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING, TR_CACHE_DB: gone }, + stdio: 'pipe', + }, + ), + ); + const listed = fs.readFileSync(listFile, 'utf8').split('\n').filter(Boolean); + assert.ok(listed.length > 0, 'the bootstrap must actually produce candidates'); + assert.equal(new Set(listed).size, listed.length); + runLoad(); // restore the full built state for any later reader +}); + +function emitCandidates(trCacheDb) { + execFileSync( + 'node', + [ + '--import', + path.join(HERE, 'register-ts.mjs'), + path.join(HERE, 'load.mjs'), + '--emit-candidates', + ], + { + cwd: ROOT, + env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING, TR_CACHE_DB: trCacheDb }, + stdio: 'pipe', + }, + ); +} + +test('a bootstrap pass leaves the REAL work DB untouched — it runs on a throwaway copy', () => { + // Reaching the candidate list means rebuilding the corpus tables, and the pass never publishes, so + // against the real DB it would leave interest_links empty. That is the damage: not the pass itself, + // but what the NEXT run then reads. + runLoad(); + const db = open(); + const before = db.prepare('SELECT COUNT(*) AS n FROM interest_links').get().n; + db.close(); + assert.ok(before > 0, 'the fixture must actually publish something'); + + emitCandidates(path.join(dir, 'bootstrap-absent-2.sqlite')); + + const after = open(); + assert.equal( + after.prepare('SELECT COUNT(*) AS n FROM interest_links').get().n, + before, + 'the bootstrap pass must not empty the domain it was pointed at', + ); + after.close(); + assert.equal(fs.existsSync(`${DB}.bootstrap`), false, 'the throwaway copy must be cleaned up'); +}); + +test('the monotonicity gate still sees a prior surface AFTER a bootstrap pass', () => { + // The end-to-end shape of the merged workflow: real run → bootstrap (to produce the crawl list) → + // real run. If the bootstrap emptied interest_links, the second real run would export an EMPTY + // prior-published set, and the gate — whose only job is to notice a published claim disappearing — + // would pass unconditionally, for ever. + const snapshot = path.join(STAGING, 'published-snapshot.json'); + runLoad(); + emitCandidates(path.join(dir, 'bootstrap-absent-3.sqlite')); + runLoad(); + const prior = JSON.parse(fs.readFileSync(snapshot, 'utf8')); + assert.ok( + prior.length > 0, + 'the snapshot went empty — the gate is now vacuous and would never fire again', + ); +}); From ae661ac3838b67459db7023302257269094e1e9b Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 14:10:17 +0300 Subject: [PATCH 28/69] ci(related-persons): run the register crawl inside the decision job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two workflows deadlocked and both were permanently green no-ops. The crawl job gated on scratch/cacbg/staging/candidate-eiks.txt, which only the decision job writes and nothing persisted, so its check was false on every run forever. The decision job restored only scratch/cacbg/raw, never scratch/tr, so it hit `REFUSE TO LOAD: no Trade Register cache` on every run. Each waited on the other half of a handoff neither performed. They cannot be joined by persisting the deeds. The crawl's `path: scratch/tr` already cached deeds/*.json — the co-owner and manager names ADR-0033 decision 5 promises a 35-day retention for. purgeExpired runs on the runner, but older cache entries live on GitHub's storage under the restore-keys chain, where eviction is a capacity policy and not retention at all. So the deeds stay on one runner: crawl and decide in one job, and delete them explicitly at the end. The cadence follows from that. A decision needs the raw deeds — evidenceVerdict's strongest rung matches the declarant's name against the register's own text — so every run that decides must also crawl, and a daily decision would mean ~400 daily requests against somebody else's register. The schedule is therefore monthly, not daily as §9 and ADR-0033 describe. Restoring a daily decision needs the crawl to emit per-(link, ЕИК) verdicts so only booleans cross a run boundary; that is a design change and is deliberately not folded in here. Also export interest_links + interest_link_evidence in the hydrate step. Not for the resolver — load.mjs rebuilds them — but for the monotonicity gate: the work DB had no interest_links at all, so the pre-wipe export read "no such table", the snapshot was [] every run, and the gate added in 2327e62 protected nothing. --- .github/workflows/related-persons-data.yml | 111 ++++++++++++++-- .../workflows/related-persons-tr-refresh.yml | 119 ------------------ 2 files changed, 104 insertions(+), 126 deletions(-) delete mode 100644 .github/workflows/related-persons-tr-refresh.yml diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index c09bed4d..7058fd32 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -16,19 +16,38 @@ name: Related-persons data foundation (build + ship) # XML (fetch.mjs skips files already on disk) — it fills gaps, not a from-scratch rebuild unless the cache is cleared. on: - # DAILY, and deliberately on the EXISTING workflow rather than a second one. #279 §9 asks for the + # MONTHLY, and deliberately on the EXISTING workflow rather than a second one. #279 §9 asks for the # publishing decisions to run on a cadence; duplicating the job would create a second ship path with # its own copy of the credential guards, the D1-target guard and the ship floor — and the copy is the - # one nobody exercises. A scheduled run takes the same path as a manual one, with two differences: + # one nobody exercises. + # + # WHY MONTHLY AND NOT DAILY, which is what §9 and ADR-0033 describe. A decision cannot be recomputed + # without the raw deeds — evidenceVerdict's strongest rung matches the declarant's name against the + # register's own text — and the raw deeds must not be persisted between runs: they carry the names and + # addresses of co-owners and managers who hold no public office, and an Actions cache entry lives on + # GitHub's storage under a restore-keys chain where ADR-0033 decision 5's 35-day retention cannot reach + # it. So the deeds live and die with one runner, and every run that decides must also crawl. A daily + # decision would therefore mean ~400 daily requests against somebody else's register, which is what + # spec §3.3 exists to prevent — so the cadence follows the deeds, not the other way round. + # + # Restoring a daily decision needs the crawl to emit the per-(link, ЕИК) match verdict, so only + # booleans cross a run boundary and nothing has to hold a name. That is a design change, not a + # schedule change, and it is deliberately not folded in here. + # + # A scheduled run takes the same path as a manual one, with two differences: # • it targets STAGING (the environment default below). Production stays manual, so the # assertD1TargetAuthorized guard keeps a prod write a deliberate act. - # • `full_crawl` is empty and therefore false, so it never crawls the register — it re-runs the + # • `full_crawl` is empty and therefore false, so it never crawls the CACBG corpus — it re-runs the # decisions over the cached corpus, which is exactly what #279 §9 means by a pure, zero-network - # decision. Registry lookups have their own workflow (related-persons-tr-refresh.yml). + # decision. Registry lookups are the one exception and run from inside this job on the 1st of the + # month (see „Decide whether to hit the register"): they cannot be a separate workflow, because the + # raw deeds they fetch carry third-party names and must not be persisted between runners. # If the corpus cache has been evicted, extract yields nothing, the ship floor refuses, and the run # fails loudly rather than wiping the surface. schedule: - - cron: '0 3 * * *' + # 03:00 UTC on the 1st. See the note above: the decision and the crawl are one pass, so the decision + # cadence is the crawl cadence — #279 §9's monthly registry rhythm. + - cron: '0 3 1 * *' workflow_dispatch: inputs: environment: @@ -40,6 +59,14 @@ on: description: Crawl the CACBG corpus (slow). RESUMES from the cached raw XML — fetch.mjs skips files already on disk, so this fills gaps rather than re-fetching from scratch (clear the cache for a true rebuild). Off = extract from the existing raw cache without crawling. type: boolean default: false + tr_max_age_days: + description: Re-fetch deeds older than this many days (default 30). Use a large value to force a full refresh. + type: string + default: '30' + tr_limit: + description: Stop after this many registry lookups (blank = all pending). Bound a first run while watching for a 429. + type: string + default: '' # Least-privilege: this workflow only reads the repo; it authenticates to Cloudflare with a scoped token. permissions: @@ -54,8 +81,9 @@ jobs: build-and-ship: runs-on: ubuntu-latest # A from-scratch corpus crawl of the whole register is the long pole (all year-folders, politely - # throttled) and must fit in ONE job with extract/resolve/audit/ship after it — 120 was too tight and - # timed out mid-crawl. 300 leaves headroom under GitHub's 6-hour hard cap. A run past this is wedged + # throttled) and must fit in ONE job with the registry lookups, extract/resolve/audit/ship after it — + # 120 was too tight and timed out mid-crawl. The Търговски регистър pass adds ~20 minutes on top + # (~400 candidates at 1 request / 3 s). 300 leaves headroom under GitHub's 6-hour hard cap. A run past this is wedged # (stalled gov-server I/O) — fail rather than burn the slot. The raw corpus is cached (below) so a # re-run resumes instead of re-crawling from scratch. timeout-minutes: 300 @@ -164,11 +192,29 @@ jobs: work="$GITHUB_WORKSPACE/data/work" mkdir -p "$work" rm -f "$work/eop.sql" "$work/backfill.sqlite" + # interest_links + interest_link_evidence come along too, and NOT for the resolver's benefit — + # load.mjs drops and rebuilds them. They are the input to the monotonicity gate: load.mjs + # exports the currently-published set BEFORE the wipe, and audit.mjs fails the run on a link + # that was published last time, under an unchanged rules_version, and is not published now. + # Without them the work DB starts with no interest_links at all, the pre-wipe export reads + # „no such table", the snapshot is [] on every run, and the gate protects nothing. + # They are absent on a first run (0003/0006 not yet applied), which is why they are exported + # separately and tolerated as missing rather than folded into the parity loop below. pnpm --filter @sigma/web exec wrangler d1 export "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote -y \ --table bidders --table contracts --table tenders --table authorities \ --output "$work/eop.sql" sqlite3 "$work/backfill.sqlite" < "$work/eop.sql" + if pnpm --filter @sigma/web exec wrangler d1 export "$SIGMA_D1_NAME" \ + --config wrangler.deploy.jsonc --remote -y \ + --table interest_links --table interest_link_evidence \ + --output "$work/prior-links.sql"; then + sqlite3 "$work/backfill.sqlite" < "$work/prior-links.sql" + echo " prior surface: $(sqlite3 "$work/backfill.sqlite" \ + "SELECT COUNT(*) FROM interest_links WHERE status='published';") published link(s)" + else + echo "::notice::No prior interest_links on this D1 (first run) — the monotonicity gate has no baseline yet." + fi for t in bidders contracts tenders authorities; do local_n=$(sqlite3 "$work/backfill.sqlite" "SELECT COUNT(*) FROM $t;") remote_json=$(pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ @@ -182,9 +228,60 @@ jobs: fi done + # ── Trade Register lookups ──────────────────────────────────────────────────────────────────── + # In THIS job, deliberately, and not in a workflow of their own. The decision run needs the raw + # deeds (evidenceVerdict matches declarant names against the register's own text, which is exactly + # the data the index refuses to store), and those deeds carry the names and addresses of co-owners + # and managers who hold no public office. Handing them between runners means persisting them — + # an Actions cache entry survives on GitHub's storage under a restore-keys chain, where the 35-day + # retention ADR-0033 decision 5 promises cannot reach it. Same runner, same job, deleted with it. + # + # The previous split also could not work on its own terms: the crawl job gated on a candidate list + # only the decision job produced, the decision job refused without a cache only the crawl job + # produced, and neither persisted its half. Both were permanently green no-ops. + - name: Emit the candidate ЕИК list for the crawler + run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/load.mjs --emit-candidates + + # ALWAYS, on every run that reaches here — not conditionally. The deeds do not survive the runner, + # so „decide without crawling" is not a state this job can be in: load.mjs's coverage gate below + # would refuse the empty cache and the run would fail. Skipping the crawl is only meaningful once + # the crawl emits verdicts rather than deeds (see the note on the schedule above). + # + # Pace and refusals live in scripts/tr/fetch-deeds.mjs and are not configurable here on purpose: + # 1 request / 3 s, sequential, a closed candidate set, and a 429 ends the run without marking + # anything. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the limiter is + # the operator's only way to state a rate preference, so we do not tune around it. + - name: Refresh the Trade Register deed cache + env: + MAX_AGE: ${{ inputs.tr_max_age_days || '30' }} + LIMIT: ${{ inputs.tr_limit }} + run: | + set -euo pipefail + ARGS=(--eiks-file scratch/cacbg/staging/candidate-eiks.txt --max-age-days "$MAX_AGE") + [ -n "$LIMIT" ] && ARGS+=(--limit "$LIMIT") + set +e + node scripts/tr/fetch-deeds.mjs "${ARGS[@]}" + code=$? + set -e + # Exit 2 is the rate limiter, and it is NOT a build failure: the run stopped politely, marked + # nothing, and the partial cache is valid. It is not a licence to publish either — load.mjs's + # coverage gate below refuses a partial cache on its own, which is where that decision belongs. + if [ "$code" -eq 2 ]; then + echo "::warning::The register rate-limited us; the crawl stopped and the cache is resumable." + exit 0 + fi + exit "$code" + - name: Resolve → build domain in a work sqlite (libel gate must pass) run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/load.mjs + # The raw deeds are third-party personal data and exist only to produce booleans. They are deleted + # with the runner regardless; deleting them explicitly means a failure AFTER this point cannot + # leave them sitting in a job artifact or a debug upload someone adds later. + - name: Delete the raw deeds + if: always() + run: rm -rf scratch/tr/deeds + - name: Audit published links (fails the run on any libel-invariant violation) run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/audit.mjs diff --git a/.github/workflows/related-persons-tr-refresh.yml b/.github/workflows/related-persons-tr-refresh.yml deleted file mode 100644 index 9cc09add..00000000 --- a/.github/workflows/related-persons-tr-refresh.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: Related-persons — Trade Register cache refresh (monthly) - -# Refreshes the Търговски регистър deed cache the publishing decision rests on (#279, ADR-0033). -# -# WHY THIS IS ITS OWN WORKFLOW, separate from the decision run: this is the only step in the project -# that touches a public register at volume, and it is the only one with a network budget. The decisions -# themselves are a pure function of declarations, cached deeds and contracts — they must be able to run -# daily without ever calling the register. Splitting them keeps that true by construction. -# -# It SHIPS NOTHING. It writes only the cache under scratch/tr/, which is git-ignored and never leaves -# the runner except as an artifact. A run that fails leaves the previous cache intact. -# -# It also PURGES: raw deeds past the 35-day retention (ADR-0033 decision 5) are deleted at the end of -# every run, including one a 429 cut short. Retention is not the same knob as freshness — --max-age-days -# decides what gets re-requested, --retention-days decides what stops being kept — and the default is -# the ADR's 35 days, so the rail holds even when the workflow passes neither. -# -# Pace and refusals live in scripts/tr/fetch-deeds.mjs and are not configurable here on purpose: -# 1 request / 3 s, sequential, a closed candidate set, and a 429 ends the run (exit 2) without marking -# anything. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the limiter is the -# operator's only way to state a rate preference, so we do not tune around it. - -on: - schedule: - # 02:00 UTC on the 1st. Monthly, per #279 §9 — a deed changes rarely, and every extra pass is - # ~400 requests against somebody else's service for almost no new information. - - cron: '0 2 1 * *' - workflow_dispatch: - inputs: - max_age_days: - description: Re-fetch deeds older than this many days (default 30). Use a large value to force a full refresh. - type: string - default: '30' - limit: - description: Stop after this many lookups (blank = all pending). Bound a first run while watching for a 429. - type: string - default: '' - -concurrency: - # Never two crawls at once: they would double the request rate against the register, which is the one - # thing the pacing exists to prevent. Queue rather than cancel — a cancelled crawl wastes its progress. - group: tr-cache-refresh - cancel-in-progress: false - -permissions: - contents: read - -jobs: - refresh: - runs-on: ubuntu-latest - # ~400 candidates at 1 per 3 s is ~20 minutes; the ceiling allows retries and a slower day without - # letting a hung run sit for the 6-hour default. - timeout-minutes: 60 - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: 24 - cache: pnpm - - # The cache is the whole point of the workflow: without restoring it, every run would re-fetch - # every deed and turn a 400-request refresh into a 400-request full crawl each month. - - name: Restore the deed cache - id: cache - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.2 - with: - path: scratch/tr - key: tr-deed-cache-${{ github.run_id }} - restore-keys: | - tr-deed-cache- - - - run: pnpm install --frozen-lockfile - - # The closed candidate set. It is produced by the decision run (load.mjs writes it), so a cache - # refresh can never invent an ЕИК to look up — it can only refresh what the corpus already - # resolved. Absent on a first run, in which case there is nothing to refresh yet. - - name: Check for a candidate list - id: candidates - run: | - if [ -s scratch/cacbg/staging/candidate-eiks.txt ]; then - echo "count=$(wc -l < scratch/cacbg/staging/candidate-eiks.txt)" >> "$GITHUB_OUTPUT" - echo "ok=true" >> "$GITHUB_OUTPUT" - else - echo "No candidate-eiks.txt — run the decision workflow first. Nothing to refresh." - echo "ok=false" >> "$GITHUB_OUTPUT" - fi - - - name: Refresh deeds - if: steps.candidates.outputs.ok == 'true' - env: - MAX_AGE: ${{ inputs.max_age_days || '30' }} - LIMIT: ${{ inputs.limit }} - run: | - set -euo pipefail - ARGS=(--eiks-file scratch/cacbg/staging/candidate-eiks.txt --max-age-days "$MAX_AGE") - [ -n "$LIMIT" ] && ARGS+=(--limit "$LIMIT") - set +e - node scripts/tr/fetch-deeds.mjs "${ARGS[@]}" - code=$? - set -e - # Exit 2 is the rate limiter, and it is NOT a build failure: the run stopped politely, marked - # nothing, and the partial cache is valid and resumable. Failing the workflow here would train - # everyone to ignore a red X on the one signal that means "the register asked us to stop". - if [ "$code" -eq 2 ]; then - echo "::warning::The register rate-limited us; the run stopped and the cache is resumable." - exit 0 - fi - exit "$code" - - - name: Publish the cache index as an artifact (no raw deeds — they carry third-party data) - if: always() && steps.candidates.outputs.ok == 'true' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: tr-cache-index - # The SQLite index only. scratch/tr/deeds/*.json holds owner and manager names and company - # addresses (ADR-0033 decision 5) and must not leave the runner. - path: scratch/tr/tr-cache.sqlite - retention-days: 7 From 9555fbfd066127b35a152280529c7780edf6413b Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 14:14:08 +0300 Subject: [PATCH 29/69] test(db): type the seal-gate fixture arrays explicitly `as const` narrowed the WITHHELD tuples to literal unions, so appending the no-seal and positive-control rows failed typecheck. --- packages/db/src/related-persons-sql.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index 380949a0..512da590 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -551,9 +551,12 @@ describe('свързани-лица SQL (real SQLite)', () => { ] as const; function seedRungs(dbPath: string): void { - const people = [...WITHHELD.map(([, id, name]) => [id, name] as const)]; - people.push(['person:none', 'Безпечатен Тестов'], ['person:ok', 'Потвърден Тестов']); - const links = [...WITHHELD.map(([, id]) => id), 'person:none', 'person:ok']; + const people: [string, string][] = [ + ...WITHHELD.map(([, id, name]) => [id, name] as [string, string]), + ['person:none', 'Безпечатен Тестов'], + ['person:ok', 'Потвърден Тестов'], + ]; + const links: string[] = [...WITHHELD.map(([, id]) => id), 'person:none', 'person:ok']; sqlite( dbPath, `INSERT INTO persons (id, name) VALUES ${people.map(([id, n]) => `('${id}','${n}')`).join(',')}; From 6265854db74d3209e2698da089b70bf4ffed3018 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 7 Aug 2026 20:51:39 +0000 Subject: [PATCH 30/69] =?UTF-8?q?fix(ingest):=20=D0=B7=D0=B0=D0=BF=D0=B0?= =?UTF-8?q?=D0=B7=D0=B8=20=D0=BE=D1=82=D1=80=D0=B8=D1=86=D0=B0=D1=82=D0=B5?= =?UTF-8?q?=D0=BB=D0=BD=D0=B0=D1=82=D0=B0=20=D1=80=D0=B0=D0=B7=D0=BB=D0=B8?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=BF=D1=80=D0=B8=20=D0=B0=D0=BD=D0=B5=D0=BA?= =?UTF-8?q?=D1=81,=20=D0=BA=D0=BE=D0=B9=D1=82=D0=BE=20=D0=BD=D0=B0=D0=BC?= =?UTF-8?q?=D0=B0=D0=BB=D1=8F=D0=B2=D0=B0=20=D0=B4=D0=BE=D0=B3=D0=BE=D0=B2?= =?UTF-8?q?=D0=BE=D1=80=D0=B0=20(#285)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit toReal отхвърля всяко отрицателно число, а value_delta се пълнеше през него, така че всеки анекс, който намалява стойността, губеше разликата си. На staging: 0 отрицателни value_delta от 30 425 реда, при 916 реда с value_after < value_before и NULL разлика - и нула в обратната посока. Източникът наистина ги публикува: в open-data-2026-03-05 договор 212221 има contractValueDifference "-345,16" (24837,96 -> 24492,80). Отделен coercion (real_signed) само за това поле - останалите REAL колони са големини, където минусът значи повреден вход, и там toReal остава както е. Co-authored-by: t --- packages/ingest/src/base.test.ts | 39 ++++++++++++++++++++++++++++++++ packages/ingest/src/base.ts | 19 ++++++++++++++-- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/packages/ingest/src/base.test.ts b/packages/ingest/src/base.test.ts index 48f5f3e5..d4d89b65 100644 --- a/packages/ingest/src/base.test.ts +++ b/packages/ingest/src/base.test.ts @@ -12,6 +12,7 @@ import { toInt, toPeriodDate, toReal, + toSignedReal, } from './base'; const FIXED_NOW = new Date('2026-06-11T12:00:00Z'); @@ -111,6 +112,44 @@ describe('base EOP mapper', () => { expect(toPeriodDate('2025-06-01', FIXED_NOW)).toBe('2025-06-01'); expect(baseSqlLiteral('tenders', 'end_date', '2043-12-31')).toBe("'2043-12-31'"); }); + + // An annex can REDUCE a contract, and ЦАИС ЕОП publishes that as a negative + // contractValueDifference. Coercing it with toReal dropped the sign - and with it the whole row's + // delta - so every value-reducing annex silently lost its recorded change. + it('keeps a value-reducing annex delta negative instead of nulling it', () => { + const row = mapBaseRecord( + 'annexes', + { + uniqueProcurementNumber: '00224-2025-0005', + contractNumber: '212221', + publicationDate: '05.03.2026', + lastContractValue: '24837,96', + currentContractValue: '24492,80', + contractValueDifference: '-345,16', + }, + { day: '2026-03-05', fetchedAt: '2026-03-05T00:00:00Z' }, + ); + + expect(row?.value_before).toBe(24837.96); + expect(row?.value_after).toBe(24492.8); + expect(row?.value_delta).toBe(-345.16); + // The literal must stay a bare number, not a quoted string, or the staging INSERT changes type. + expect(baseSqlLiteral('annexes', 'value_delta', row?.value_delta)).toBe('-345.16'); + }); + + it('coerces signed reals without loosening the magnitude-only fields', () => { + expect(toSignedReal('-345,16')).toBe(-345.16); + expect(toSignedReal('-1 234,56')).toBe(-1234.56); + expect(toSignedReal('5833333,33')).toBe(5833333.33); + expect(toSignedReal('0')).toBe(0); + expect(toSignedReal(null)).toBeNull(); + expect(toSignedReal('')).toBeNull(); + expect(toSignedReal('--5')).toBeNull(); + expect(toSignedReal('-abc')).toBeNull(); + expect(toSignedReal(-(MAX_PLAUSIBLE_VALUE + 1))).toBeNull(); + // Magnitude fields keep rejecting a minus sign: there a negative means corrupt input. + expect(toReal('-345,16')).toBeNull(); + }); }); describe('offline SQL literal hardening', () => { diff --git a/packages/ingest/src/base.ts b/packages/ingest/src/base.ts index a2c43c01..fea2b5ca 100644 --- a/packages/ingest/src/base.ts +++ b/packages/ingest/src/base.ts @@ -7,6 +7,7 @@ export type BaseCoercionKind = | 'real' | 'bool' | 'date' + | 'real_signed' | 'secured_inverse' | 'variants_enum'; export type BaseStagingValue = string | number | null; @@ -94,6 +95,19 @@ export function toReal(v: unknown): number | null { return Number.isFinite(n) && n >= 0 && n <= MAX_PLAUSIBLE_VALUE ? n : null; } +// contractValueDifference is the one published figure that is legitimately negative - an annex can +// reduce a contract, and ЦАИС ЕОП publishes that as e.g. "-345,16". Every other REAL column here is a +// magnitude (values, estimates, subcontracting), where a minus sign means corrupt input, so toReal +// keeps rejecting negatives rather than being loosened for all of them. +export function toSignedReal(v: unknown): number | null { + const s = clean(v); + if (s === null) return null; + const compact = s.replace(/\s/g, ''); + if (!compact.startsWith('-')) return toReal(compact); + const magnitude = toReal(compact.slice(1)); + return magnitude === null ? null : -magnitude; +} + export function toBool(v: unknown): number | null { const s = clean(v); if (s === null) return null; @@ -131,6 +145,7 @@ function toVariants(v: unknown): number | null { export function coerce(kind: BaseCoercionKind, v: unknown): BaseStagingValue { if (kind === 'int') return toInt(v); if (kind === 'real') return toReal(v); + if (kind === 'real_signed') return toSignedReal(v); if (kind === 'bool') return toBool(v); if (kind === 'date') return toISODate(v); if (kind === 'secured_inverse') return toSecuredFinancing(v); @@ -297,7 +312,7 @@ export const BASE_CATEGORIES: Record = { field('contract_date', 'contractDate', 'date'), field('value_before', 'lastContractValue', 'real'), field('value_after', 'currentContractValue', 'real'), - field('value_delta', 'contractValueDifference', 'real'), + field('value_delta', 'contractValueDifference', 'real_signed'), field('currency', 'contractCurrency', 'text'), field('contract_subject', 'contractSubject', 'text'), field('awarded_to_group', 'awardedToGroup', 'bool'), @@ -432,7 +447,7 @@ export function baseSqlLiteral( ): string { if (value === null || value === undefined) return 'NULL'; const kind = baseColumnKind(cat, column); - if (['int', 'real', 'bool', 'secured_inverse', 'variants_enum'].includes(kind)) { + if (['int', 'real', 'real_signed', 'bool', 'secured_inverse', 'variants_enum'].includes(kind)) { return String(value); } return escapeSqlText(String(value)); From 37e12588879eacb00bfd6287d621de197b23f770 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 7 Aug 2026 23:21:57 +0000 Subject: [PATCH 31/69] =?UTF-8?q?build(deps):=20=D0=B2=D0=B4=D0=B8=D0=B3?= =?UTF-8?q?=D0=BD=D0=B8=20nanoid=20=D0=B4=D0=BE=203.3.17+=20(GHSA-2v37-7h3?= =?UTF-8?q?g-55p8)=20(#291)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ново предупреждение с висока тежест (8.2) обяви CI-то червено на всеки отворен PR. nanoid идва през postcss, който вече е закачен, но 8.5.24 сам продължава да решава 3.3.16 - затова иска изрично закачане. Само за времето на сглобяване, през vite/vitest; не стига до Worker-а. Co-authored-by: t --- pnpm-lock.yaml | 9 +++++---- pnpm-workspace.yaml | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 304ec682..a8037bfb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,7 @@ overrides: sharp: ^0.35.3 postcss: ^8.5.18 valibot: ^1.4.2 + nanoid: ^3.3.17 importers: @@ -1643,8 +1644,8 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - nanoid@3.3.16: - resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} + nanoid@3.3.18: + resolution: {integrity: sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -3369,7 +3370,7 @@ snapshots: ms@2.1.3: {} - nanoid@3.3.16: {} + nanoid@3.3.18: {} node-releases@2.0.45: {} @@ -3406,7 +3407,7 @@ snapshots: postcss@8.5.24: dependencies: - nanoid: 3.3.16 + nanoid: 3.3.18 picocolors: 1.1.1 source-map-js: 1.2.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 200559f1..87ca4b8e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -38,6 +38,10 @@ overrides: # valibot <1.4.2 — GHSA-5qjj-4xww-7phc (MEDIUM); transitive, build-time only, never # ships to the Worker. valibot: '^1.4.2' + # nanoid <3.3.17 — GHSA-2v37-7h3g-55p8 (HIGH, 8.2); pulled in by postcss (itself pinned + # above), so postcss 8.5.24 keeps resolving 3.3.16 on its own. Build-time + # only, via vite/vitest — never ships to the Worker. + nanoid: '^3.3.17' onlyBuiltDependencies: - esbuild From 17f94722a8d47230d1961eeaa227639973d08758 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 7 Aug 2026 23:25:39 +0000 Subject: [PATCH 32/69] =?UTF-8?q?fix(scripts):=20=D0=BF=D0=B5=D1=81=D1=82?= =?UTF-8?q?=D0=B5=D0=BB=D0=B8=D0=B2=D0=BE=20=D0=BA=D0=B0=D1=87=D0=B2=D0=B0?= =?UTF-8?q?=D0=BD=D0=B5=20=D0=BD=D0=B0=20=D1=81=D0=B2=D1=8A=D1=80=D0=B7?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=20=D0=BB=D0=B8=D1=86=D0=B0=20=D0=B8=20=D1=81?= =?UTF-8?q?=D0=B2=D0=B5=D1=80=D0=BA=D0=B0=20=D1=81=D0=BB=D0=B5=D0=B4=20?= =?UTF-8?q?=D0=BD=D0=B5=D0=B3=D0=BE=20(#283)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(scripts): pace the свързани-лица ship and verify what landed The first full-corpus ship wrote each table as ONE `d1 execute --file` bulk import. On sigma-stage-green that came to 516k rows written in a single hour with a p90 batch time of 18.2s — against 2.9-4.5k rows and 19-450ms in a normal cron hour, two orders of magnitude heavier. For several hours afterwards the database returned "internal error" on every query, including its own metadata endpoint, and the whole staging site served 500. It recovered on its own with all data intact. The Cloudflare-side mechanism is not provable from outside, and the 6-hourly ETL cron is ruled out (its Workflow instances all completed; the one immediately before the outage ran 30 steps in 92s with a single attempt each). So this is a deliberate defensive bound, not a proven fix: cap what one request carries (25 statements = 10k rows) and leave 500ms between requests, so a re-seed is a series of ordinary writes instead of one shock. Both are flags — --max-statements-per-request, --pace-ms. Second, verify. The ship wipes and then writes over several requests with no cross-request transaction, so a failure part-way leaves the target holding some tables and not others — and nothing noticed: the run exited 0 and the surface rendered as a smaller corpus. That is glaring next to the READ path, where the EOP hydrate already refuses on a local↔remote row-count mismatch. Read the counts back and fail on any drift, naming the numbers; a read-back that cannot answer fails too, since a verification step that cannot verify must not pass. The I/O boundary of the paced loop is injected (applyTableChunks) so the WIRING is covered, not just the pure chunker — "one request per table" is exactly the shape a refactor drifts back to. Mutation-checked: restoring the single request, removing the pacing gap, or disabling the drift check each fails its own tests. --emit still writes one file per table: those are applied by hand and numbered fragments would only add ordering rope to a manual run. * test(scripts): покрий свързването на качването, не само чистите функции Ревюто извади, че две от трите обещани гаранции се промъкват: връщането на call site-а към една заявка на таблица и изтриването на сверката оставяха целия набор зелен (19/19). Тестваха се помощните функции, а не пътят, който ги вика. runShip поема целия рушащ път - изтриване, паузирани заявки по парчета, сверка - зад инжектирани граници, и тестовете го карат наистина. Четирите мутации вече падат: една заявка на таблица, махната сверка, махната пауза, брояч по таблица. Плюс два реални дефекта, които това извади: - броячът за паузите се въртеше по таблица, тъй че всеки преход между таблици оставаше непаузиран - включително изтриване -> първо вмъкване, най-рушащият момент в пробега; - Number(null) е 0, тъй че нечислова клетка в четенето минаваше за "таблицата е празна"; сега всичко нечислово е NaN и сверката пада. * style(scripts): форматиране по prettier * test(scripts): покрий самия път на качване, а не поредната функция Ревюто на разликата показа, че предишната поправка е преместила дупката, а не я е затворила: заобикалянето на runShip директно в main() пак оставяше целия набор зелен (22/22). Проверих го сам - същата мутация минаваше. Всяка инжектирана граница оставя най-външното извикване непокрито. Затова този тест кара истинския скрипт като отделен процес, с подставен wrangler най-отпред в PATH, и гледа какво реално се иска от него: изтриването е първата заявка, таблица с 801 реда пристига като три номерирани заявки, и се пуска точно едно четене обратно, което покрива всичките пет таблици. Втори тест доказва, че къса цел проваля пробега. Мутацията, която досега се промъкваше, вече проваля и двата. * test(scripts): проверявай какво е в целта, не какви заявки са пратени Ревюто разби първия вариант на теста през край: той твърдеше имена на файлове и брой заявки, тоест мутации, които корумпират самия SQL, минаваха. Проверих ги сам - празен товар, изтриване без ефект, обърнат ред на таблиците, и пращане на всичко към sigma-green по remote: всички зелени. Подставеният wrangler вече не е замазка. Прилага всеки --file върху истинска sqlite цел с включени външни ключове и отговаря на всеки --command от нея. Твърденията са за това какво целта наистина държи, плюс че всяка заявка именува декларираната база и остава local. Целта тръгва с оставени стари редове, иначе изтриване без ефект не се различава от вярно. Една таблица е нарочно празна - това е единствената форма, в която Number(null) === 0 би минало за „таблицата е празна". Размерът на фикстурата вече се извежда от MAX_BATCH_ROWS, а не е магическо число, което вдигането на прага би обявило за провал. Пътьом тестът намери истински бъг: readShippedCounts режеше от първата [, а известие от вида „▲ [WARNING]" съдържа скоба - тоест здраво качване се проваляше с фалшива грешка при сверката. Вече се пробва всяка скоба. Петте мутации от ревюто вече падат. * test(scripts): покрий незащитените предпазители и производствените стойности Ревюто изпълни мутации по всеки предпазител в скрипта; всички минаваха: - махането на пода за качване; - махането на проверката коя база е разрешена; - обръщането на сверката да се проваля отворено (catch да връща очакваното вместо празно); - махането на предупредителния заглавен блок от изнесеното изтриване, и изобщо целият клон --emit; - и най-същественото: чупенето на парчета го предизвиквах с флаг в теста, тоест производствената стойност можеше да се вдигне до безкрайност със зелен набор. Сега главният случай върви на подразбиращите се стойности с корпус един ред над това, което една заявка носи, а фикстурата се извежда от двете константи. Останалите случаи ползват малка фикстура, за да не плаща целият набор за 10 000 реда - 25 секунди общо. Плюс тестове, че всеки отказ идва ПРЕДИ първата рушаща заявка (четенето за разрешаване на идентификатора не се брои). * test(scripts): назови зависимостта от sqlite3 и махни забравен файл Тестът през край кара истинска база през двоичния `sqlite3` - точно затова твърденията му значат нещо. Досега това никъде не беше казано, а коментарът в работния поток твърдеше обратното. Сега проверката е отпред и вали ясно вместо да остави неясно ENOENT от произволен тест. Нарочно вали, а не прескача: прескачането връща набора точно в състоянието, през което мутациите минаваха. Махнат е и файлът „ПАТЪРН" - отпадък от сгрешената мутационна проверка, влязъл в 9a1d64ba по невнимание. --------- Co-authored-by: t --- .github/workflows/scripts-test.yml | 4 +- scripts/ship-e2e.test.mjs | 358 ++++++++++++++++++++++++++ scripts/ship-related-persons.mjs | 214 ++++++++++++++- scripts/ship-related-persons.test.mjs | 126 +++++++++ 4 files changed, 687 insertions(+), 15 deletions(-) create mode 100644 scripts/ship-e2e.test.mjs diff --git a/.github/workflows/scripts-test.yml b/.github/workflows/scripts-test.yml index 7ee3ef4c..6ce4fa7d 100644 --- a/.github/workflows/scripts-test.yml +++ b/.github/workflows/scripts-test.yml @@ -25,7 +25,9 @@ jobs: with: node-version: 22 # node:test only — the scripts under test import nothing outside node built-ins and each other, - # so no `pnpm install` is required. + # so no `pnpm install` is required. The one external dependency is the `sqlite3` binary, which + # ship-e2e.test.mjs drives as the real target its fake wrangler writes into; it ships with the + # ubuntu-latest image, and the test fails loudly (never skips) if it is ever absent. - run: node --test scripts/*.test.mjs # свързани-лица CACBG pipeline tests (parse/classify/load/audit/tr-census/extract). These use diff --git a/scripts/ship-e2e.test.mjs b/scripts/ship-e2e.test.mjs new file mode 100644 index 00000000..af27ccf1 --- /dev/null +++ b/scripts/ship-e2e.test.mjs @@ -0,0 +1,358 @@ +// End-to-end coverage of the SHIP PATH ITSELF — the one thing unit tests of the helpers cannot give. +// +// Three times a refactor of this script silently dropped a guarantee while the suite stayed green: +// when main() called applyTableChunks, again after that moved into runShip(), and again when a first +// cut of THIS file asserted only on request filenames and counts — so mutations that shipped an empty +// payload, a wipe that deleted nothing, or every request to a PRODUCTION slot all passed. +// +// The fix is to stop trusting a stub: the fake `wrangler` here APPLIES each --file to a real sqlite +// target and answers each --command FROM that target. The assertions are then about what the target +// actually holds, which no amount of correct-looking request plumbing can fake. +// +// Run: node --test scripts/ship-e2e.test.mjs +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + MAX_BATCH_ROWS, + MAX_STATEMENTS_PER_REQUEST, + PACE_MS, + TABLES, +} from './ship-related-persons.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(HERE, '..'); +const SCRIPT = resolve(HERE, 'ship-related-persons.mjs'); +const MIG = resolve(ROOT, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const D1_NAME = 'sigma-test-local'; + +// Derived from BOTH production constants, and the run below does NOT override either: one row past +// what a single default request can carry, so chunking is exercised at the shipped settings. Forcing +// it with --max-statements-per-request=1 (the first cut of this file) left the real constant free to +// be retuned to infinity with the suite still green. +const LINKS = MAX_STATEMENTS_PER_REQUEST * MAX_BATCH_ROWS + 1; +const EXPECTED_CHUNKS = Math.ceil(LINKS / MAX_BATCH_ROWS / MAX_STATEMENTS_PER_REQUEST); +// The failure-mode tests below do not need the full-size corpus — they force chunking with a flag and +// keep the fixture small, so only the one test that constrains the defaults pays for 10k rows. +const LINKS_SMALL = MAX_BATCH_ROWS * 2 + 1; + +const sqlite = (db, input) => execFileSync('sqlite3', ['-bail', db], { input, stdio: 'pipe' }); + +// Unlike the other scripts/*.test.mjs this one needs the `sqlite3` BINARY (present on ubuntu-latest +// and in the devcontainer): the fake wrangler applies real SQL to a real database, which is the whole +// reason these assertions mean anything. Say so up front — a missing binary would otherwise surface as +// an opaque ENOENT from whichever test happened to run first. Failing, never skipping: a skip here +// silently returns the suite to the state where mutations walked through it. +if (spawnSync('sqlite3', ['-version'], { stdio: 'ignore' }).error) { + throw new Error('scripts/ship-e2e.test.mjs requires the sqlite3 binary on PATH'); +} + +/** The five served tables plus the two FK parents they reference. */ +const SCHEMA = `PRAGMA foreign_keys=ON; +CREATE TABLE bidders(id TEXT PRIMARY KEY); +CREATE TABLE authorities(id TEXT PRIMARY KEY); +.read ${MIG} +INSERT INTO bidders(id) VALUES('eik:1'); +INSERT INTO authorities(id) VALUES('auth:1');`; + +const corpus = (links) => ` +INSERT INTO persons(id,name) VALUES('p1','П Тест'); +INSERT INTO declarations(id,person_id,xml_file,folder_year,template,source_url) VALUES('d1','p1','x.xml','2024','assets','u'); +INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VALUES('di1','d1','E','e','shares'); +${Array.from( + { length: links }, + (_, i) => + `INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('il${i}','p1|${i}','p1','eik:1','1','e','v1','B_distinctive','owns','published');`, +).join('\n')} +`; +// interest_link_authorities is deliberately left EMPTY above: its expected count is 0, which is the +// only shape where `Number(null) === 0` would let an unanswered read-back pass for "table is empty". + +const STALE = ` +INSERT INTO persons(id,name) VALUES('stale','Стар запис'); +INSERT INTO declarations(id,person_id,xml_file,folder_year,template,source_url) VALUES('sd','stale','s.xml','2019','assets','u'); +INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VALUES('sdi','sd','S','s','shares'); +INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('sil','stale|0','stale','eik:1','1','s','v0','B_distinctive','owns','published'); +INSERT INTO interest_link_authorities(link_key,authority_id,authority_name) VALUES('stale|0','auth:1','A');`; + +const EXPECTED_ROWS = { + persons: 1, + declarations: 1, + declared_interests: 1, + interest_links: LINKS, + interest_link_authorities: 0, +}; + +/** + * A `wrangler` that touches no network but is otherwise faithful: it records the FULL argv, applies + * every --file to the target sqlite DB with foreign keys ON (so a wrong wipe order fails exactly as + * D1 would), and answers every --command from that same DB. Notices go to stderr and pure JSON to + * stdout, mirroring real `wrangler --json`. + * + * SHIP_FAKE_SKIP — drop one --file by name, to simulate a request that never landed. + * SHIP_FAKE_NULLN — answer the read-back with a non-numeric count, to exercise the fail-closed guard. + * SHIP_FAKE_NOISE — emit a `[WARNING]`-shaped line on stdout before the JSON. + * SHIP_FAKE_READFAIL — make the read-back call itself fail, so the catch path is exercised. + */ +function fakeWrangler(dir) { + const bin = join(dir, 'bin'); + mkdirSync(bin, { recursive: true }); + const log = join(dir, 'calls.jsonl'); + const target = join(dir, 'target.sqlite'); + // Seeded with STALE rows on purpose: against an empty target a wipe that deletes nothing is + // indistinguishable from a correct one, and that mutation escaped the first cut of this test. + sqlite(target, SCHEMA + STALE); + const exe = join(bin, 'wrangler'); + writeFileSync( + exe, + `#!/usr/bin/env node +import { appendFileSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +const argv = process.argv.slice(2); +const at = (f) => (argv.indexOf(f) >= 0 ? argv[argv.indexOf(f) + 1] : null); +const file = at('--file'); +const command = at('--command'); +const TARGET = ${JSON.stringify(target)}; +appendFileSync(${JSON.stringify(log)}, JSON.stringify({ argv, file: file && file.split('/').pop() }) + '\\n'); +const run = (input) => + execFileSync('sqlite3', ['-bail', TARGET], { input: 'PRAGMA foreign_keys=ON;\\n' + input, encoding: 'utf8' }); +try { + if (file) { + const skip = process.env.SHIP_FAKE_SKIP; + if (!skip || !file.endsWith(skip)) run('.read ' + file); + } else if (command) { + if (process.env.SHIP_FAKE_READFAIL) { process.stderr.write('read-back exploded'); process.exit(1); } + const rows = JSON.parse(run('.mode json\\n' + command) || '[]'); + const shaped = process.env.SHIP_FAKE_NULLN + ? rows.map((r) => (r.t === process.env.SHIP_FAKE_NULLN ? { ...r, n: null } : r)) + : rows; + if (process.env.SHIP_FAKE_NOISE) process.stdout.write('▲ [WARNING] Processing wrangler.jsonc\\n'); + process.stdout.write(JSON.stringify([{ results: shaped, success: true }])); + } +} catch (err) { + process.stderr.write(String(err.stderr || err.message)); + process.exit(1); +} +`, + { mode: 0o755 }, + ); + chmodSync(exe, 0o755); + // package.json so the extensionless fake is unambiguously ESM wherever os.tmpdir() lives. + writeFileSync(join(bin, 'package.json'), '{"type":"module"}'); + return { + bin, + target, + calls: () => + readFileSync(log, 'utf8') + .trim() + .split('\n') + .filter(Boolean) + .map((l) => JSON.parse(l)), + count: (t) => + Number(execFileSync('sqlite3', [target, `SELECT COUNT(*) FROM ${t};`]).toString()), + }; +} + +function runShip( + dir, + { + env = {}, + links = LINKS_SMALL, + forceChunks = true, + minLinks = 1, + remote = false, + yes = false, + emit = null, + } = {}, +) { + const work = join(dir, 'work.sqlite'); + sqlite(work, SCHEMA + corpus(links)); + const fake = fakeWrangler(dir); + const res = spawnSync( + process.execPath, + [ + SCRIPT, + `--work-db=${work}`, + ...(remote ? ['--remote'] : ['--local']), + ...(yes ? ['--yes'] : []), + ...(emit ? [`--emit=${emit}`] : []), + `--min-links=${minLinks}`, + // The pacing delay is always zeroed to keep the suite quick — it is covered by the runShip unit + // tests. Whether the REQUEST SIZE is overridden matters: the defaults-constraining test leaves + // it alone on purpose. + '--pace-ms=0', + ...(forceChunks ? ['--max-statements-per-request=1'] : []), + ], + { + cwd: ROOT, + encoding: 'utf8', + env: { + ...process.env, + ...env, + SIGMA_D1_NAME: D1_NAME, + PATH: `${fake.bin}:${process.env.PATH}`, + }, + }, + ); + return { res, fake }; +} + +test('a real ship run leaves the target holding exactly what the work DB held', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + const { res, fake } = runShip(dir, { links: LINKS, forceChunks: false }); + assert.equal(res.status, 0, `ship failed:\n${res.stderr}`); + + // The assertion that no amount of correct-looking plumbing can fake: the SQL really applied. + for (const [table, n] of Object.entries({ ...EXPECTED_ROWS, interest_links: LINKS })) + assert.equal(fake.count(table), n, `${table} did not land`); + + const calls = fake.calls(); + // Every request must name the declared DB and stay local — a mutation that retargets a production + // slot is the single highest-consequence regression this script can suffer. + for (const c of calls) { + assert.deepEqual(c.argv.slice(0, 3), ['d1', 'execute', D1_NAME]); + assert.ok(c.argv.includes('--local'), `request escaped --local: ${c.argv.join(' ')}`); + assert.ok(!c.argv.includes('--remote'), `request went remote: ${c.argv.join(' ')}`); + } + + const applies = calls.filter((c) => c.file); + assert.match(applies[0].file, /^0_wipe\./, 'the wipe must be the first request'); + + // Chunking: a table past the batch budget must arrive as several CONTIGUOUSLY numbered requests. + const nums = applies + .map((c) => /^interest_links\.(\d+)\./.exec(c.file)) + .filter(Boolean) + .map((m) => Number(m[1])); + assert.ok( + nums.length >= 2, + `interest_links must be chunked, got ${JSON.stringify(applies.map((c) => c.file))}`, + ); + assert.equal(nums.length, EXPECTED_CHUNKS); + assert.deepEqual( + nums, + Array.from({ length: nums.length }, (_, i) => i + 1), + ); + + // The read-back must be the LAST thing the run does, and must count each table from that table. + const last = calls.at(-1); + assert.ok(last.argv.includes('--command'), 'the read-back must come after the inserts'); + const sql = last.argv[last.argv.indexOf('--command') + 1]; + for (const table of TABLES) + assert.match( + sql, + new RegExp(`COUNT\\(\\*\\) AS n FROM "${table}"`), + `${table} not really counted`, + ); +}); + +test('a request that never landed fails the run', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-short-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + const { res } = runShip(dir, { env: { SHIP_FAKE_SKIP: 'interest_links.2.sql' } }); + assert.notEqual(res.status, 0, 'a short target must fail the run'); + assert.match(res.stderr, /ship verification FAILED/); + assert.match(res.stderr, /interest_links: shipped \d+, target has \d+/); +}); + +test('a read-back that answers with a non-number fails closed', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-nan-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + // `Number(null)` is 0, which would read as "the table is empty" and quietly pass. + const { res } = runShip(dir, { env: { SHIP_FAKE_NULLN: 'interest_link_authorities' } }); + assert.notEqual(res.status, 0, 'an unanswered count must fail the run'); + assert.match(res.stderr, /ship verification FAILED/); +}); + +test('a bracketed notice on stdout does not corrupt the read-back', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-noise-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + // The script slices from the first '[' to skip wrangler's preamble — a `[WARNING]` line would + // send that slice to the wrong offset. Real wrangler --json keeps notices on stderr, but the + // defensive slice exists precisely because that has not always been true. + const { res } = runShip(dir, { env: { SHIP_FAKE_NOISE: '1' } }); + assert.equal(res.status, 0, `a stdout notice broke the read-back:\n${res.stderr}`); +}); + +// The guards below all sit at UNPROTECTED call sites: deleting each one, or moving it after the +// destructive run, left the whole suite green. They are the last thing standing between a mistyped +// flag and a wiped production surface, so each gets an end-to-end test that also proves NO request +// was issued before the refusal. +// `d1 info` (the id resolution the authorization guard itself needs) is a read and is fine; what must +// never happen before a refusal is an `execute`, which is what carries the wipe. +const noWrites = (fake) => { + try { + return !fake.calls().some((c) => c.argv[0] === 'd1' && c.argv[1] === 'execute'); + } catch { + return true; // the log file is only created by the first invocation + } +}; + +test('an under-floor corpus refuses to wipe, before any request', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-floor-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + const { res, fake } = runShip(dir, { minLinks: LINKS_SMALL + 1 }); + assert.notEqual(res.status, 0); + assert.match(res.stderr, /refusing to ship/i); + assert.ok(noWrites(fake), 'the refusal must come before the wipe'); +}); + +test('a bare --remote refuses without --yes, before any request', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-remote-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + const { res, fake } = runShip(dir, { remote: true }); + assert.notEqual(res.status, 0); + assert.match(res.stderr, /--remote requires --yes/); + assert.ok(noWrites(fake), 'the refusal must come before the wipe'); +}); + +test('a --remote ship with no declared environment refuses, before any request', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-env-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + const { res, fake } = runShip(dir, { remote: true, yes: true, env: { SIGMA_SHIP_ENV: '' } }); + assert.notEqual(res.status, 0); + assert.match(res.stderr, /SIGMA_SHIP_ENV/); + assert.ok(noWrites(fake), 'the refusal must come before the wipe'); +}); + +test('a read-back that cannot answer at all fails the run', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-readfail-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + // „a verification step that cannot verify must not pass" — the catch returning {} is what makes + // that true, and returning the expectation instead would silently pass. + const { res } = runShip(dir, { env: { SHIP_FAKE_READFAIL: '1' } }); + assert.notEqual(res.status, 0); + assert.match(res.stderr, /ship verification FAILED/); + assert.match(res.stderr, /no answer/); +}); + +test('--emit writes a guarded wipe plus one file per table, and touches no database', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-emit-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + const out = join(dir, 'emitted'); + const { res, fake } = runShip(dir, { emit: out }); + assert.equal(res.status, 0, `emit failed:\n${res.stderr}`); + assert.ok(noWrites(fake), '--emit must not touch a database'); + + const wipe = readFileSync(join(out, '0_wipe.sql'), 'utf8'); + assert.match(wipe, /DESTRUCTIVE, UNGUARDED/, 'the emitted wipe must carry its warning header'); + for (const table of TABLES) assert.match(wipe, new RegExp(`DELETE FROM "${table}"`)); + for (const table of TABLES) { + const body = readFileSync(join(out, `${table}.sql`), 'utf8'); + if (table === 'interest_link_authorities') assert.equal(body, '', 'empty table, empty file'); + else assert.match(body, new RegExp(`INSERT INTO "${table}"`)); + } +}); diff --git a/scripts/ship-related-persons.mjs b/scripts/ship-related-persons.mjs index 54da33de..baf6646a 100644 --- a/scripts/ship-related-persons.mjs +++ b/scripts/ship-related-persons.mjs @@ -45,7 +45,81 @@ export function wipeSql() { return WIPE_ORDER.map((t) => `DELETE FROM ${sqlIdent(t)};`).join('\n') + '\n'; } const MAX_BATCH_BYTES = 90_000; -const MAX_BATCH_ROWS = 400; +export const MAX_BATCH_ROWS = 400; + +// One `d1 execute --file` per TABLE meant the whole table went up as a single bulk import: on the first +// full-corpus ship that was 516k rows written in one hour with a p90 batch time of 18.2s, two orders of +// magnitude above a normal cron hour (2.9-4.5k rows, 19-450ms) — after which the database returned +// „internal error" for hours, including on its own metadata endpoint, before recovering with all data +// intact. The Cloudflare-side mechanism is not provable from outside, so this is a deliberate defensive +// bound rather than a proven fix: cap how much one request carries and leave a gap between requests, so a +// re-seed is a series of ordinary writes instead of one shock. 25 × MAX_BATCH_ROWS = 10 000 rows/request. +export const MAX_STATEMENTS_PER_REQUEST = 25; +export const PACE_MS = 500; + +/** Group per-table INSERT statements into request-sized chunks. Pure — unit-tested. */ +export function chunkStatements(statements, maxPerRequest = MAX_STATEMENTS_PER_REQUEST) { + if (!Number.isInteger(maxPerRequest) || maxPerRequest < 1) + throw new Error( + `maxPerRequest must be a positive integer, got ${JSON.stringify(maxPerRequest)}`, + ); + const chunks = []; + for (let i = 0; i < statements.length; i += maxPerRequest) + chunks.push(statements.slice(i, i + maxPerRequest)); + return chunks; +} + +/** Block the (synchronous) ship loop without burning CPU. */ +const sleepSync = (ms) => { + if (ms > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +}; + +/** + * The whole destructive live path: wipe, then paced request-sized inserts per table, then the read-back + * check. Both guarantees live HERE, behind injected I/O (`apply`, `sleep`, `readCounts`), because both are + * one refactor away from silently vanishing — „one request per table" is exactly the shape this drifts back + * to, and a `if (!emit) assert…` line at a call site is exactly the kind of line that gets dropped. + * Testing the pure helpers alone did NOT catch either: reverting the call site and deleting the + * verification each left the whole suite green. Keep the orchestration itself covered. + * @returns {Record} rows shipped per table + */ +export function runShip({ + tables, + readTable, + wipeSql, + apply, + sleep, + readCounts, + maxStatements, + paceMs, +}) { + // ONE counter for the whole run, not one per table. Pacing per table left every table boundary + // unpaced — including wipe → first insert, which is the single most destructive transition here. + let requests = 0; + const applyPaced = (label, sql) => { + if (requests++) sleep(paceMs); // between requests only — never before the first + apply(label, sql); + }; + + applyPaced('0_wipe', wipeSql); + + const summary = {}; + for (const table of tables) { + const read = readTable(table); + if (!read) { + summary[table] = 'absent (skipped)'; + continue; + } + summary[table] = read.rowCount; + const chunks = chunkStatements(read.statements, maxStatements); + chunks.forEach((chunk, i) => + applyPaced(chunks.length > 1 ? `${table}.${i + 1}` : table, chunk.join('')), + ); + } + + assertShippedCounts(summary, readCounts(summary)); + return summary; +} // Supports --name=value, --name value, and bare --name (boolean). A --name whose next token is another // --flag (or absent) is a boolean; otherwise it consumes the next token as its value. @@ -107,6 +181,17 @@ export function parseMinLinks(raw) { return n; } +/** Shared shape check for the pacing flags: a bare `--flag` must not silently mean 1 (or 0). */ +function parseIntFlag(raw, name, min) { + if (raw === true) throw new Error(`--${name} requires a value, e.g. --${name}=25`); + const n = Number(raw); + if (!Number.isInteger(n) || n < min) + throw new Error(`--${name} must be an integer >= ${min}, got ${JSON.stringify(raw)}.`); + return n; +} +const parsePositiveInt = (raw, name) => parseIntFlag(raw, name, 1); +const parseNonNegativeInt = (raw, name) => parseIntFlag(raw, name, 0); + /** * The D1 name to ship to. A --remote write MUST name its target explicitly: this path DELETEs every * свързани-лица table before re-inserting, so a silent fallback on a remote run is unacceptable — an unset @@ -187,6 +272,54 @@ export function assertD1TargetAuthorized({ remote, shipEnv, d1Name, expectedId, /** Live lookup of the uuid Cloudflare maps `d1Name` to, via `wrangler d1 info --json`. Returns '' on any * failure (unknown name, network, parse) so assertD1TargetConsistent turns that into an explicit refusal. */ +/** + * Read the shipped tables' row counts back off the target, in one query. Returns {} when the read itself + * fails — `assertShippedCounts` then reports every table as unanswered and fails the run, which is the + * right way round: a verification step that cannot verify must not pass. + */ +/** First bracket that actually parses — notices may contain '[' of their own. Exported for the test. */ +export function parseWranglerJson(out) { + for (let i = out.indexOf('['); i >= 0; i = out.indexOf('[', i + 1)) { + try { + return JSON.parse(out.slice(i)); + } catch { + // not the payload — keep looking + } + } + return JSON.parse(out); // no usable bracket: let the original parse error surface +} + +function readShippedCounts(d1Name, remote, expected) { + const tables = Object.entries(expected) + .filter(([, n]) => typeof n === 'number') + .map(([t]) => t); + if (!tables.length) return {}; + const sql = tables + .map((t) => `SELECT ${sqlLiteral(t)} AS t, COUNT(*) AS n FROM ${sqlIdent(t)}`) + .join(' UNION ALL '); + try { + const out = execFileSync( + 'wrangler', + ['d1', 'execute', d1Name, remote ? '--remote' : '--local', '--json', '--command', sql], + { cwd: resolve('apps/web'), encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }, + ); + // wrangler prefixes its own notices before the JSON payload; take the document, not the whole + // stream. Slicing from the FIRST '[' is not enough — a notice can itself contain one („▲ + // [WARNING] Processing wrangler.jsonc"), which sliced to the wrong offset and turned a perfectly + // healthy ship into a verification failure. Try each candidate and keep the first that parses. + const parsed = parseWranglerJson(out); + const rows = (Array.isArray(parsed) ? parsed[0]?.results : parsed?.results) ?? []; + // Only a real number counts as an answer. `Number(null)` is 0, which would let a null-valued cell pass + // for „the table is empty"; anything non-numeric must land as NaN so assertShippedCounts fails closed. + return Object.fromEntries(rows.map((r) => [r.t, typeof r.n === 'number' ? r.n : Number.NaN])); + } catch (err) { + console.error( + `ship: could not read back row counts — ${err instanceof Error ? err.message : err}`, + ); + return {}; + } +} + function resolveD1Id(d1Name) { try { const out = execFileSync('wrangler', ['d1', 'info', d1Name, '--json'], { @@ -200,6 +333,34 @@ function resolveD1Id(d1Name) { } } +/** + * Compare what we meant to ship against what the target actually holds. Pure — unit-tested; the live read + * is injected as `readCounts`. + * + * The ship is a wipe followed by SEVERAL independent requests (no cross-request transaction), so a failure + * part-way leaves the target holding some tables and not others — and today nothing notices: the run exits 0 + * and the surface renders as a smaller corpus. That asymmetry is glaring next to the READ path, where the + * EOP hydrate already refuses to proceed on a local↔remote row-count mismatch. Close it on the destructive + * side too: any drift fails the run, loudly and with the numbers. + */ +export function assertShippedCounts(expected, actual) { + const drift = Object.entries(expected) + .filter(([, n]) => typeof n === 'number') + .map(([table, n]) => ({ table, expected: n, actual: actual[table] })) + .filter(({ expected: e, actual: a }) => a !== e); + if (drift.length) + throw new Error( + 'ship verification FAILED — the target does not hold what was shipped:\n' + + drift + .map( + ({ table, expected: e, actual: a }) => + ` ${table}: shipped ${e}, target has ${a === undefined ? 'no answer' : a}`, + ) + .join('\n') + + '\nThe wipe already ran, so the surface is now partial. Re-run the ship.', + ); +} + /** Batched multi-row INSERTs for one table, bounded by D1's statement size. Pure — unit-tested. */ export function insertStatements(table, cols, rows) { if (!cols.length || !rows.length) return []; @@ -238,6 +399,11 @@ function main() { const remote = Boolean(arg('remote', false)); const d1Name = resolveD1Name({ remote, envName: process.env.SIGMA_D1_NAME }); const minLinks = parseMinLinks(arg('min-links', 50)); + const maxStatements = parsePositiveInt( + arg('max-statements-per-request', MAX_STATEMENTS_PER_REQUEST), + 'max-statements-per-request', + ); + const paceMs = parseNonNegativeInt(arg('pace-ms', PACE_MS), 'pace-ms'); if (remote && !arg('yes', false)) throw new Error('--remote requires --yes (guards against an accidental prod write)'); @@ -298,26 +464,46 @@ function main() { '-- ⚠ DESTRUCTIVE, UNGUARDED: this wipe was emitted with --emit and did NOT pass the live D1\n' + '-- target-authorization check (SIGMA_SHIP_ENV allowlist + name↔SIGMA_D1_ID). If you apply it by hand,\n' + '-- YOU are responsible for confirming the target D1 is the intended one before running it.\n'; - if (emit) writeFileSync(resolve(emit, '0_wipe.sql'), EMIT_WIPE_HEADER + wipeSql()); - else applyFile('0_wipe', wipeSql()); + // One read of a source table: null when the table is absent from the work DB. + const readTable = (table) => { + const cols = sqliteJson(`PRAGMA table_info(${sqlIdent(table)})`).map((r) => r.name); + if (!cols.length) return null; + const rows = sqliteJson(`SELECT * FROM ${sqlIdent(table)}`); + return { rowCount: rows.length, statements: insertStatements(table, cols, rows) }; + }; - const summary = {}; + let summary = {}; try { - for (const table of TABLES) { - const cols = sqliteJson(`PRAGMA table_info(${sqlIdent(table)})`).map((r) => r.name); - if (!cols.length) { - summary[table] = 'absent (skipped)'; - continue; + if (emit) { + // --emit keeps ONE file per table: those are applied by hand, and numbered fragments would only add + // ordering rope to a manual run. Nothing is written to a DB, so there is nothing to pace or verify — + // the header on 0_wipe.sql puts the target check on whoever applies them. + writeFileSync(resolve(emit, '0_wipe.sql'), EMIT_WIPE_HEADER + wipeSql()); + for (const table of TABLES) { + const read = readTable(table); + if (!read) { + summary[table] = 'absent (skipped)'; + continue; + } + summary[table] = read.rowCount; + writeFileSync(resolve(emit, `${table}.sql`), read.statements.join('')); } - const rows = sqliteJson(`SELECT * FROM ${sqlIdent(table)}`); - const inserts = insertStatements(table, cols, rows).join(''); - summary[table] = rows.length; - if (emit) writeFileSync(resolve(emit, `${table}.sql`), inserts); - else if (inserts) applyFile(table, inserts); // wipe already cleared it; skip an empty INSERT batch + } else { + summary = runShip({ + tables: TABLES, + readTable, + wipeSql: wipeSql(), + apply: applyFile, + sleep: sleepSync, + readCounts: (expected) => readShippedCounts(d1Name, remote, expected), + maxStatements, + paceMs, + }); } } finally { if (tmp) rmSync(tmp, { recursive: true, force: true }); } + console.log( JSON.stringify( { workDb, target: emit ? `emit:${emit}` : remote ? 'D1:remote' : 'D1:local', rows: summary }, diff --git a/scripts/ship-related-persons.test.mjs b/scripts/ship-related-persons.test.mjs index b15612d1..b0d2d56a 100644 --- a/scripts/ship-related-persons.test.mjs +++ b/scripts/ship-related-persons.test.mjs @@ -7,6 +7,9 @@ import { parseMinLinks, resolveD1Name, insertStatements, + chunkStatements, + runShip, + assertShippedCounts, sqlLiteral, sqlIdent, TABLES, @@ -181,3 +184,126 @@ test('related_persons_internal (relative-name PII) is NOT shipped to the served // build/work DB only. If a real read path is ever added, ship it deliberately and revisit anonymization. assert.ok(!TABLES.includes('related_persons_internal')); }); + +// The first full-corpus ship put 516k rows through in one hour, p90 batch 18.2s, and the database then +// returned „internal error" for hours. Chunking bounds what a single request carries; these lock the +// boundaries so a refactor cannot quietly restore the one-request-per-table shape. +test('chunkStatements splits into request-sized groups and preserves order', () => { + const stmts = Array.from({ length: 7 }, (_, i) => `S${i};`); + assert.deepEqual(chunkStatements(stmts, 3), [ + ['S0;', 'S1;', 'S2;'], + ['S3;', 'S4;', 'S5;'], + ['S6;'], + ]); + assert.deepEqual(chunkStatements(stmts, 100), [stmts], 'fits in one request'); + assert.deepEqual(chunkStatements([], 3), [], 'nothing to ship, nothing to send'); +}); + +test('chunkStatements refuses a non-positive size rather than looping forever', () => { + assert.throws(() => chunkStatements(['a'], 0), /positive integer/); + assert.throws(() => chunkStatements(['a'], -1), /positive integer/); + assert.throws(() => chunkStatements(['a'], 1.5), /positive integer/); +}); + +// The ship wipes, then writes over SEVERAL requests with no cross-request transaction. Before this, a +// failure part-way exited 0 and the surface silently served a partial corpus — while the READ path +// (EOP hydrate) already refused on a row-count mismatch. Same standard on the destructive side. +test('assertShippedCounts passes when the target holds exactly what was shipped', () => { + assert.doesNotThrow(() => + assertShippedCounts({ persons: 3, interest_links: 2 }, { persons: 3, interest_links: 2 }), + ); +}); + +test('assertShippedCounts fails on a short table and names the numbers', () => { + assert.throws( + () => assertShippedCounts({ persons: 3, interest_links: 2 }, { persons: 3, interest_links: 1 }), + (e) => + /interest_links: shipped 2, target has 1/.test(e.message) && + /verification FAILED/.test(e.message), + ); +}); + +test('assertShippedCounts fails closed when the read-back returned nothing', () => { + assert.throws( + () => assertShippedCounts({ persons: 3 }, {}), + /persons: shipped 3, target has no answer/, + ); +}); + +test('assertShippedCounts ignores tables the ship skipped as absent', () => { + assert.doesNotThrow(() => + assertShippedCounts({ persons: 1, gone: 'absent (skipped)' }, { persons: 1 }), + ); +}); + +// These drive the REAL ship path, not the helpers in isolation. That distinction is the whole point: +// with the previous shape — helpers unit-tested, main() calling them — reverting the call site to one +// request per table, and deleting the verification line outright, BOTH left the suite fully green. +const shipHarness = (over = {}) => { + const calls = []; + const naps = []; + const source = over.source ?? { + persons: { rowCount: 5, statements: ['A;', 'B;', 'C;', 'D;', 'E;'] }, + declarations: { rowCount: 1, statements: ['F;'] }, + }; + const opts = { + tables: over.tables ?? ['persons', 'declarations'], + readTable: (t) => source[t] ?? null, + wipeSql: 'DELETE FROM persons;', + apply: (name, sql) => calls.push([name, sql]), + sleep: (ms) => naps.push(ms), + readCounts: over.readCounts ?? ((expected) => ({ ...expected })), + maxStatements: over.maxStatements ?? 2, + paceMs: 500, + }; + return { calls, naps, run: () => runShip(opts) }; +}; + +test('runShip wipes, then ships every table in request-sized chunks, in order', () => { + const h = shipHarness(); + const summary = h.run(); + + assert.deepEqual( + h.calls.map(([name]) => name), + ['0_wipe', 'persons.1', 'persons.2', 'persons.3', 'declarations'], + 'wipe first, chunks numbered so a failed request is identifiable, single-chunk table stays bare', + ); + assert.deepEqual( + h.calls.map(([, sql]) => sql), + ['DELETE FROM persons;', 'A;B;', 'C;D;', 'E;', 'F;'], + ); + assert.deepEqual(summary, { persons: 5, declarations: 1 }); +}); + +// THE regression this exists to prevent: the counter used to restart per table, so every table +// boundary — including wipe → first insert, the most destructive transition in the run — was unpaced. +test('runShip paces every request boundary, including wipe → first insert', () => { + const h = shipHarness(); + h.run(); + assert.equal(h.calls.length, 5); + assert.deepEqual( + h.naps, + [500, 500, 500, 500], + 'one gap between each pair of requests: none before the first, none after the last, and none skipped at a table boundary', + ); +}); + +test('runShip verifies what landed — a short table fails the run', () => { + const h = shipHarness({ readCounts: (expected) => ({ ...expected, persons: 4 }) }); + assert.throws(() => h.run(), /ship verification FAILED[\s\S]*persons: shipped 5, target has 4/); +}); + +test('runShip fails the run when the read-back itself could not answer', () => { + const h = shipHarness({ readCounts: () => ({}) }); + assert.throws(() => h.run(), /no answer/); +}); + +test('runShip skips a table absent from the work DB without shipping or verifying it', () => { + const h = shipHarness({ tables: ['persons', 'declarations', 'ghost'] }); + const summary = h.run(); + assert.equal(summary.ghost, 'absent (skipped)'); + assert.ok( + !h.calls.some(([name]) => name.startsWith('ghost')), + 'an absent table must issue no request', + ); +}); From ddaaacb6275c5858283c795f99e3ee7b329d4ba7 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 7 Aug 2026 23:30:20 +0000 Subject: [PATCH 33/69] =?UTF-8?q?fix(etl):=20=D0=B8=D0=B7=D1=82=D0=B8?= =?UTF-8?q?=D1=87=D0=B0=D1=89=D0=B8=20=D0=BF=D0=BE=D1=82=D0=BE=D1=86=D0=B8?= =?UTF-8?q?=20=D0=BF=D1=80=D0=B8=20=D1=87=D0=B5=D1=82=D0=B5=D0=BD=D0=B8?= =?UTF-8?q?=D1=8F=D1=82=D0=B0=20=D0=BE=D1=82=20=D0=95=D0=9E=D0=9F=20=D0=B8?= =?UTF-8?q?=20=D1=81=D0=B8=D0=B3=D0=BD=D0=B0=D0=BB=20=D0=B7=D0=B0=20=D1=83?= =?UTF-8?q?=D1=81=D0=BF=D0=B5=D1=88=D0=B5=D0=BD=20=D0=BA=D1=80=D0=BE=D0=BD?= =?UTF-8?q?=20(#284)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(etl): release EOP response bodies on every abandoned path listBucketForDay walked away from a response without reading it on three paths - a blocked redirect, a 403/404 bucket, any other non-OK status - and fetchJson did the same on non-OK. Each left the body stream open for the rest of the invocation. Verified against the live staging Workflow that this is NOT the cause of the per-instance "hung" error: an instance whose every body was drained still produced it. Filed as hygiene, not as that fix. * feat(etl): announce a finished cron refresh The runtime logs an error on every successful instance of the refresh Workflow, so "no errors" says nothing about whether the cron ran. Emit etl_refresh_complete with the run summary after the transient staging is dropped, so success has a positive signal - and a failed run stays silent. * docs(etl): record the false hung error on completed Workflow instances * fix(etl): не чакай освобождаването на тялото и покрий третия път Ревюто извади три неща по тази промяна: - освобождаването на тялото се чакаше с await, тъй че поток, чийто cancel() не приключва, заклещваше четенето - точно провалът, който този PR трябва да намалява. Освобождаването вече само се задейства, без да се чака; мутационно проверено (връщането на await виси 120 s и проваля теста); - блокираното пренасочване при четене на обект беше единственият път без тест - заобикалянето му оставяше набора зелен. Добавен; - текстът в docs/etl.md звучеше като „въпросът е приключен". Същият обход пропусна жив теч във fx.ts, тъй че сега е записано какво би го отворило наново, плюс поправка на остаряла обосновка за FX (от #158 Worker-ът зарежда курсове). * fix(ingest): затвори и изтичащите потоци при тегленето на курсове Обходът, който придружаваше поправката в eop.ts, беше непълен: fx.ts носи същите три форми - хвърляне при блокирано пренасочване, continue при 404 и хвърляне при не-OK - и трите изоставяха тялото непрочетено. 404 е най-утъпканият път от тях, понеже Frankfurter отговаря така за всяка валута, която не обслужва, и зареждащият нарочно продължава нататък. Освобождаването вече е една обща функция в @sigma/ingest, която ползват и двата четящи пътя, вместо да се дублира. Мутационно проверено. * fix(ingest): дръж освобождаването местно, за да не счупиш CLI-то за курсовете Предишният ангажимент извади освобождаването в нов споделен модул и го внасяше от fx.ts без разширение. Но fx.ts се зарежда като суров TypeScript и от scripts/load-fx.mjs под чист Node, където такъв внос не се решава - ERR_MODULE_NOT_FOUND, тоест целият CLI път за курсовете умираше. Нито един тест не стартира този скрипт, затова CI мина зелено. Разширението .ts решава вноса за Node, но иска allowImportingTsExtensions за целия монорепо - твърде голяма промяна за този PR. Затова всеки от двата пакета си държи своя малка функция, а споделеният модул отпада. * test(scripts): пази това, че CLI-тата за курсове и внос се зареждат под чист Node Споделените модули в packages/ се внасят като суров TypeScript от скриптовете в scripts/, а разрешаването на Node е по-строго от това на пакетчика: vite намира внос без разширение, чистият Node - не. Затова внос, добавен в споделен модул, може да е зелен по всички наборни тестове и пак да убива `node scripts/load-fx.mjs` при старта. Точно това стана в този клон и CI не мигна. Тестът извежда двойките скрипт/внос от самия код, разрешава ги както ги разрешава ESM зареждачът и ги внася в отделен процес без кука и без пакетчик. Мутациите: връщането на вноса без разширение вали новия тест, а наборът на packages/ingest остава 49 от 49 зелени; изкривяване на израза за търсене вали проверката, че изобщо намира двойки. --------- Co-authored-by: t --- apps/etl/src/eop.test.ts | 118 +++++++++++++++++++++++++++++++- apps/etl/src/eop.ts | 39 ++++++++--- apps/etl/src/index.test.ts | 54 +++++++++++++++ apps/etl/src/index.ts | 15 +++- docs/etl.md | 42 +++++++++++- packages/ingest/src/fx.test.ts | 59 ++++++++++++++++ packages/ingest/src/fx.ts | 28 +++++++- scripts/cli-ts-imports.test.mjs | 69 +++++++++++++++++++ 8 files changed, 410 insertions(+), 14 deletions(-) create mode 100644 scripts/cli-ts-imports.test.mjs diff --git a/apps/etl/src/eop.test.ts b/apps/etl/src/eop.test.ts index a96a50df..851ff51f 100644 --- a/apps/etl/src/eop.test.ts +++ b/apps/etl/src/eop.test.ts @@ -1,5 +1,28 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { computeWorkerCatchupPlan, listBucketForDay } from './eop'; +import { computeWorkerCatchupPlan, listBucketForDay, stageBaseFromBucket } from './eop'; + +/** + * A response whose stream is deliberately left open, so `cancel()` on the underlying source really + * fires. A stream that is enqueued *and closed* would report success without proving anything: the + * spec short-circuits `cancel()` once a stream is already closed. + */ +function openBodyResponse(init: ResponseInit & { url?: string }): { + response: Response; + cancelled: () => boolean; +} { + let cancelled = false; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('')); + }, + cancel() { + cancelled = true; + }, + }); + const response = new Response(body, init); + if (init.url) Object.defineProperty(response, 'url', { value: init.url }); + return { response, cancelled: () => cancelled }; +} function fakeDbFromFreshness(maxLoadedDate: string): D1Database { const db = { @@ -53,3 +76,96 @@ describe('EOP fetch host allowlist', () => { ); }); }); + +describe('EOP responses the ingest walks away from', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('releases the body of a missing bucket instead of leaving the stream open', async () => { + const { response, cancelled } = openBodyResponse({ status: 403 }); + vi.stubGlobal('fetch', vi.fn(async () => response) as unknown as typeof fetch); + + await expect(listBucketForDay('2026-06-01')).resolves.toBeNull(); + expect(cancelled()).toBe(true); + }); + + it('releases the body of a failed bucket listing before throwing', async () => { + const { response, cancelled } = openBodyResponse({ status: 500 }); + vi.stubGlobal('fetch', vi.fn(async () => response) as unknown as typeof fetch); + + await expect(listBucketForDay('2026-06-01')).rejects.toThrow(/bucket 2026-06-01: HTTP 500/); + expect(cancelled()).toBe(true); + }); + + it('releases the body of a blocked redirect before throwing', async () => { + const { response, cancelled } = openBodyResponse({ + status: 200, + url: 'https://evil.example/open-data-2026-06-01/', + }); + vi.stubGlobal('fetch', vi.fn(async () => response) as unknown as typeof fetch); + + await expect(listBucketForDay('2026-06-01')).rejects.toThrow(/blocked redirected EOP fetch/); + expect(cancelled()).toBe(true); + }); + + // The one drain site the first cut of this change left uncovered: bypassing it kept the whole suite + // green. A blocked redirect on an OBJECT fetch is a different call path from the bucket listing. + it('releases the body of a redirected object fetch before throwing', async () => { + const { response, cancelled } = openBodyResponse({ + status: 200, + url: 'https://evil.example/open-data-2026-06-01/contracts.json', + }); + vi.stubGlobal('fetch', vi.fn(async () => response) as unknown as typeof fetch); + + await expect( + stageBaseFromBucket( + {} as D1Database, + { + day: '2026-06-01', + bucketUrl: 'https://storage.eop.bg/open-data-2026-06-01/', + keys: { contracts: 'contracts.json' }, + }, + '2026-06-01T00:00:00.000Z', + ), + ).rejects.toThrow(/blocked redirected EOP fetch from storage\.eop\.bg to evil\.example/); + expect(cancelled()).toBe(true); + }); + + // Cancellation must be initiated, never awaited: a stream whose cancel() never settles must not be + // able to wedge the ingest. Before this, `await res.body.cancel()` made listBucketForDay hang forever. + it('does not wait for a cancel that never settles', async () => { + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('x')); + }, + cancel() { + return new Promise(() => {}); // never settles + }, + }); + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(body, { status: 403 })) as unknown as typeof fetch, + ); + + await expect(listBucketForDay('2026-06-01')).resolves.toBeNull(); + }); + + it('releases the body of a failed object fetch before throwing', async () => { + const { response, cancelled } = openBodyResponse({ status: 500 }); + vi.stubGlobal('fetch', vi.fn(async () => response) as unknown as typeof fetch); + + await expect( + stageBaseFromBucket( + {} as D1Database, + { + day: '2026-06-01', + bucketUrl: 'https://storage.eop.bg/open-data-2026-06-01/', + keys: { contracts: 'contracts.json' }, + }, + '2026-06-01T00:00:00.000Z', + ), + ).rejects.toThrow(/HTTP 500/); + expect(cancelled()).toBe(true); + }); +}); diff --git a/apps/etl/src/eop.ts b/apps/etl/src/eop.ts index 96f63195..7a0fffab 100644 --- a/apps/etl/src/eop.ts +++ b/apps/etl/src/eop.ts @@ -70,14 +70,32 @@ const dayUrl = (baseUrl: string, day: string): string => const objectUrl = (bucketUrl: string, key: string): string => `${bucketUrl}${encodeURIComponent(key)}`; -function assertAllowedFinalHost(requestUrl: string, responseUrl: string): void { +function disallowedFinalHost(requestUrl: string, responseUrl: string): string | null { const requested = new URL(requestUrl); const final = new URL(responseUrl || requestUrl); - if (final.host !== requested.host) { - throw new Error(`blocked redirected EOP fetch from ${requested.host} to ${final.host}`); + if (final.host === requested.host) return null; + return `blocked redirected EOP fetch from ${requested.host} to ${final.host}`; +} + +// A response body that is never read keeps its stream open for the rest of the invocation; the +// collector is not a substitute for releasing it. Every path below that walks away from a response +// without reading it - a blocked redirect, a missing bucket, any non-OK status - releases it here. +// Deliberately NOT awaited: cancelling only needs to be INITIATED for the runtime to release the +// stream, and awaiting it would make every caller hostage to a cancel() that never settles, which +// is precisely the failure mode this file exists to reduce. +function discardBody(res: Response): void { + try { + void res.body?.cancel().catch(() => {}); + } catch { + // Already consumed, locked, or errored - there is nothing left to release either way. } } +function releaseAndFail(res: Response, message: string): never { + discardBody(res); + throw new Error(message); +} + function decodeXml(s: string): string { return s .replace(/</g, '<') @@ -175,9 +193,13 @@ export async function listBucketForDay( ): Promise { const bucketUrl = dayUrl(opts.baseUrl ?? DEFAULT_BASE_URL, day); const res = await fetch(bucketUrl); - assertAllowedFinalHost(bucketUrl, res.url); - if (res.status === 403 || res.status === 404) return null; - if (!res.ok) throw new Error(`bucket ${day}: HTTP ${res.status}`); + const blocked = disallowedFinalHost(bucketUrl, res.url); + if (blocked) return releaseAndFail(res, blocked); + if (res.status === 403 || res.status === 404) { + discardBody(res); + return null; + } + if (!res.ok) return releaseAndFail(res, `bucket ${day}: HTTP ${res.status}`); const keys: BucketKeys = {}; for (const key of parseBucketKeys(await res.text())) { @@ -189,8 +211,9 @@ export async function listBucketForDay( async function fetchJson(url: string): Promise { const res = await fetch(url); - assertAllowedFinalHost(url, res.url); - if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`); + const blocked = disallowedFinalHost(url, res.url); + if (blocked) return releaseAndFail(res, blocked); + if (!res.ok) return releaseAndFail(res, `${url}: HTTP ${res.status}`); return res.json(); } diff --git a/apps/etl/src/index.test.ts b/apps/etl/src/index.test.ts index 536ef5ba..c1c62649 100644 --- a/apps/etl/src/index.test.ts +++ b/apps/etl/src/index.test.ts @@ -214,3 +214,57 @@ describe('RefreshWorkflow FX loading (#158)', () => { expect(usd.amount_eur).toBeCloseTo(EXPECTED_USD_EUR, 2); }); }); + +// The Workers runtime logs an error on every *successful* instance of this Workflow, so the absence +// of errors proves nothing about whether the cron actually ran. A finished refresh has to announce +// itself — and, just as importantly, a failed one must not. +describe('RefreshWorkflow completion signal', () => { + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + function captureLogs(): () => Record[] { + const lines: string[] = []; + vi.spyOn(console, 'log').mockImplementation((line: unknown) => { + lines.push(String(line)); + }); + return () => + lines.flatMap((line) => { + try { + return [JSON.parse(line) as Record]; + } catch { + return []; + } + }); + } + + it('announces a finished refresh with the run summary', async () => { + const db = freshServedDb(); + stubFetchRoutes(); + const logs = captureLogs(); + + const result = await runRefresh(makeWorkflow(db)); + + const done = logs().find((e) => e.event === 'etl_refresh_complete'); + expect(done).toBeDefined(); + expect(done?.staged).toBe(result.staged); + expect(done?.derived).toBe(result.derived); + expect(done?.to).toBe(TODAY); + }); + + it('stays silent when the refresh throws', async () => { + const db = freshServedDb(); + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new Error('storage.eop.bg unreachable'); + }) as unknown as typeof fetch, + ); + const logs = captureLogs(); + + await expect(runRefresh(makeWorkflow(db))).rejects.toThrow(/storage\.eop\.bg unreachable/); + + expect(logs().some((e) => e.event === 'etl_refresh_complete')).toBe(false); + }); +}); diff --git a/apps/etl/src/index.ts b/apps/etl/src/index.ts index a3ad9afc..cb352af5 100644 --- a/apps/etl/src/index.ts +++ b/apps/etl/src/index.ts @@ -94,6 +94,12 @@ export class RefreshWorkflow extends WorkflowEntrypoint { let results: Awaited> = []; let staged = 0; let derived = 0; + // The runtime logs an error ("...your Worker's code had hung...") on every *successful* instance + // of this Workflow, at the instant run() returns - measured across runs of 5 and 30 steps, see + // docs/etl.md. "No errors in the dashboard" is therefore not a health signal here, so a refresh + // that actually finished has to say so itself. Logged from the finally, after the staging drop, + // so it only ever claims success for a run that survived its own cleanup. + let outcome: RefreshResult | null = null; try { await step.do('create-transient-staging', async () => @@ -109,7 +115,8 @@ export class RefreshWorkflow extends WorkflowEntrypoint { if (staged === 0) { console.warn(JSON.stringify({ level: 'warn', event: 'etl_zero_ingest', fetchedAt, plan })); - return { ...plan, days: results.length, staged: 0, derived: 0 }; + outcome = { ...plan, days: results.length, staged: 0, derived: 0 }; + return outcome; } // FX rates BEFORE the derive (#158): the CLI paths run scripts/load-fx.mjs first, but this @@ -180,9 +187,13 @@ export class RefreshWorkflow extends WorkflowEntrypoint { } }); - return { ...plan, days: results.length, staged, derived }; + outcome = { ...plan, days: results.length, staged, derived }; + return outcome; } finally { await step.do('drop-transient-staging', async () => dropTransientStaging(this.env.DB)); + if (outcome) { + console.log(JSON.stringify({ level: 'info', event: 'etl_refresh_complete', ...outcome })); + } } } } diff --git a/docs/etl.md b/docs/etl.md index 6c624620..d7b0ec96 100644 --- a/docs/etl.md +++ b/docs/etl.md @@ -286,11 +286,51 @@ cross-check показват изпуснати десетични запета 6. пуска `refresh-slice.sql`; 7. пуска reconciliation gate-а (#97, `runIntegrityChecks`) върху обслужвания D1 в стъпка `integrity-gate` и проваля стъпката при дрейф (виж [`integrity-gate.md`](integrity-gate.md)). - FX-проверката нарочно **не** е включена тук — Worker-ът не зарежда курсове (проследено в #154). + FX-проверката нарочно **не** е включена тук (проследено в #154). Старата обосновка „Worker-ът не + зарежда курсове" вече не важи — от #158 стъпката `load-fx` тегли липсващите курсове преди derive-а. Worker-ът е нарочно ограничен до малък скорошен прозорец. Ако базата е далеч назад, логва, че прозорецът е capped, и оставя голямото догонване на CLI-то (`pnpm run import --catchup`). +### Лъжливата грешка „hung" при всеки успешен цикъл + +Всяка **успешно завършила** инстанция на Workflow-а оставя и запис от ниво `error`: + +> The Workers runtime canceled this request because it detected that your Worker's code had hung +> and would never generate a response. + +Тя пада на същата милисекунда, на която инстанцията получава своя `end` — тоест **след** като +всички стъпки са върнали `success` и изходът е записан. Извикването се отчита с +`outcome: exception`, докато самата инстанция е `status: complete`. Нищо не се губи. + +Измерено на `sigma-etl-stage` (2026-08-06) с три инстанции, при които варирахме всичко, което е +в наши ръце: + +| Пуск | Стъпки | Време | Неизчерпани тела на отговори | „hung" | +| --- | --- | --- | --- | --- | +| крон, прозорец от 6 дни | 30 | 91 s | да | да | +| празен прозорец (`today` в бъдещето) | 5 | 1,6 s | да (един 403) | да | +| един пълен ден | 30 | 26 s | нула | да | + +Продължителност, брой стъпки, обем към D1, изход от четенията и кой `return` се ползва — грешката +е неизменна спрямо всяко от тях. Затова към днешна дата я третираме като страничен продукт на +средата (`executionModel: "stateless"`), а не като дефект в нашия код. + +**Това е измерване, не присъда.** Обходът на четенията, направен заедно с тези опити, пропусна +един жив теч в `packages/ingest/src/fx.ts` — тоест „проверихме всичко наше" не се оказа вярно. +Отваряйте въпроса наново, ако: грешката спре да пада точно на `end` времето; появи се инстанция +със `status: complete`, но с непълни стъпки; или се намери в наш код изчакване, което не приключва. + +**Следствие за наблюдението:** „няма грешки в таблото" не е показател за здраве на този Worker. +Здравето се чете от `etl_refresh_complete` — записът, който `run()` оставя след успешното +изчистване на преходния staging. Точната истина винаги може да се провери и през +`GET /accounts/{acct}/workflows/{name}/instances/{id}`, който дава списъка със стъпките. + +Предупреждението „An RPC result was not disposed properly" се появява само при крон-цикли с +многодневен прозорец и не се възпроизведе при контролираните еднодневни пускания. В `apps/etl` и +`packages/ingest` няма нищо, което държи RPC резултат, дължащ `dispose` — тоест източникът е под +нас, в свързването. + **Текущо ограничение на Worker-а:** той стейджва само вложения OCDS файл. Coercion-ът на плоския базов JSON все още живее в CLI loader-а и трябва да се извади в споделени Worker-safe helper-и, преди Worker-ът да стейджва и базови contracts, tenders и annexes. Големите backfill-и остават diff --git a/packages/ingest/src/fx.test.ts b/packages/ingest/src/fx.test.ts index b6470b36..7f8fdc2c 100644 --- a/packages/ingest/src/fx.test.ts +++ b/packages/ingest/src/fx.test.ts @@ -209,6 +209,65 @@ describe('loadFxRates', () => { await expect(loadFxRates(d1, { fetchedAt: FETCHED_AT, fetchFn })).rejects.toThrow(/HTTP 500/); }); + // Same defect class as the EOP reads: a body that is never read holds its stream open for the rest + // of the invocation. The 404 path is the most-travelled one here — Frankfurter answers 404 for any + // base currency it does not serve, and the loader deliberately continues past it. + it('releases the body of every response it walks away from', async () => { + const openBody = () => { + let cancelled = false; + const body = new ReadableStream({ + start(c) { + c.enqueue(new TextEncoder().encode('x')); + }, + cancel() { + cancelled = true; + }, + }); + return { body, cancelled: () => cancelled }; + }; + + // 404 — unsupported currency, the loader warns and moves on. + { + const { db, d1 } = fxDb(); + stageContract(db, 'USD', '2026-07-08'); + const b = openBody(); + const fetchFn = vi.fn( + async () => new Response(b.body, { status: 404 }), + ) as unknown as typeof fetch; + const summary = await loadFxRates(d1, { fetchedAt: FETCHED_AT, fetchFn }); + expect(summary.warnings.join(' ')).toMatch(/not served by frankfurter/); + expect(b.cancelled()).toBe(true); + } + + // Non-OK — the loader throws. + { + const { db, d1 } = fxDb(); + stageContract(db, 'USD', '2026-07-08'); + const b = openBody(); + const fetchFn = vi.fn( + async () => new Response(b.body, { status: 500 }), + ) as unknown as typeof fetch; + await expect(loadFxRates(d1, { fetchedAt: FETCHED_AT, fetchFn })).rejects.toThrow(/HTTP 500/); + expect(b.cancelled()).toBe(true); + } + + // Redirected to another host — the host pin throws. + { + const { db, d1 } = fxDb(); + stageContract(db, 'USD', '2026-07-08'); + const b = openBody(); + const fetchFn = vi.fn(async () => { + const res = new Response(b.body, { status: 200 }); + Object.defineProperty(res, 'url', { + value: 'https://evil.example/v1/2026-07-01..2026-07-08', + }); + return res; + }) as unknown as typeof fetch; + await expect(loadFxRates(d1, { fetchedAt: FETCHED_AT, fetchFn })).rejects.toThrow(); + expect(b.cancelled()).toBe(true); + } + }); + it('keeps successfully loaded currencies when another currency fails', async () => { const { db, d1 } = fxDb(); stageContract(db, 'USD', '2026-07-08'); diff --git a/packages/ingest/src/fx.ts b/packages/ingest/src/fx.ts index 7d78a4ba..f24f5323 100644 --- a/packages/ingest/src/fx.ts +++ b/packages/ingest/src/fx.ts @@ -4,6 +4,20 @@ // validation) live here once so the two implementations cannot drift; the Worker-native loader and // its coverage guard are D1-based and pure-fetch, safe for workerd (no Node APIs). +// A response body that is never read keeps its stream - and the connection behind it - open for the +// rest of the invocation; the collector is not a substitute. Deliberately NOT awaited: cancelling only +// needs to be INITIATED for the runtime to release the stream, and awaiting it would make the caller +// hostage to a cancel() that never settles. Kept private here rather than shared with apps/etl: this +// file is also loaded as raw TypeScript by the Node CLI (scripts/load-fx.mjs), where an extensionless +// relative import does not resolve and a `.ts` one needs allowImportingTsExtensions repo-wide. +function discardBody(res: Response): void { + try { + void res.body?.cancel().catch(() => {}); + } catch { + // Already consumed, locked, or errored - there is nothing left to release either way. + } +} + // Canonical host. The legacy api.frankfurter.app now 301-redirects here — with host-pinned // fetches (assertSameFinalHost) the legacy host would fail closed, so point at the target // directly. Same response shape; the /v1 prefix is required on the .dev host. @@ -220,15 +234,25 @@ export async function loadFxRates(db: D1Database, opts: LoadFxOptions): Promise< try { const url = fxSeriesUrl(gap.currency, start, end, api); const res = await fetchFn(url); - assertSameFinalHost(url, res.url); + try { + assertSameFinalHost(url, res.url); + } catch (err) { + discardBody(res); + throw err; + } if (res.status === 404) { // Frankfurter answers 404 for a base currency it does not serve — permanent, not // transient: warn and move on (CLI parity), never brick the cron on one odd currency. + // The body still has to be released — this `continue` is the most-travelled path here. + discardBody(res); load.status = 'unsupported'; summary.warnings.push(`currency ${gap.currency} not served by frankfurter`); continue; } - if (!res.ok) throw new Error(`HTTP ${res.status}`); + if (!res.ok) { + discardBody(res); + throw new Error(`HTTP ${res.status}`); + } const { rows, warnings } = parseFxSeries(await res.json(), gap.currency, `${start}..${end}`); summary.warnings.push(...warnings); await upsertFxRates(db, rows, opts.fetchedAt); diff --git a/scripts/cli-ts-imports.test.mjs b/scripts/cli-ts-imports.test.mjs new file mode 100644 index 00000000..3b59e5f1 --- /dev/null +++ b/scripts/cli-ts-imports.test.mjs @@ -0,0 +1,69 @@ +// The plain-Node CLI scripts import shared logic straight out of packages/ as raw TypeScript, which +// Node type-strips on the fly. That resolver is far stricter than the bundler one the vitest suites +// run under: vite happily resolves an extensionless relative specifier, plain Node does not. So an +// import added to a shared module can be green across every package suite and still make +// `node scripts/load-fx.mjs` die at startup with ERR_MODULE_NOT_FOUND — which is exactly what +// happened while writing the drain fix in this branch, and nothing in CI noticed. +// +// This test closes that gap the only way that proves anything: it resolves each specifier the way +// Node's ESM loader does (relative to the importing script) and imports it in a child process with +// no hooks, no register shim and no bundler, asserting the whole graph loads. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { readdirSync, readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const here = dirname(fileURLToPath(import.meta.url)); + +// Every `import … from ''` in a CLI script that reaches outside scripts/ — any extension or +// none, since an extensionless one is precisely the shape that breaks. Only the top-level scripts: +// scripts/cacbg/* run under the register-ts hook and are a different contract. +const IMPORT_RE = /\bfrom\s+'(\.\.\/[^']+)'/g; + +function crossPackageImports() { + const found = []; + for (const name of readdirSync(here).sort()) { + if (!name.endsWith('.mjs') || name.endsWith('.test.mjs')) continue; + const file = resolve(here, name); + const src = readFileSync(file, 'utf8'); + for (const [, specifier] of src.matchAll(IMPORT_RE)) found.push({ name, file, specifier }); + } + return found; +} + +const imports = crossPackageImports(); + +test('the scan actually finds the cross-package CLI imports', () => { + // Without this the whole file degrades to a no-op the moment the regex or the layout drifts — + // zero pairs would mean zero assertions and a green run. The three known importers today are + // import.mjs, load-eop.mjs and load-fx.mjs. + assert.ok(imports.length >= 3, `expected cross-package imports, found ${imports.length}`); + const importers = new Set(imports.map((i) => i.name)); + for (const expected of ['import.mjs', 'load-eop.mjs', 'load-fx.mjs']) { + assert.ok(importers.has(expected), `${expected} should import shared package code`); + } +}); + +for (const { name, file, specifier } of imports) { + test(`${name} → ${specifier} loads under plain node`, () => { + // ESM relative resolution IS new URL(specifier, parent) — no extension search, no index lookup. + // Doing it here reproduces the loader's answer for the specifier exactly as the script wrote it. + const target = new URL(specifier, pathToFileURL(file)).href; + try { + execFileSync( + 'node', + ['--input-type=module', '-e', `await import(${JSON.stringify(target)})`], + { + encoding: 'utf8', + stdio: 'pipe', + }, + ); + } catch (e) { + assert.fail( + `${name} cannot load ${specifier} under plain node — the CLI would die at startup:\n${e.stderr}`, + ); + } + }); +} From 8139adb28e3cb20ffb43b12e1f33e383ea0cffe1 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Sat, 8 Aug 2026 00:23:33 +0000 Subject: [PATCH 34/69] fix(etl): never pair a full derive with a partial catch-up window (#270) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(etl): never pair a full derive with a partial catch-up window `--catchup` escalated to `derive=full` whenever the gap exceeded LARGE_GAP_DAYS (14), but the catch-up window is gap-aware and only covers the tail of the feed. A full derive rebuilds the domain from staging (normalize-raw.sql opens with `DELETE FROM contracts`), so the corpus was rebuilt from those few weeks and every older contract was dropped and never reloaded. The two halves were never meant to go together: a full derive belongs with an explicit, feed-wide window, a gap-aware window with a slice derive. Only the "existing corpus + large gap" branch mismatched them. - catch-up now always derives a slice, however wide the gap; an explicit `--derive=full` still overrides - `assertDeriveWindowSafe` refuses a full derive whose window does not reach the start of the feed while a corpus exists, before the load rather than after the delete - the invariant lives in @sigma/ingest as `fullDeriveIsSafe`, unit-tested alongside computeCatchupWindow The deployed cron is unaffected: it refreshes every 6 hours so it never accumulates the gap, and RefreshWorkflow does not run normalize-raw. This bites local and recovery runs, exactly when a corpus is being restored. * fix(etl): питай за всяка изчиствана таблица, не само за договорите Пазачът срещу пълен derive с частичен прозорец броеше редовете в `contracts`, а изчистването в normalize-raw.sql изпразва четиринайсет таблици. Корпус без договори, но с попълнени `tenders`, `bidders` или `authorities` - точно каквото оставя половинчат пробег - минаваше през пазача и презиждането ги отнасяше, без да има откъде да се върнат. Списъкът вече идва от самия SQL през нов знак `@full-clear`, вместо да е преписан в JS: преписът на един рушащ списък е бъг със закъснител. Обхватът спира преди `data_freshness` и `pipeline_stats` - те се пишат наново всеки пробег и ако се броят за корпус, пазачът отказва всичко, включително първоначалното зареждане. Проверката вали и когато не може да отговори: липсваща таблица правеше `safeD1` да върне празно, което се четеше като „няма корпус" и пускаше рушащия път. Отказът вече и разчиства преходния staging след себе си. Тестовете карат истинския `import.mjs` като отделен процес с подставени `wrangler` и `node`, защото проверката само на предиката е точно каквото пропусна този бъг. Мутацията „питай пак само за договорите" вали три теста. Описанието в docs/etl.md твърдеше, че отказът важи само при изрично `--derive=full`; всъщност важи и за подразбиращия се - поправено. * fix(etl): разпознавай изчистваната таблица и когато е в кавички Ревюто счупи пазача с промяна, която в преглед изглежда като форматиране: `DELETE FROM "search_index";` е валиден SQLite, но разборът признаваше само голи имена, тъй че таблицата изпадаше от списъка и дупката се отваряше наново - корпус, в който само тя е попълнена, минаваше през пазача. По-лошото беше в теста. Той извеждаше очаквания списък от същия файл със свое копие на същия израз, тъй че наследяваше същата слепота: и двата набора оставаха зелени, докато пазачът пропускаше рушащия път. Оракул, който споделя допусканията на кода, не проверява нищо. Затова списъкът в теста вече е изписан на ръка, а отделна проверка държи него и SQL-а един срещу друг - всяко разминаване вали, вместо да изчезне. Разборът приема и трите форми кавички на SQLite. Поправено и твърдението в docs/etl.md, че `--catchup` **винаги** прави slice: при първо пускане, когато няма заредени дни, той взима прозорец от началото на емисията и пълен derive. --------- Co-authored-by: t --- docs/etl.md | 23 +- packages/ingest/src/ocds.test.ts | 22 ++ packages/ingest/src/ocds.ts | 21 ++ .../ingest/src/refresh-full-clear.test.ts | 77 +++++++ packages/ingest/src/refresh.ts | 39 ++++ scripts/import-guard.test.mjs | 201 ++++++++++++++++++ scripts/import.mjs | 75 ++++++- scripts/normalize-raw.sql | 4 + 8 files changed, 450 insertions(+), 12 deletions(-) create mode 100644 packages/ingest/src/refresh-full-clear.test.ts create mode 100644 scripts/import-guard.test.mjs diff --git a/docs/etl.md b/docs/etl.md index d7b0ec96..c0f5a9a7 100644 --- a/docs/etl.md +++ b/docs/etl.md @@ -116,12 +116,27 @@ node scripts/load-eop.mjs --from=YYYY-MM-DD --to=YYYY-MM-DD --no-ocds Първоначалният backfill и ежедневният refresh ползват едни и същи staging таблици, mapper-и и SQL. Различават се по прозореца от дати и режима на derive: -- **голямо или първоначално догонване:** CLI прозорец + пълен derive; -- **малък steady-state refresh:** gap-aware прозорец + slice derive. +- **първоначално зареждане или пълно презареждане:** прозорец от началото на емисията + пълен derive; +- **догонване и steady-state refresh:** gap-aware прозорец + slice derive. `--derive=full` пуска amendment rollup, FX, NUTS, пълна нормализация и precompute. `--derive=slice` -пуска scoped refresh SQL-а. По подразбиране catch-up логиката избира full derive за големи -празнини и slice derive за малки. +пуска scoped refresh SQL-а. + +Пълният derive **презижда** доменните таблици от staging — `normalize-raw.sql` започва с +`DELETE FROM contracts` — тоест всичко извън заредения прозорец отпада и не се връща. Затова той е +допустим само когато прозорецът стига до началото на емисията (или когато още няма корпус). Понеже +gap-aware прозорецът по устройство покрива само опашката, `--catchup` върху **вече зареден** корпус +прави slice derive, колкото и голяма да е празнината. Изключението е първото пускане: когато няма +никакви заредени дни, догонването взима прозорец от началото на емисията и пълен derive — там няма +какво да се загуби. + +Пълен derive с частичен прозорец върху вече зареден корпус `import.mjs` отказва да продължи — преди +зареждането — вместо да изтрие историята. Отказът важи за **действащия** режим, не само за изрично +подадения: без `--catchup` подразбиращият се derive е `full`, тъй че и `import.mjs --from=2026-06-01` +получава същия отказ. Проверката пита за всяка таблица, която `normalize-raw.sql` изпразва (виж +`@full-clear` там), не само за `contracts` — корпус без договори, но с попълнени `tenders` или +`bidders` е точно състоянието, което половинчат пробег оставя. Ако някоя от тях не може да бъде +прочетена, отказът пак важи: проверка, която не може да провери, не бива да пуска нататък. ## Доменна нормализация (derive) diff --git a/packages/ingest/src/ocds.test.ts b/packages/ingest/src/ocds.test.ts index 014f65f9..fd42274a 100644 --- a/packages/ingest/src/ocds.test.ts +++ b/packages/ingest/src/ocds.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { classifyBucketKey, computeCatchupWindow, + fullDeriveIsSafe, releaseToAmendments, releaseToContracts, releaseToLots, @@ -352,6 +353,27 @@ describe('bucket key and catchup helpers', () => { computeCatchupWindow({ maxLoadedDate: '2026-06-01', today: '2026-06-07', lookbackDays: 3 }), ).toEqual({ from: '2026-05-29', to: '2026-06-07' }); }); + + it('allows a full derive only when the window reaches the start of the feed', () => { + // Initial backfill: nothing to lose, any window is safe. + expect( + fullDeriveIsSafe({ windowFrom: '2026-06-10', feedStart: '2020-01-01', hasCorpus: false }), + ).toBe(true); + // Whole feed reloaded into staging — the rebuild is complete. + expect( + fullDeriveIsSafe({ windowFrom: '2020-01-01', feedStart: '2020-01-01', hasCorpus: true }), + ).toBe(true); + // A catch-up window over an existing corpus would drop everything before it. + expect( + fullDeriveIsSafe({ windowFrom: '2026-06-10', feedStart: '2020-01-01', hasCorpus: true }), + ).toBe(false); + }); + + it('rejects malformed days rather than silently allowing a full derive', () => { + expect(() => + fullDeriveIsSafe({ windowFrom: '10-06-2026', feedStart: '2020-01-01', hasCorpus: true }), + ).toThrow(/windowFrom/); + }); }); describe('splitSqlStatements', () => { diff --git a/packages/ingest/src/ocds.ts b/packages/ingest/src/ocds.ts index fb460cfa..1133671e 100644 --- a/packages/ingest/src/ocds.ts +++ b/packages/ingest/src/ocds.ts @@ -452,6 +452,27 @@ export function computeCatchupWindow({ return { from: from > today ? today : from, to: today }; } +/** + * A full derive rebuilds the domain from whatever the staging tables hold — `scripts/normalize-raw.sql` + * opens with `DELETE FROM contracts` — so every contract outside the loaded window is dropped. It is + * only sound when the window reaches back to the first day the feed is loaded from, or when there is + * no corpus yet (the initial backfill). A gap-aware catch-up window never does, which is why + * `--catchup` derives a slice. + */ +export function fullDeriveIsSafe({ + windowFrom, + feedStart, + hasCorpus, +}: { + windowFrom: string; + feedStart: string; + hasCorpus: boolean; +}): boolean { + validateDay(windowFrom, 'windowFrom'); + validateDay(feedStart, 'feedStart'); + return !hasCorpus || windowFrom <= feedStart; +} + export function daysInWindow(from: string, to: string): number { validateDay(from, 'from'); validateDay(to, 'to'); diff --git a/packages/ingest/src/refresh-full-clear.test.ts b/packages/ingest/src/refresh-full-clear.test.ts new file mode 100644 index 00000000..c90f9f4a --- /dev/null +++ b/packages/ingest/src/refresh-full-clear.test.ts @@ -0,0 +1,77 @@ +/// +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { fullClearTables } from './refresh'; + +// fullClearTables answers one question for scripts/import.mjs: which tables does a full derive empty, +// and therefore what does a partial window destroy? The guard that used to ask it named `contracts` +// alone while the clear had reached fourteen tables — so the tests that matter here are the ones that +// keep the answer tied to the SQL rather than to a copy of it. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const normalizeRaw = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); + +describe('fullClearTables', () => { + it('reads the block out of the real normalize-raw.sql', () => { + const tables = fullClearTables(normalizeRaw); + // The base domain tables: whatever else the block grows, losing any of these is losing the corpus. + expect(tables).toEqual( + expect.arrayContaining(['contracts', 'lots', 'tenders', 'bidders', 'authorities']), + ); + }); + + it('stops before the per-run metadata resets further down the file', () => { + // normalize-raw.sql also clears data_freshness and pipeline_stats, which are rewritten every run. + // Counting them as corpus would make the guard refuse EVERY full derive, initial backfill included + // — a guard that always refuses gets deleted, so this boundary is load-bearing. + const tables = fullClearTables(normalizeRaw); + expect(tables).not.toContain('data_freshness'); + expect(tables).not.toContain('pipeline_stats'); + }); + + it('keeps the marker and the block adjacent', () => { + // If the marker is dropped or drifts away from the DELETEs, this returns [] and import.mjs throws + // rather than silently deciding the corpus is empty and letting the destructive path through. + expect(fullClearTables(normalizeRaw).length).toBeGreaterThanOrEqual(5); + }); + + it('takes only the marked block, and only unqualified deletes', () => { + const sql = [ + 'DELETE FROM before_the_marker;', + '-- @full-clear', + 'DROP TABLE IF EXISTS scratch;', + 'DELETE FROM contracts;', + "DELETE FROM lots WHERE id = 'x';", // scoped: not a full clear + 'DELETE FROM authorities;', + '', + 'DELETE FROM after_the_block;', + ].join('\n'); + expect(fullClearTables(sql)).toEqual(['contracts', 'authorities']); + }); + + it('sees a table however it is quoted', () => { + // `DELETE FROM "search_index";` is valid SQLite and reads as pure formatting. A bare-identifier + // matcher dropped it from the list, which silently reopened the data-loss hole: the guard would + // then wave through a corpus whose only populated table was the re-quoted one. + const sql = [ + '-- @full-clear', + 'DELETE FROM "search_index";', + 'DELETE FROM `flow_pairs`;', + 'DELETE FROM [home_totals];', + 'delete from contracts;', + '', + ].join('\n'); + expect(fullClearTables(sql)).toEqual([ + 'search_index', + 'flow_pairs', + 'home_totals', + 'contracts', + ]); + }); + + it('returns nothing when the marker is absent', () => { + expect(fullClearTables('DELETE FROM contracts;\nDELETE FROM lots;\n')).toEqual([]); + }); +}); diff --git a/packages/ingest/src/refresh.ts b/packages/ingest/src/refresh.ts index 4159da90..45808bf3 100644 --- a/packages/ingest/src/refresh.ts +++ b/packages/ingest/src/refresh.ts @@ -108,6 +108,45 @@ export function transientStagingStatements(workStagingSchemaSql: string): string ); } +const FULL_CLEAR_MARKER = /^--\s*@full-clear\b/i; +// All three SQLite quoting styles, not just the bare identifier. Rewriting one line as +// `DELETE FROM "search_index";` is a valid, invisible formatting change — and with a bare-only +// matcher it would drop that table out of the guard's list and quietly reopen the hole this parser +// exists to close. +const DELETE_FROM = + /^DELETE\s+FROM\s+(?:"([^"]+)"|`([^`]+)`|\[([^\]]+)\]|([A-Za-z_][A-Za-z0-9_]*))\s*;?\s*$/i; + +/** + * The tables `scripts/normalize-raw.sql` empties before rebuilding the domain from staging, read out + * of the SQL rather than restated in JS. The guard that consumes this list used to ask about + * `contracts` alone while the clear had grown to fourteen tables — a hardcoded copy of a destructive + * list is a data-loss bug on a timer, so the list has exactly one home. + * + * Scoped to the `@full-clear` block on purpose: the same file later resets `data_freshness` and + * `pipeline_stats`, which are per-run metadata. Counting those as corpus would make the guard refuse + * every full derive, including the initial backfill it is supposed to let through. + */ +export function fullClearTables(normalizeRawSql: string): string[] { + const tables: string[] = []; + let inBlock = false; + for (const line of normalizeRawSql.split(/\r?\n/)) { + const trimmed = line.trim(); + if (FULL_CLEAR_MARKER.test(trimmed)) { + inBlock = true; + continue; + } + if (!inBlock) continue; + const hit = trimmed.match(DELETE_FROM); + if (hit) { + tables.push((hit[1] ?? hit[2] ?? hit[3] ?? hit[4])!); + continue; + } + // Comments and the DROP TABLEs share the block; a blank line ends it. + if (trimmed === '') break; + } + return tables; +} + export function dropTransientStagingStatements(): string[] { return [...TRANSIENT_STAGING_TABLES, ...LEGACY_TRANSIENT_STAGING_TABLES] .reverse() diff --git a/scripts/import-guard.test.mjs b/scripts/import-guard.test.mjs new file mode 100644 index 00000000..f82c8b8c --- /dev/null +++ b/scripts/import-guard.test.mjs @@ -0,0 +1,201 @@ +// The --derive=full window guard, exercised through the real scripts/import.mjs. +// +// Testing the predicate alone is what let the first version ship: fullDeriveIsSafe() was green while +// the call site asked `SELECT COUNT(*) FROM contracts` and the clear it was protecting emptied +// fourteen tables. So this drives the actual script as a subprocess, with a fake `wrangler` and a +// fake `node` first on PATH, and asserts on what the script DID: whether it refused, which tables it +// named, whether it cleaned up after itself, and whether the load ever started. +// +// Run: node --test scripts/import-guard.test.mjs +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(HERE, '..'); +const SCRIPT = resolve(HERE, 'import.mjs'); + +// `wrangler` answers the probe from GUARD_FAKE_POPULATED and records every call; `node` stands in for +// the child scripts import.mjs shells out to (load-eop and friends), so a run that gets PAST the guard +// stops here instead of hitting the network. Launching the script under test with process.execPath +// keeps the real node for the parent while the child's PATH lookup finds the stub. +// The shebangs name the real interpreter outright. `#!/usr/bin/env node` would resolve through the +// very PATH these stubs sit at the front of, so the fake `node` would end up interpreting the fake +// `wrangler` and every answer would come back empty. +const FAKE_WRANGLER = `#!${process.execPath} +import { appendFileSync } from 'node:fs'; +const argv = process.argv.slice(2); +appendFileSync(process.env.GUARD_FAKE_LOG, JSON.stringify(argv) + '\\n'); +const ci = argv.indexOf('--command'); +if (argv.includes('--json') && ci !== -1) { + const sql = argv[ci + 1]; + const populated = (process.env.GUARD_FAKE_POPULATED || '').split(',').filter(Boolean); + const missing = (process.env.GUARD_FAKE_MISSING || '').split(',').filter(Boolean); + const aliases = [...sql.matchAll(/AS "([^"]+)"/g)].map((m) => m[1]); + if (aliases.length) { + const row = {}; + for (const a of aliases) if (!missing.includes(a)) row[a] = populated.includes(a) ? 1 : 0; + process.stdout.write(JSON.stringify([{ results: [row], success: true }])); + process.exit(0); + } + process.stdout.write(JSON.stringify([{ results: [], success: true }])); +} +process.exit(0); +`; + +const FAKE_NODE = `#!${process.execPath} +import { appendFileSync } from 'node:fs'; +appendFileSync(process.env.GUARD_FAKE_LOG, JSON.stringify(['node', ...process.argv.slice(2)]) + '\\n'); +process.exit(0); +`; + +function bin(dir, name, source) { + const file = join(dir, name); + writeFileSync(file, source); + chmodSync(file, 0o755); +} + +/** Runs the real import.mjs with the fakes in front, and returns what it did. */ +function runImport(args, { populated = '', missing = '' } = {}) { + const dir = mkdtempSync(join(tmpdir(), 'guard-')); + try { + const binDir = join(dir, 'bin'); + mkdirSync(binDir); + writeFileSync(join(binDir, 'package.json'), '{"type":"module"}'); + bin(binDir, 'wrangler', FAKE_WRANGLER); + bin(binDir, 'node', FAKE_NODE); + const log = join(dir, 'calls.log'); + writeFileSync(log, ''); + const res = spawnSync(process.execPath, [SCRIPT, ...args], { + cwd: ROOT, + encoding: 'utf8', + timeout: 30_000, + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + GUARD_FAKE_LOG: log, + GUARD_FAKE_POPULATED: populated, + GUARD_FAKE_MISSING: missing, + SIGMA_D1_NAME: 'sigma-test-local', + }, + }); + const calls = readFileSync(log, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse); + return { + status: res.status, + stderr: res.stderr ?? '', + calls, + refused: /refusing --derive=full/.test(res.stderr ?? ''), + loadStarted: calls.some((c) => c[0] === 'node' && String(c[1]).includes('load-eop')), + // The guard's own query, not merely any --json call: the derive paths issue plenty of their own. + probe: calls.find((c) => /EXISTS\(SELECT 1 FROM contracts\)/.test(String(c[c.length - 1]))), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +const PARTIAL = ['--from=2026-06-01', '--derive=full']; + +test('refuses a partial-window full derive over a populated corpus, before the load', () => { + const r = runImport(PARTIAL, { populated: 'contracts,lots,tenders,bidders,authorities' }); + assert.equal(r.refused, true, r.stderr); + assert.equal(r.status, 1); + assert.equal(r.loadStarted, false, 'the load must not start after a refusal'); + assert.match(r.stderr, /contracts/); +}); + +test('a corpus with NO contracts but populated tenders still refuses', () => { + // The regression this guard exists for. The previous call site asked COUNT(*) FROM contracts, so + // exactly this state - the one a half-failed run leaves behind - was waved through, and the rebuild + // would then drop tenders, bidders and authorities with nothing to reload them from. + const r = runImport(PARTIAL, { populated: 'tenders,bidders,authorities' }); + assert.equal(r.refused, true, r.stderr); + assert.equal(r.status, 1); + assert.match(r.stderr, /tenders/); + assert.doesNotMatch(r.stderr, /including [^\n]*\bcontracts\b/); +}); + +test('refuses when the probe cannot answer for every cleared table', () => { + // A missing table makes safeD1 return nothing at all, which would otherwise read as "no corpus". + const r = runImport(PARTIAL, { populated: 'contracts', missing: 'facet_counts' }); + assert.equal(r.status, 1); + assert.match(r.stderr, /could not read the corpus/); +}); + +test('an empty corpus is the initial backfill and passes', () => { + const r = runImport(PARTIAL, { populated: '' }); + assert.equal(r.refused, false, r.stderr); + assert.equal(r.loadStarted, true, 'the load should start when there is nothing to lose'); +}); + +test('a window reaching the start of the feed passes even over a full corpus', () => { + const r = runImport(['--from=2020-01-01', '--derive=full'], { + populated: 'contracts,authorities', + }); + assert.equal(r.refused, false, r.stderr); + assert.equal(r.loadStarted, true); +}); + +test('a slice derive is never probed at all', () => { + const r = runImport(['--from=2026-06-01', '--derive=slice'], { populated: 'contracts' }); + assert.equal(r.refused, false, r.stderr); + assert.equal(r.probe, undefined, 'slice derives must not pay for the corpus probe'); +}); + +// Written out by hand ON PURPOSE. The first version of this test re-derived the list from +// normalize-raw.sql using its own copy of the parser's regex — so the oracle inherited the parser's +// blind spot, and rewriting one line as `DELETE FROM "search_index";` (valid SQLite, invisible in +// review) dropped that table out of the probe with both suites still green. A list that shares the +// implementation's assumptions cannot test them. +const CLEARED = [ + 'search_index', + 'flow_pairs', + 'company_totals', + 'authority_joint_participation', + 'authority_totals', + 'sector_totals', + 'facet_counts', + 'home_totals', + 'contract_co_authorities', + 'contracts', + 'lots', + 'tenders', + 'bidders', + 'authorities', +]; + +test('the probe covers every table the SQL clears, not a subset', () => { + const r = runImport(PARTIAL, { populated: 'contracts' }); + const sql = String(r.probe[r.probe.indexOf('--command') + 1]); + for (const table of CLEARED) { + assert.match(sql, new RegExp(`FROM ${table}\\)`), `probe is missing ${table}`); + } +}); + +test('the hand-written list still matches what the SQL clears', () => { + // The other half of the cross-check: the list above holds the parser to account, and this holds the + // list to account. Either one drifting is caught here instead of in production. The matcher is + // deliberately loose about quoting so that a re-quoted table shows up as a MISMATCH rather than + // vanishing the way it did from the first version. + const sql = readFileSync(resolve(ROOT, 'scripts/normalize-raw.sql'), 'utf8'); + const marker = sql.indexOf('-- @full-clear'); + assert.notEqual(marker, -1, 'normalize-raw.sql lost its @full-clear marker'); + const block = sql.slice(marker).split(/\r?\n\s*\r?\n/)[0]; + const deletes = [...block.matchAll(/DELETE\s+FROM\s+(.+?)\s*;/gi)].map((m) => + m[1].replace(/^["`[]/, '').replace(/["`\]]$/, ''), + ); + assert.deepEqual(deletes, CLEARED); +}); + +test('a refusal leaves no transient staging behind', () => { + const r = runImport(PARTIAL, { populated: 'contracts' }); + const files = r.calls.filter((c) => c.includes('--file')).map((c) => c[c.indexOf('--file') + 1]); + assert.ok( + files.some((f) => /drop-transient-staging\.sql$/.test(String(f))), + 'the guard should tear down the staging it found created', + ); +}); diff --git a/scripts/import.mjs b/scripts/import.mjs index 578a1132..43ec75c5 100644 --- a/scripts/import.mjs +++ b/scripts/import.mjs @@ -14,9 +14,14 @@ import { } from 'node:fs'; import { basename, dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { computeCatchupWindow, daysInWindow } from '../packages/ingest/src/ocds.ts'; +import { + computeCatchupWindow, + daysInWindow, + fullDeriveIsSafe, +} from '../packages/ingest/src/ocds.ts'; import { dropTransientStagingStatements, + fullClearTables, refreshSliceStatementGroups, } from '../packages/ingest/src/refresh.ts'; import { assertIntegrity } from './integrity-checks.mjs'; @@ -37,7 +42,6 @@ function reportAnomalies(runner, label) { const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const apiDir = resolve(root, 'apps/web'); const DEFAULT_FROM = '2020-01-01'; -const LARGE_GAP_DAYS = 14; const DEFAULT_LOOKBACK_DAYS = 3; const remote = process.argv.includes('--remote'); @@ -215,12 +219,11 @@ function resolveCatchupPlan() { const to = String(arg('to') || window.to); const gapDays = daysInWindow(from, to); const requestedDerive = arg('derive'); - const derive = - requestedDerive && requestedDerive !== true - ? String(requestedDerive) - : gapDays > LARGE_GAP_DAYS - ? 'full' - : 'slice'; + // The catch-up window is gap-aware, so it only ever covers the tail of the feed. A full derive + // rebuilds `contracts` from staging (normalize-raw.sql opens with DELETE FROM contracts), which + // would drop every contract older than the window — so catch-up always derives a slice, however + // wide the gap. An operator who really has loaded the whole feed can still pass --derive=full. + const derive = requestedDerive && requestedDerive !== true ? String(requestedDerive) : 'slice'; return { from, to, maxLoadedDate, gapDays, derive }; } @@ -229,6 +232,57 @@ function validateDeriveMode(mode) { throw new Error(`unknown --derive=${mode}; expected full|slice`); } +// Refuse the one combination that silently destroys data: a full derive (which rebuilds the domain +// from staging) driven by a window that does not reach back to the start of the feed. Anything the +// window misses is deleted and never reloaded. Checked before the load, and the refusal tears the +// transient staging back down: it cannot run any earlier, because the catch-up plan reads +// raw_contracts, but it must not leave a half-built schema behind for a run that never started. +// +// The question is asked of EVERY table the full clear empties, not of `contracts` alone: a corpus +// with no contracts but populated tenders, bidders or authorities is exactly the state a half-failed +// run leaves behind, and the narrow-window rebuild would then wipe those too while the guard waved +// it through. The list comes out of normalize-raw.sql itself (see @full-clear there). +function assertDeriveWindowSafe(mode, from) { + if (mode !== 'full') return; + const tables = fullClearTables(readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8')); + if (tables.length === 0) + throw new Error( + 'normalize-raw.sql has no @full-clear block — refusing to guess what it clears', + ); + // EXISTS per table, not COUNT(*) over the union: this runs on every full derive and must stay + // cheap on a corpus of hundreds of thousands of rows. + const probe = tables.map((t) => `(SELECT EXISTS(SELECT 1 FROM ${t})) AS "${t}"`).join(', '); + const row = safeD1(`SELECT ${probe}`)[0]; + // safeD1 turns a missing table into an empty result, which would otherwise read as "no corpus" and + // wave the destructive path through — one absent table blinding the guard about the other thirteen. + // A probe that could not answer is not an answer: fail closed. + if (!row || tables.some((table) => !(table in row))) { + console.error( + `!! refusing --derive=full: could not read the corpus. Every table normalize-raw.sql clears ` + + `(${tables.join(', ')}) must be answerable before a rebuild may drop it.`, + ); + execSqlStatements(dropTransientStagingStatements(), 'drop-transient-staging'); + process.exit(1); + } + const populated = Object.entries(row) + .filter(([, present]) => Number(present) > 0) + .map(([table]) => table); + if ( + fullDeriveIsSafe({ windowFrom: from, feedStart: DEFAULT_FROM, hasCorpus: populated.length > 0 }) + ) + return; + console.error( + `!! refusing --derive=full: the load window starts ${from}, but the corpus is already ` + + `populated back to ${DEFAULT_FROM}.\n` + + ` A full derive rebuilds the domain from staging, so everything before ${from} would be ` + + `dropped and not reloaded — including ${populated.join(', ')}.\n` + + ` Use --derive=slice for an incremental refresh, or reload the whole feed with ` + + `--from=${DEFAULT_FROM}.`, + ); + execSqlStatements(dropTransientStagingStatements(), 'drop-transient-staging'); + process.exit(1); +} + async function runFullDerive() { execSql(resolve(root, 'scripts/derive-amendments.sql')); run('node', ['scripts/load-fx.mjs', '--apply', ...passthru]); @@ -370,19 +424,24 @@ if (arg('work-db') !== undefined) { console.log(`==> Sigma import (${remote ? 'REMOTE' : 'local'})`); run('wrangler', ['d1', 'migrations', 'apply', d1Name, loc, ...d1PersistArgs], apiDir); execSqlStatements(dropTransientStagingStatements(), 'drop-stale-transient-staging'); +// Must precede resolveCatchupPlan(): latestLoadedDate() reads raw_contracts, which lives here. execSql(resolve(root, 'scripts/work-staging-schema.sql')); let deriveMode = String(arg('derive') || 'full'); let loadFlags = explicitRangeFlags(); +// Mirrors load-eop.mjs, which also falls back to DEFAULT_FROM when no --from is given. +let windowFrom = String(arg('from') || DEFAULT_FROM); if (catchup) { const plan = resolveCatchupPlan(); deriveMode = plan.derive; loadFlags = rangeFlags(plan.from, plan.to); + windowFrom = plan.from; console.log( `==> catchup window ${plan.from}..${plan.to} (${plan.gapDays} days, latest=${plan.maxLoadedDate || 'none'}, derive=${deriveMode})`, ); } validateDeriveMode(deriveMode); +assertDeriveWindowSafe(deriveMode, windowFrom); run('node', ['scripts/load-eop.mjs', '--apply', ...loadFlags, ...passthru]); if (deriveMode === 'slice') await runSliceDerive(); diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index ebfef975..68c3dbf5 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -31,6 +31,10 @@ -- every contract has a parent. bids stays empty (the data has a bid COUNT, not bids). -- Full clear in child→parent order (D1 enforces FKs). +-- @full-clear — everything emptied between this marker and the blank line below is rebuilt from +-- staging alone, so a full derive driven by a partial window loses whatever the window misses. +-- scripts/import.mjs reads the list from here to decide whether refusing is warranted; keep new +-- DELETEs inside the block so the guard picks them up on its own. DROP TABLE IF EXISTS joint_tender_leads; DROP TABLE IF EXISTS unp_prefix_authorities; DROP TABLE IF EXISTS joint_authority_members; From 8db751d97610e8ef8f09be726132bfbd33fd29ba Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Sat, 8 Aug 2026 00:30:33 +0000 Subject: [PATCH 35/69] fix(etl): detect wrangler's SQLITE errors from stdout in safeD1 (#277) safeD1 tested err.message for "no such table", but wrangler prints the SQLite error as JSON on stdout and leaves the exception message as a bare "Command failed: wrangler d1 execute ...". The guard therefore never matched, safeD1 rethrew instead of returning [], and the data_freshness fallback in latestLoadedDate was unreachable. The visible effect is that `pnpm run import --catchup --plan-only` crashes whenever the transient staging tables are absent, which is their normal steady state: drop-transient-staging removes them in a finally, and --plan-only exits before the main flow recreates them. Ordinary runs are unaffected, since they create an empty raw_contracts first and so reach the fallback. Cherry-picked verbatim from e3fc6b3 in #188 by Bilko (StanislavBG), split out so the fix can land without the contract health index feature. Co-authored-by: Bilko --- scripts/import.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/import.mjs b/scripts/import.mjs index 43ec75c5..a841759c 100644 --- a/scripts/import.mjs +++ b/scripts/import.mjs @@ -131,7 +131,8 @@ function safeD1(sql) { try { return d1(sql); } catch (err) { - const msg = String(err?.message ?? err); + // wrangler writes the SQLITE error to stdout, not the exception message. + const msg = `${err?.message ?? err} ${err?.stdout ?? ''} ${err?.stderr ?? ''}`; if (/no such table|does not exist/i.test(msg)) return []; throw err; } From 3e047c6b27550334c38bf7079a975b96a174ea49 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Sun, 9 Aug 2026 11:07:33 +0000 Subject: [PATCH 36/69] =?UTF-8?q?docs(scripts):=20=D0=BF=D0=BE=D0=B4=D1=81?= =?UTF-8?q?=D0=B8=D0=B3=D1=83=D1=80=D1=8F=D0=B2=D0=B0=D0=BD=D0=B5,=20?= =?UTF-8?q?=D0=B0=20=D0=BD=D0=B5=20=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=BD=D0=B0=20=D0=BD=D0=B0=D0=B1=D0=BB=D1=8E?= =?UTF-8?q?=D0=B4=D0=B0=D0=B2=D0=B0=D0=BD=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B0?= =?UTF-8?q?=D0=BB=20(#293)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Коментарът твърдеше, че известието на wrangler застава пред JSON-а и е превръщало здраво качване в провал при сверката. Проверено срещу истинския инструмент - не е вярно: `wrangler d1 execute --json` пише известията на stderr, stdout остава чист JSON, а execFileSync връща само stdout. Рязането от първата скоба е било безопасно през цялото време. Грешката дойде от подставката в теста, която печаташе известието на stdout. Наборът беше зелен, защото проверяваше кода срещу неверен модел на инструмента - мутациите не хващат това, те гледат дали твърденията са вързани за кода, а не дали подставката прилича на истината. `parseWranglerJson` остава - евтино е и покрива бъдещо издание, което мести известията. Само вече не се представя за нещо, което не е. Тестът казва изрично, че моделира поток, какъвто wrangler днес не произвежда. --- scripts/ship-e2e.test.mjs | 11 ++++++++--- scripts/ship-related-persons.mjs | 12 +++++++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/scripts/ship-e2e.test.mjs b/scripts/ship-e2e.test.mjs index af27ccf1..cf971a2e 100644 --- a/scripts/ship-e2e.test.mjs +++ b/scripts/ship-e2e.test.mjs @@ -275,9 +275,14 @@ test('a bracketed notice on stdout does not corrupt the read-back', (t) => { const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-noise-')); t.after(() => rmSync(dir, { recursive: true, force: true })); - // The script slices from the first '[' to skip wrangler's preamble — a `[WARNING]` line would - // send that slice to the wrong offset. Real wrangler --json keeps notices on stderr, but the - // defensive slice exists precisely because that has not always been true. + // This is a HYPOTHETICAL, and saying so is the point. `wrangler d1 execute --json` was run against + // the real tool afterwards: the notice goes to stderr, stdout is clean JSON, and execFileSync + // returns stdout alone. So this models a stream layout wrangler does not currently produce. + // + // It stays as a cheap guard against a future release moving notices onto stdout — but the PR that + // added it billed the scan as fixing an observed failure, which it never was. A fake is a claim + // about the world; this one went unchecked, the suite was green, and the false claim shipped. + // Anything modelled here that has not been confirmed against the real binary gets labelled as such. const { res } = runShip(dir, { env: { SHIP_FAKE_NOISE: '1' } }); assert.equal(res.status, 0, `a stdout notice broke the read-back:\n${res.stderr}`); }); diff --git a/scripts/ship-related-persons.mjs b/scripts/ship-related-persons.mjs index baf6646a..4188f6c8 100644 --- a/scripts/ship-related-persons.mjs +++ b/scripts/ship-related-persons.mjs @@ -277,7 +277,8 @@ export function assertD1TargetAuthorized({ remote, shipEnv, d1Name, expectedId, * fails — `assertShippedCounts` then reports every table as unanswered and fails the run, which is the * right way round: a verification step that cannot verify must not pass. */ -/** First bracket that actually parses — notices may contain '[' of their own. Exported for the test. */ +/** First bracket that actually parses — a notice on the same stream could contain one of its own. + * Belt-and-braces: today wrangler keeps notices on stderr (see the call site). Exported for the test. */ export function parseWranglerJson(out) { for (let i = out.indexOf('['); i >= 0; i = out.indexOf('[', i + 1)) { try { @@ -303,10 +304,11 @@ function readShippedCounts(d1Name, remote, expected) { ['d1', 'execute', d1Name, remote ? '--remote' : '--local', '--json', '--command', sql], { cwd: resolve('apps/web'), encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }, ); - // wrangler prefixes its own notices before the JSON payload; take the document, not the whole - // stream. Slicing from the FIRST '[' is not enough — a notice can itself contain one („▲ - // [WARNING] Processing wrangler.jsonc"), which sliced to the wrong offset and turned a perfectly - // healthy ship into a verification failure. Try each candidate and keep the first that parses. + // Defensive, NOT a fix for an observed failure — the earlier wording here claimed otherwise and + // was wrong. Checked against the real tool: `wrangler d1 execute --json` writes its notices + // („▲ [WARNING] Processing wrangler.jsonc") to STDERR and leaves stdout as clean JSON, and + // execFileSync returns stdout alone, so slicing from the first '[' is in fact safe today. The + // scan below survives a future release that changes that, and costs one failed parse if it does. const parsed = parseWranglerJson(out); const rows = (Array.isArray(parsed) ? parsed[0]?.results : parsed?.results) ?? []; // Only a real number counts as an answer. `Number(null)` is 0, which would let a null-valued cell pass From 4f4ff6031b81b96050ecc51ac85951fc06ac1807 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Sun, 9 Aug 2026 12:31:00 +0000 Subject: [PATCH 37/69] =?UTF-8?q?test(scripts):=20=D0=BF=D0=BE=D0=BA=D1=80?= =?UTF-8?q?=D0=B8=D0=B9=20=D1=80=D0=B0=D0=B7=D0=BF=D0=BE=D0=B7=D0=BD=D0=B0?= =?UTF-8?q?=D0=B2=D0=B0=D0=BD=D0=B5=D1=82=D0=BE=20=D0=BD=D0=B0=20=D0=BB?= =?UTF-8?q?=D0=B8=D0=BF=D1=81=D0=B2=D0=B0=D1=89=D0=B0=20=D1=82=D0=B0=D0=B1?= =?UTF-8?q?=D0=BB=D0=B8=D1=86=D0=B0=20=D0=B2=20safeD1=20(#295)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Поправката от #277 нямаше нищо, което да я пази: `import.mjs` изпълнява код при внасяне, тъй че `safeD1` не се тества модулно. Сглобката от #270 обаче кара истинския скрипт като отделен процес с подставен `wrangler`, и това е достатъчно. Подставката е сверена срещу истинския двоичен файл, а не измислена - при `d1 execute --json` върху липсваща таблица грешката от SQLite излиза на stdout, известията на stderr, а съобщението на изключението е само „Command failed". Точно това разминаване изпускаше старата проверка. Наблюдаваният изход е записан в теста. Мутациите: връщането на проверката само върху `err.message` вали два теста, а превръщането на safeD1 в общ капан вали третия. --- scripts/import-catchup.test.mjs | 117 ++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 scripts/import-catchup.test.mjs diff --git a/scripts/import-catchup.test.mjs b/scripts/import-catchup.test.mjs new file mode 100644 index 00000000..dc434472 --- /dev/null +++ b/scripts/import-catchup.test.mjs @@ -0,0 +1,117 @@ +// safeD1's missing-table detection, exercised through the real scripts/import.mjs. +// +// safeD1 has to tell "this table does not exist yet" (recoverable — latestLoadedDate falls back to +// data_freshness) apart from every other failure (must propagate). Getting that wrong breaks +// `import --catchup --plan-only` in its NORMAL state: drop-transient-staging removes raw_contracts in +// a finally, and --plan-only exits before the main flow recreates it. #277 fixed it; nothing tested it, +// because import.mjs runs on import and cannot be imported. The subprocess harness from #270 can. +// +// Run: node --test scripts/import-catchup.test.mjs +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(HERE, '..'); +const SCRIPT = resolve(HERE, 'import.mjs'); + +// VERIFIED AGAINST THE REAL BINARY, not imagined — the whole point of this test is the stream layout, +// so modelling it from memory would test nothing. Observed from +// `wrangler d1 execute sigma --local --json --command "SELECT 1 FROM no_such_table_xyz"`: +// +// exit code : 1 +// stdout : {"error":{"text":"no such table: no_such_table_xyz: SQLITE_ERROR"}} +// stderr : ▲ [WARNING] Processing wrangler.jsonc configuration: ... +// err.message from execFileSync : "Command failed: wrangler d1 execute ..." +// +// So the SQLite error is on STDOUT while wrangler's notices are on STDERR, and the exception message +// carries neither. That asymmetry is exactly what the old `err.message`-only check missed. +const SQLITE_ERROR_ON_STDOUT = '{"error":{"text":"no such table: raw_contracts: SQLITE_ERROR"}}'; + +const FAKE_WRANGLER = `#!${process.execPath} +import { appendFileSync } from 'node:fs'; +const argv = process.argv.slice(2); +appendFileSync(process.env.CU_LOG, JSON.stringify(argv) + '\\n'); +const ci = argv.indexOf('--command'); +if (ci !== -1) { + const sql = argv[ci + 1]; + if (/raw_contracts/.test(sql)) { + if (process.env.CU_OTHER_ERROR) { + process.stdout.write(JSON.stringify({ error: { text: 'database is locked: SQLITE_BUSY' } })); + } else { + process.stdout.write(${JSON.stringify(SQLITE_ERROR_ON_STDOUT)}); + } + process.stderr.write('\\u25b2 [WARNING] Processing wrangler.jsonc configuration:\\n'); + process.exit(1); + } + if (/data_freshness/.test(sql)) { + process.stdout.write(JSON.stringify([{ results: [{ max_loaded_date: '2026-07-27' }], success: true }])); + process.exit(0); + } + process.stdout.write(JSON.stringify([{ results: [], success: true }])); +} +process.exit(0); +`; + +// Stands in for the child scripts import.mjs shells out to, so a run that gets past planning stops +// here rather than reaching the network. The parent runs under process.execPath, so it keeps the real +// node while the child's PATH lookup finds this. +const FAKE_NODE = `#!${process.execPath} +process.exit(0); +`; + +function runImport(args, env = {}) { + const dir = mkdtempSync(join(tmpdir(), 'catchup-')); + try { + const binDir = join(dir, 'bin'); + mkdirSync(binDir); + writeFileSync(join(binDir, 'package.json'), '{"type":"module"}'); + for (const [name, src] of [ + ['wrangler', FAKE_WRANGLER], + ['node', FAKE_NODE], + ]) { + writeFileSync(join(binDir, name), src); + chmodSync(join(binDir, name), 0o755); + } + const log = join(dir, 'calls.log'); + writeFileSync(log, ''); + const res = spawnSync(process.execPath, [SCRIPT, ...args], { + cwd: ROOT, + encoding: 'utf8', + timeout: 30_000, + env: { ...process.env, PATH: `${binDir}:${process.env.PATH}`, CU_LOG: log, ...env }, + }); + const calls = readFileSync(log, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse); + return { status: res.status, stdout: res.stdout ?? '', stderr: res.stderr ?? '', calls }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +test('a missing raw_contracts falls back to data_freshness instead of crashing', () => { + // The reported failure: --plan-only in its normal steady state, with the transient staging gone. + const r = runImport(['--catchup', '--plan-only']); + assert.equal(r.status, 0, `--plan-only should survive a missing raw_contracts:\n${r.stderr}`); + assert.match(r.stdout, /catchup plan/); + // The date can only have come from the data_freshness fallback — raw_contracts never answered. + assert.match(r.stdout, /maxLoadedDate=2026-07-27/); +}); + +test('the fallback query is actually reached, not skipped', () => { + // Guards against a future "fix" that returns a plan without asking data_freshness at all, which + // would pass the assertion above by accident. + const r = runImport(['--catchup', '--plan-only']); + const asked = r.calls.some((c) => /data_freshness/.test(String(c[c.length - 1]))); + assert.ok(asked, 'safeD1 swallowed the error but nothing consulted data_freshness'); +}); + +test('an error that is NOT a missing table still propagates', () => { + // safeD1 must not become a blanket catch. A locked database is not "no corpus yet", and treating it + // as one would let the catch-up plan be computed from a lie. + const r = runImport(['--catchup', '--plan-only'], { CU_OTHER_ERROR: '1' }); + assert.notEqual(r.status, 0, 'a SQLITE_BUSY must not be swallowed as an empty result'); +}); From 73e055df47a9cfda9c49ac04649b73e84f2b0c52 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Sun, 9 Aug 2026 14:19:17 +0000 Subject: [PATCH 38/69] =?UTF-8?q?ci:=20=D0=B7=D0=B0=D0=B1=D1=80=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=20Co-Authored-By=20=D0=BA=D1=8A=D0=BC=20=D0=B0?= =?UTF-8?q?=D0=B3=D0=B5=D0=BD=D1=82,=20=D0=B7=D0=B0=D0=BF=D0=B0=D0=B7?= =?UTF-8?q?=D0=B8=20=D1=82=D1=80=D0=B5=D0=B9=D0=BB=D1=8A=D1=80=D0=B8=D1=82?= =?UTF-8?q?=D0=B5=20=D1=81=20=D1=85=D0=BE=D1=80=D0=B0=20(#296)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: забрани трейлър, който кредитира агент, и запази тези с хора Правилото в AGENTS.md се четеше като пълна забрана на `Co-Authored-By:`, а буквалното му спазване би изтрило заслугата на сътрудниците: при squash GitHub съставя трейлърите от авторите на коммитите в PR-а и те са единственото, което държи външния автор в историята - авторът на самия squash коммит винаги е този, който е отворил PR-а. 22 от 376 коммита на main ги носят, включително тези, които пазят заслугата на StanislavBG, Румен и Йоан. Забраненото е друго: трейлър, който кредитира агент - Claude Code, Codex, Cursor, Copilot. Те са инструменти, не сътрудници. Проверката ги лови на ниво PR и казва как се оправя, без да праща никого да пренаписва чужд форк. Тестовете карат проверката срещу истинска история: човешки съавтор и dependabot не се маркират, а трейлърът на Cursor в PR #118 се маркира. Пътьом: post-create.sh предупреждава при самоличност като `t@e.com` или `...MacBook-Pro.local`. И двете вече са влизали в публичната история точно по този път. * style(scripts): форматиране по prettier * ci: премести проверката в задължителната работа, за да блокира наистина Отделната работа се вижда в списъка, но правилото на main изисква само `check`. Значи червен кръст, който нищо не спира - точно класът „проверка, която изглежда, че пази". Стъпката вече е вътре в `check`, тъй че отказът е реален и не зависи от админска промяна по правилото. --- .devcontainer/post-create.sh | 13 +++++ .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/ci.yml | 20 +++++++ AGENTS.md | 4 +- scripts/check-agent-trailers.mjs | 75 ++++++++++++++++++++++++++ scripts/check-agent-trailers.test.mjs | 76 +++++++++++++++++++++++++++ 6 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 scripts/check-agent-trailers.mjs create mode 100644 scripts/check-agent-trailers.test.mjs diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index 9478f075..51aabfe3 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -21,4 +21,17 @@ if [ ! -f .dev.vars ] && [ -f .dev.vars.example ]; then echo "==> Copied .dev.vars.example → .dev.vars (fill in real keys before pnpm dev)" fi +# A placeholder git identity does not stay local: GitHub's squash merge turns every distinct commit +# author in a PR into a `Co-authored-by:` line on `main`, so `t ` and +# `you@Your-MacBook.local` end up permanently in the public history. Both have already happened here. +# Warn rather than set anything — the right identity is the developer's, not this script's. +git_email="$(git config user.email || true)" +case "$git_email" in + '' | *@e.com | *.local | *@localhost | *localdomain*) + echo "==> WARNING: git user.email is '${git_email:-unset}', which looks like a placeholder." + echo " Commits made with it land in the public history as a stray Co-authored-by line." + echo " Set it before committing: git config user.email you@example.com" + ;; +esac + echo "==> Done. Next: pnpm run setup, then pnpm dev" diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index dbee8724..f01f1b8a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -24,7 +24,7 @@ ## Чеклист -- [ ] Комитите следват [conventional commits](https://www.conventionalcommits.org) и **нямат** `Co-Authored-By:` trailer +- [ ] Комитите следват [conventional commits](https://www.conventionalcommits.org) и **нямат** `Co-Authored-By:` trailer, който сочи към агент (Claude Code, Codex, Cursor, Copilot). Трейлъри с **хора** са наред и не се махат — те са начинът заслугата на сътрудника да оцелее при squash - [ ] PR-ът е с **един логически обхват** и е от форк към `midt-bg/sigma:main` - [ ] `pnpm typecheck` минава - [ ] `pnpm test` (поне за засегнатите пакети) минава diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cebb18a..2ba34161 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,26 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Full history so the agent-trailer step below can walk base..head; the default PR + # checkout is a shallow merge ref with neither endpoint reachable. + fetch-depth: 0 + + # `Co-authored-by:` naming a coding agent (Claude Code, Codex, Cursor, Copilot …). Trailers + # naming PEOPLE are deliberately left alone: GitHub's squash merge builds them from the PR's + # commit authors and they are the only thing that keeps a contributor's credit on main, since + # the squash commit's own author is always the PR opener. Dependency bots are exempt likewise. + # Fixing a hit never requires touching a contributor's fork — see the script's own advice. + # + # Deliberately a STEP of `check` rather than a job of its own: `check` is the only context the + # main ruleset actually requires, so a separate job would show a red X that nothing enforces. + # Runs first — it costs a second and needs no install. + - name: No agent co-author trailers + if: github.event_name == 'pull_request' + run: node scripts/check-agent-trailers.mjs "$BASE" "$HEAD" + env: + BASE: ${{ github.event.pull_request.base.sha }} + HEAD: ${{ github.event.pull_request.head.sha }} # Secret scan FIRST, on the pristine checkout (tracked files only) — before `pnpm install` # creates node_modules. The gitleaks *Action* requires a paid GITLEAKS_LICENSE on GitHub diff --git a/AGENTS.md b/AGENTS.md index 8ed755cf..064ff65d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,9 @@ No `develop`, no `staging`. Maintainers with write access work on short-lived fe - Use [conventional commits](https://www.conventionalcommits.org): `(): `. Types: `feat`, `fix`, `docs`, `refactor`, `test`, `build`, `ci`, `chore`, `perf`, `style`. Subject is lowercase imperative, no trailing period. - Use the `/smart-commit` and `/suggest-commit` skills when drafting messages. They produce the canonical format for this repo. -- **Never include `Co-Authored-By:` trailers.** Keep the history clean; CI may grep for this. +- **Never credit a coding agent in a `Co-Authored-By:` trailer** — Claude Code, Codex, Cursor, Copilot and the like. They are tools we drive, not contributors, and naming them misrepresents who wrote the code. CI enforces this (`scripts/check-agent-trailers.mjs`). +- Trailers naming **people** are fine, and mostly not ours to write: GitHub's squash merge generates them from the PR's commit authors, and they are the only thing that keeps a contributor's credit on `main` — the squash commit's own author is always the PR opener, whoever wrote the code. Do not strip those. Dependency bots (dependabot, renovate) are left alone for the same reason. +- If an agent trailer does reach a PR, the maintainer merging it edits the squash message (`gh pr merge --squash --body "…"`). Never force-push a contributor's fork to clean this up. - Small, focused commits are encouraged. Commit as you go — not all at the end. Easier to review and revert. Don't mix unrelated changes in one commit. ## Pull requests diff --git a/scripts/check-agent-trailers.mjs b/scripts/check-agent-trailers.mjs new file mode 100644 index 00000000..9b2eda39 --- /dev/null +++ b/scripts/check-agent-trailers.mjs @@ -0,0 +1,75 @@ +#!/usr/bin/env node +// Refuse `Co-authored-by:` trailers that credit a CODING AGENT. +// +// The repo convention is narrower than it used to read. Trailers naming HUMANS are not just allowed, +// they are load-bearing: GitHub's squash merge builds them from the PR's commit authors, and they are +// the only reason an external contributor keeps credit on `main` (the squash commit's own author is +// always the PR opener). Stripping those would erase the people this project runs on. +// +// What must not appear is an agent: Claude Code, Codex, Cursor, Copilot and friends. They are tools +// the maintainers drive, not contributors, and crediting them as co-authors misrepresents who wrote +// the code. Dependency and CI bots (dependabot, renovate) are deliberately NOT in the list — their +// trailers are how their own automated PRs are attributed, which is a different thing. +// +// node scripts/check-agent-trailers.mjs +// +// Exits 1 and prints the offending commits when it finds any. +import { execFileSync } from 'node:child_process'; + +// Matched case-insensitively against the whole trailer line, so either the display name or the +// address is enough. Extend as new agents appear; keep humans and dependency bots out of it. +const AGENTS = [ + 'claude', + 'anthropic\\.com', + 'codex', + 'openai\\.com', + 'cursor', + 'copilot', + 'devin', + 'aider', + 'sourcegraph\\.com', + 'sweep-ai', +]; + +const TRAILER = /^\s*co-authored-by:\s*(.+)$/gim; +const AGENT_RE = new RegExp(AGENTS.join('|'), 'i'); + +const [base, head] = process.argv.slice(2); +if (!base || !head) { + console.error('usage: check-agent-trailers.mjs '); + process.exit(2); +} + +// %H then the body, one record per commit, NUL-separated so multi-line bodies stay intact. +const log = execFileSync('git', ['log', `${base}..${head}`, '--format=%H%n%B%x00'], { + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, +}); + +const offenders = []; +for (const record of log.split('\0')) { + const trimmed = record.trim(); + if (!trimmed) continue; + const [sha, ...rest] = trimmed.split('\n'); + const body = rest.join('\n'); + for (const [, who] of body.matchAll(TRAILER)) { + if (AGENT_RE.test(who)) offenders.push({ sha: sha.slice(0, 8), who: who.trim() }); + } +} + +if (offenders.length === 0) { + console.log('no agent co-author trailers'); + process.exit(0); +} + +console.error('Co-authored-by trailers crediting a coding agent:\n'); +for (const o of offenders) console.error(` ${o.sha} Co-authored-by: ${o.who}`); +console.error(` +Agents are tools we drive, not contributors. Trailers naming PEOPLE are fine and should stay - +they are how a contributor keeps credit through a squash merge. + +To fix without rewriting anyone's branch: the maintainer merging the PR can drop these lines from +the squash message (\`gh pr merge --squash --body "..."\`), which never touches the contributor's +fork. Rewriting published commits is not required and, on a fork, not wanted. +`); +process.exit(1); diff --git a/scripts/check-agent-trailers.test.mjs b/scripts/check-agent-trailers.test.mjs new file mode 100644 index 00000000..304473e8 --- /dev/null +++ b/scripts/check-agent-trailers.test.mjs @@ -0,0 +1,76 @@ +// The agent-trailer guard, exercised against real commit ranges in this repository. +// +// A denylist check is worth exactly as much as its two failure modes: it must fire on the thing it +// exists for, and it must stay silent on the humans and dependency bots whose trailers are how their +// credit survives a squash merge. Both are asserted here against real history, not fixtures, because +// the interesting cases already exist on the remote. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(HERE, '..'); +const SCRIPT = resolve(HERE, 'check-agent-trailers.mjs'); + +const run = (base, head) => + spawnSync(process.execPath, [SCRIPT, base, head], { cwd: ROOT, encoding: 'utf8' }); + +/** A commit that exists locally, or null — keeps the suite green on a shallow clone. */ +function have(rev) { + const r = spawnSync('git', ['rev-parse', '--verify', `${rev}^{commit}`], { + cwd: ROOT, + encoding: 'utf8', + }); + return r.status === 0 ? r.stdout.trim() : null; +} + +test('a range with no agent trailers passes', (t) => { + const head = have('HEAD'); + const base = have('HEAD~1'); + if (!base || !head) return t.skip('shallow clone'); + const r = run(base, head); + assert.equal(r.status, 0, r.stderr); +}); + +test('a human co-author is NOT flagged', (t) => { + // 8db751d carries `Co-authored-by: Bilko ` — the exact shape that must + // survive, since it is the only thing keeping his credit on main after the squash. + const c = have('8db751d'); + if (!c) return t.skip('commit not present locally'); + const r = run(`${c}~1`, c); + assert.equal(r.status, 0, `a human co-author was flagged:\n${r.stderr}`); +}); + +test('a dependency bot is NOT flagged', (t) => { + // dependabot's trailer is how its own PRs are attributed. It is automation, not a coding agent. + const c = have('0d9d0ad'); + if (!c) return t.skip('commit not present locally'); + const r = run(`${c}~1`, c); + assert.equal(r.status, 0, `dependabot was flagged:\n${r.stderr}`); +}); + +test('an agent co-author IS flagged, and named in the output', (t) => { + // PR #118 really does carry `Co-authored-by: Cursor `. Fetched on demand so + // the assertion is about a real contributor branch rather than a fixture written to match the regex. + const fetched = spawnSync('git', ['fetch', '--quiet', '--depth=50', 'origin', 'pull/118/head'], { + cwd: ROOT, + encoding: 'utf8', + }); + if (fetched.status !== 0) return t.skip('cannot reach the PR ref'); + const head = execFileSync('git', ['rev-parse', 'FETCH_HEAD'], { + cwd: ROOT, + encoding: 'utf8', + }).trim(); + const base = execFileSync('git', ['rev-list', '--max-count=1', `${head}~40`], { + cwd: ROOT, + encoding: 'utf8', + }).trim(); + const r = run(base, head); + assert.equal(r.status, 1, 'the Cursor trailer should have failed the check'); + assert.match(r.stderr, /Cursor/i); + // The advice matters as much as the verdict: the fix must not send anyone at a contributor's fork + // with a force push. + assert.match(r.stderr, /gh pr merge --squash/); +}); From 8689212203bbbc573c71ab8a2cf1072d550dd4cc Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Sun, 9 Aug 2026 14:48:40 +0000 Subject: [PATCH 39/69] =?UTF-8?q?chore:=20=D0=BC=D0=B0=D1=85=D0=BD=D0=B8?= =?UTF-8?q?=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D0=BA=D0=B0=D1=82=D0=B0?= =?UTF-8?q?=20=D0=B7=D0=B0=20=D1=82=D1=80=D0=B5=D0=B9=D0=BB=D1=8A=D1=80?= =?UTF-8?q?=D0=B8,=20=D0=BF=D1=80=D0=B0=D0=B2=D0=B8=D0=BB=D0=BE=D1=82?= =?UTF-8?q?=D0=BE=20=D0=BE=D1=81=D1=82=D0=B0=D0=B2=D0=B0=20=D0=B2=20AGENTS?= =?UTF-8?q?.md=20(#297)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Правилото е на един ред в AGENTS.md и в шаблона за PR. Отделна проверка за него не се изплаща: искаше пълна история при всеки PR в задължителната работа `check`, добавяше начин тя да падне по причина, нямаща общо с прегледания код, и щеше да блокира вече отворен PR на сътрудник заради трейлър, който се маха с един ред в squash съобщението. --- .devcontainer/post-create.sh | 13 ----- .github/workflows/ci.yml | 20 ------- AGENTS.md | 4 +- scripts/check-agent-trailers.mjs | 75 -------------------------- scripts/check-agent-trailers.test.mjs | 76 --------------------------- 5 files changed, 1 insertion(+), 187 deletions(-) delete mode 100644 scripts/check-agent-trailers.mjs delete mode 100644 scripts/check-agent-trailers.test.mjs diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index 51aabfe3..9478f075 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -21,17 +21,4 @@ if [ ! -f .dev.vars ] && [ -f .dev.vars.example ]; then echo "==> Copied .dev.vars.example → .dev.vars (fill in real keys before pnpm dev)" fi -# A placeholder git identity does not stay local: GitHub's squash merge turns every distinct commit -# author in a PR into a `Co-authored-by:` line on `main`, so `t ` and -# `you@Your-MacBook.local` end up permanently in the public history. Both have already happened here. -# Warn rather than set anything — the right identity is the developer's, not this script's. -git_email="$(git config user.email || true)" -case "$git_email" in - '' | *@e.com | *.local | *@localhost | *localdomain*) - echo "==> WARNING: git user.email is '${git_email:-unset}', which looks like a placeholder." - echo " Commits made with it land in the public history as a stray Co-authored-by line." - echo " Set it before committing: git config user.email you@example.com" - ;; -esac - echo "==> Done. Next: pnpm run setup, then pnpm dev" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ba34161..9cebb18a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,26 +38,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - # Full history so the agent-trailer step below can walk base..head; the default PR - # checkout is a shallow merge ref with neither endpoint reachable. - fetch-depth: 0 - - # `Co-authored-by:` naming a coding agent (Claude Code, Codex, Cursor, Copilot …). Trailers - # naming PEOPLE are deliberately left alone: GitHub's squash merge builds them from the PR's - # commit authors and they are the only thing that keeps a contributor's credit on main, since - # the squash commit's own author is always the PR opener. Dependency bots are exempt likewise. - # Fixing a hit never requires touching a contributor's fork — see the script's own advice. - # - # Deliberately a STEP of `check` rather than a job of its own: `check` is the only context the - # main ruleset actually requires, so a separate job would show a red X that nothing enforces. - # Runs first — it costs a second and needs no install. - - name: No agent co-author trailers - if: github.event_name == 'pull_request' - run: node scripts/check-agent-trailers.mjs "$BASE" "$HEAD" - env: - BASE: ${{ github.event.pull_request.base.sha }} - HEAD: ${{ github.event.pull_request.head.sha }} # Secret scan FIRST, on the pristine checkout (tracked files only) — before `pnpm install` # creates node_modules. The gitleaks *Action* requires a paid GITLEAKS_LICENSE on GitHub diff --git a/AGENTS.md b/AGENTS.md index 064ff65d..545ae557 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,9 +24,7 @@ No `develop`, no `staging`. Maintainers with write access work on short-lived fe - Use [conventional commits](https://www.conventionalcommits.org): `(): `. Types: `feat`, `fix`, `docs`, `refactor`, `test`, `build`, `ci`, `chore`, `perf`, `style`. Subject is lowercase imperative, no trailing period. - Use the `/smart-commit` and `/suggest-commit` skills when drafting messages. They produce the canonical format for this repo. -- **Never credit a coding agent in a `Co-Authored-By:` trailer** — Claude Code, Codex, Cursor, Copilot and the like. They are tools we drive, not contributors, and naming them misrepresents who wrote the code. CI enforces this (`scripts/check-agent-trailers.mjs`). -- Trailers naming **people** are fine, and mostly not ours to write: GitHub's squash merge generates them from the PR's commit authors, and they are the only thing that keeps a contributor's credit on `main` — the squash commit's own author is always the PR opener, whoever wrote the code. Do not strip those. Dependency bots (dependabot, renovate) are left alone for the same reason. -- If an agent trailer does reach a PR, the maintainer merging it edits the squash message (`gh pr merge --squash --body "…"`). Never force-push a contributor's fork to clean this up. +- **Never credit a coding agent in a `Co-Authored-By:` trailer** (Claude Code, Codex, Cursor, Copilot). Trailers naming **people** are fine and must not be stripped — GitHub generates them from the PR's commit authors on squash, and they are what keeps a contributor's credit on `main`, since the squash commit's own author is always the PR opener. - Small, focused commits are encouraged. Commit as you go — not all at the end. Easier to review and revert. Don't mix unrelated changes in one commit. ## Pull requests diff --git a/scripts/check-agent-trailers.mjs b/scripts/check-agent-trailers.mjs deleted file mode 100644 index 9b2eda39..00000000 --- a/scripts/check-agent-trailers.mjs +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env node -// Refuse `Co-authored-by:` trailers that credit a CODING AGENT. -// -// The repo convention is narrower than it used to read. Trailers naming HUMANS are not just allowed, -// they are load-bearing: GitHub's squash merge builds them from the PR's commit authors, and they are -// the only reason an external contributor keeps credit on `main` (the squash commit's own author is -// always the PR opener). Stripping those would erase the people this project runs on. -// -// What must not appear is an agent: Claude Code, Codex, Cursor, Copilot and friends. They are tools -// the maintainers drive, not contributors, and crediting them as co-authors misrepresents who wrote -// the code. Dependency and CI bots (dependabot, renovate) are deliberately NOT in the list — their -// trailers are how their own automated PRs are attributed, which is a different thing. -// -// node scripts/check-agent-trailers.mjs -// -// Exits 1 and prints the offending commits when it finds any. -import { execFileSync } from 'node:child_process'; - -// Matched case-insensitively against the whole trailer line, so either the display name or the -// address is enough. Extend as new agents appear; keep humans and dependency bots out of it. -const AGENTS = [ - 'claude', - 'anthropic\\.com', - 'codex', - 'openai\\.com', - 'cursor', - 'copilot', - 'devin', - 'aider', - 'sourcegraph\\.com', - 'sweep-ai', -]; - -const TRAILER = /^\s*co-authored-by:\s*(.+)$/gim; -const AGENT_RE = new RegExp(AGENTS.join('|'), 'i'); - -const [base, head] = process.argv.slice(2); -if (!base || !head) { - console.error('usage: check-agent-trailers.mjs '); - process.exit(2); -} - -// %H then the body, one record per commit, NUL-separated so multi-line bodies stay intact. -const log = execFileSync('git', ['log', `${base}..${head}`, '--format=%H%n%B%x00'], { - encoding: 'utf8', - maxBuffer: 32 * 1024 * 1024, -}); - -const offenders = []; -for (const record of log.split('\0')) { - const trimmed = record.trim(); - if (!trimmed) continue; - const [sha, ...rest] = trimmed.split('\n'); - const body = rest.join('\n'); - for (const [, who] of body.matchAll(TRAILER)) { - if (AGENT_RE.test(who)) offenders.push({ sha: sha.slice(0, 8), who: who.trim() }); - } -} - -if (offenders.length === 0) { - console.log('no agent co-author trailers'); - process.exit(0); -} - -console.error('Co-authored-by trailers crediting a coding agent:\n'); -for (const o of offenders) console.error(` ${o.sha} Co-authored-by: ${o.who}`); -console.error(` -Agents are tools we drive, not contributors. Trailers naming PEOPLE are fine and should stay - -they are how a contributor keeps credit through a squash merge. - -To fix without rewriting anyone's branch: the maintainer merging the PR can drop these lines from -the squash message (\`gh pr merge --squash --body "..."\`), which never touches the contributor's -fork. Rewriting published commits is not required and, on a fork, not wanted. -`); -process.exit(1); diff --git a/scripts/check-agent-trailers.test.mjs b/scripts/check-agent-trailers.test.mjs deleted file mode 100644 index 304473e8..00000000 --- a/scripts/check-agent-trailers.test.mjs +++ /dev/null @@ -1,76 +0,0 @@ -// The agent-trailer guard, exercised against real commit ranges in this repository. -// -// A denylist check is worth exactly as much as its two failure modes: it must fire on the thing it -// exists for, and it must stay silent on the humans and dependency bots whose trailers are how their -// credit survives a squash merge. Both are asserted here against real history, not fixtures, because -// the interesting cases already exist on the remote. -import { test } from 'node:test'; -import assert from 'node:assert/strict'; -import { execFileSync, spawnSync } from 'node:child_process'; -import { dirname, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const HERE = dirname(fileURLToPath(import.meta.url)); -const ROOT = resolve(HERE, '..'); -const SCRIPT = resolve(HERE, 'check-agent-trailers.mjs'); - -const run = (base, head) => - spawnSync(process.execPath, [SCRIPT, base, head], { cwd: ROOT, encoding: 'utf8' }); - -/** A commit that exists locally, or null — keeps the suite green on a shallow clone. */ -function have(rev) { - const r = spawnSync('git', ['rev-parse', '--verify', `${rev}^{commit}`], { - cwd: ROOT, - encoding: 'utf8', - }); - return r.status === 0 ? r.stdout.trim() : null; -} - -test('a range with no agent trailers passes', (t) => { - const head = have('HEAD'); - const base = have('HEAD~1'); - if (!base || !head) return t.skip('shallow clone'); - const r = run(base, head); - assert.equal(r.status, 0, r.stderr); -}); - -test('a human co-author is NOT flagged', (t) => { - // 8db751d carries `Co-authored-by: Bilko ` — the exact shape that must - // survive, since it is the only thing keeping his credit on main after the squash. - const c = have('8db751d'); - if (!c) return t.skip('commit not present locally'); - const r = run(`${c}~1`, c); - assert.equal(r.status, 0, `a human co-author was flagged:\n${r.stderr}`); -}); - -test('a dependency bot is NOT flagged', (t) => { - // dependabot's trailer is how its own PRs are attributed. It is automation, not a coding agent. - const c = have('0d9d0ad'); - if (!c) return t.skip('commit not present locally'); - const r = run(`${c}~1`, c); - assert.equal(r.status, 0, `dependabot was flagged:\n${r.stderr}`); -}); - -test('an agent co-author IS flagged, and named in the output', (t) => { - // PR #118 really does carry `Co-authored-by: Cursor `. Fetched on demand so - // the assertion is about a real contributor branch rather than a fixture written to match the regex. - const fetched = spawnSync('git', ['fetch', '--quiet', '--depth=50', 'origin', 'pull/118/head'], { - cwd: ROOT, - encoding: 'utf8', - }); - if (fetched.status !== 0) return t.skip('cannot reach the PR ref'); - const head = execFileSync('git', ['rev-parse', 'FETCH_HEAD'], { - cwd: ROOT, - encoding: 'utf8', - }).trim(); - const base = execFileSync('git', ['rev-list', '--max-count=1', `${head}~40`], { - cwd: ROOT, - encoding: 'utf8', - }).trim(); - const r = run(base, head); - assert.equal(r.status, 1, 'the Cursor trailer should have failed the check'); - assert.match(r.stderr, /Cursor/i); - // The advice matters as much as the verdict: the fix must not send anyone at a contributor's fork - // with a force push. - assert.match(r.stderr, /gh pr merge --squash/); -}); From bf4fa9769972f822f0bd0262b81a716568bd3b3b Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Tue, 11 Aug 2026 17:26:06 +0300 Subject: [PATCH 40/69] fix(tr): refuse a seat confirmation when the declared period is unknown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The R10 temporal guard read `firstDeclaredYear == null` as "covers the period" and short-circuited the whole disjunction. `load.mjs` passes null whenever no history row carried a parseable year, so every such link reached the seat rung with no temporal check at all — publishing „Потвърдено" on a settlement match alone, which is precisely the "company relocated INTO the declared settlement afterwards" case the guard exists to block. That made the weakest rung the only one without a temporal test, on exactly the links where we know least. Rung 4's refutation leg right below already refuses to run without a year. An unknown year now FAILS the guard instead of skipping it. The undated-seat leg stays: a seat with no entry date is the ordinary shape for a company that never moved, and it is still checkable because a known year sits on the other side. Three tests: both null-year shapes (dated and undated seat) held, plus a positive control proving a known year with an undated seat still confirms — the fix is a bound, not a blanket. The methodology page asserted this rule already; it now states the precondition it relied on. --- apps/web/app/routes/conflict.methodology.tsx | 6 ++- scripts/tr/evidence.mjs | 13 +++-- scripts/tr/evidence.test.mjs | 53 ++++++++++++++++++++ 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/apps/web/app/routes/conflict.methodology.tsx b/apps/web/app/routes/conflict.methodology.tsx index c9a432b0..b1fc692c 100644 --- a/apps/web/app/routes/conflict.methodology.tsx +++ b/apps/web/app/routes/conflict.methodology.tsx @@ -231,8 +231,10 @@ export default function ConflictMethodology() { изисква пълно съвпадение и на трите имена в{' '} един и същ запис на акта (две от три не се приема; имена с инициали или с латински букви не се приемат). Седалищата се променят, затова регистрираното - седалище се приема само ако е вписано преди декларирания период. Заличените записи - се пропускат навсякъде. + седалище се приема само ако е вписано преди декларирания период —{' '} + и само когато този период е известен. Ако от декларациите не може да + се извлече година, проверката няма с какво да се направи и връзката остава скрита. + Заличените записи се пропускат навсякъде.

Справките се опресняват месечно, а датата на справката и номерът на diff --git a/scripts/tr/evidence.mjs b/scripts/tr/evidence.mjs index 01fce0e8..51c4c770 100644 --- a/scripts/tr/evidence.mjs +++ b/scripts/tr/evidence.mjs @@ -197,10 +197,17 @@ export function evidenceVerdict(input) { const declared = declaredSeats.map(normalizeSettlement).filter((s) => s !== ''); // R10: the registered seat must have been in force for the declared period. Seats move, and a // company that relocated INTO the declared settlement afterwards would otherwise confirm falsely. + // + // A null `firstDeclaredYear` FAILS the guard rather than skipping it. `load.mjs` passes null + // whenever no history row carried a parseable year, and an unknown year is not a satisfied + // temporal test — it is the absence of one. Reading it as „covers the period" made the weakest + // rung the only one with no temporal check, on exactly the links where we know least, and rung 4 + // below already refuses to run without a year (`firstDeclaredYear != null`) on the same ground. + // The undated-seat leg is different and stays: a seat with no entry date is the ordinary shape + // for a company that never moved, and it is checkable — a known year is still on the other side. const seatCoversPeriod = - firstDeclaredYear == null || - seat.entryDate == null || - seat.entryDate <= `${firstDeclaredYear}-12-31`; + firstDeclaredYear != null && + (seat.entryDate == null || seat.entryDate <= `${firstDeclaredYear}-12-31`); if (seatCoversPeriod && declared.includes(seat.settlement)) { return verdict('confirmed', true, { matchedFact: `seat:${seat.settlement}`, diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs index 858db919..4de42a93 100644 --- a/scripts/tr/evidence.test.mjs +++ b/scripts/tr/evidence.test.mjs @@ -162,6 +162,59 @@ test('rung 3 — a seat registered AFTER the declared period does not confirm', assert.notEqual(v.kind, 'confirmed'); }); +test('rung 3 — an UNKNOWN first declared year cannot confirm on a seat', () => { + // R10 again, from the other side. `load.mjs` passes `firstDeclaredYear: null` whenever no history row + // carried a parseable year, and a null year means the temporal check has NOTHING to compare against — + // not that the seat covers the period. The same relocated company as the test above, with the year + // unknown instead of 2021, must reach the same held outcome: an unknown guard is a failed guard. + const moved = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив'), '2024-06-01T00:00:00'), + ]); + const v = evidenceVerdict({ + ...base, + deed: moved, + declaredSeats: ['Пловдив'], + firstDeclaredYear: null, + }); + assert.notEqual(v.kind, 'confirmed'); +}); + +test('rung 3 — an unknown year holds a seat match even when the seat has NO entry date', () => { + // The nastier half: with no entry date on the register side AND no year on the declaration side there + // are two unknowns and zero evidence about the period, yet both legs of the old disjunction read TRUE. + // Rung 4's refutation leg already refuses to run without a year (`firstDeclaredYear != null`); the seat + // leg must refuse on the same ground, or the weakest rung is the one with no temporal check at all. + const undated = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: undated, + declaredSeats: ['Пловдив'], + firstDeclaredYear: null, + }); + assert.notEqual(v.kind, 'confirmed', 'two unknowns must not multiply into a public claim'); +}); + +test('rung 3 — a KNOWN year with an undated seat still confirms (the guard is not a blanket)', () => { + // Positive control. Bounding the null case must not quietly kill the rung: a seat with no entry date + // is the ordinary shape for a company that never moved, and it still confirms under a known year. + const undated = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: undated, + declaredSeats: ['Пловдив'], + firstDeclaredYear: 2021, + }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'seat:ПЛОВДИВ'); +}); + test('rung 3 — the weakest rung ALSO requires global name uniqueness (ADR-0017 carried forward)', () => { const other = deed([ fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), From 466d002a4c82b8086a098078061b70a626528c7f Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Tue, 11 Aug 2026 17:32:07 +0300 Subject: [PATCH 41/69] fix(cacbg): give the monotonicity gate a path for the removals it sanctions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0033 decision 6 licenses removal by a rules bump "or a correction of wrong input", and names the court-annulled entry as wired to the ADR-0031 suppression path. Neither could get through the gate, and they failed in opposite directions: - A suppression flips status published → suppressed with rules_version untouched, so the link leaves the published set and the gate hard-fails the one takedown the ADR points at. - Correcting the input UNBUILDS the link, so a suppression on it matches nothing and load.mjs's B3 unused-entry rail fails the build instead. So the only escape was bumping rules_version, which relabels every other link. The audit now reads each vanished link's current status: `suppressed` is a declared removal. For a corrected input there is a second fingerprinted list, link-corrections.jsonl, mirroring ADR-0031 for the same reason — `pid|eik` in git records which named official was tied to which company, permanently. load.mjs flags matching keys in the pre-wipe snapshot; the audit reads the flag. Both grounds are still PRINTED, now with the ground that licensed them, since "removed" alone flattens an annulment, a corrected parse and a rules bump into one line. A correction that matches nothing fails the build, mirroring B3 and for a sharper reason: a stale acknowledgement would pre-clear a future disappearance of that same link — the exact regression the gate exists to catch — with nobody having decided it. Acknowledgements are one-shot; the runbook says so and says not to use the list on a removal you cannot explain. Six tests, including the mutation control: an unacknowledged, unsuppressed drop still hard-fails, so a gate that accepted everything would not pass the two new cases. Also lands the snapshot's atomic write (tmp + rename, as the raw deeds already do). The audit deliberately does not swallow a parse failure there, so a torn write wedged every subsequent audit until a human cleared the file by hand. --- ...-evidence-replaces-name-distinctiveness.md | 13 +++- docs/runbooks/related-persons-suppression.md | 32 ++++++++ scripts/cacbg/audit.mjs | 54 +++++++++++-- scripts/cacbg/audit.test.mjs | 56 ++++++++++++++ scripts/cacbg/link-corrections.jsonl | 0 scripts/cacbg/load.mjs | 60 +++++++++++++-- scripts/cacbg/load.test.mjs | 76 +++++++++++++++++++ scripts/cacbg/suppressions.mjs | 23 +++++- 8 files changed, 297 insertions(+), 17 deletions(-) create mode 100644 scripts/cacbg/link-corrections.jsonl diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md index e75c28c3..cfa579ed 100644 --- a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -252,7 +252,18 @@ Therefore: seals are **re-derived deterministically** on every run — a seal is including held ones, so the review queue explains itself — and monotonicity is enforced as a **gate**. The audit compares against the pre-wipe export and raises a hard finding when a previously published link disappears under an **unchanged** `rules_version`; under a changed version it degrades to a printed diff. -Removal remains an intentional event: a rules-version bump, or a correction of wrong input. +Removal remains an intentional event, and each ground has an expressible mechanism — a gate that hard-fails +the only removals it sanctions is a deadlock, not a rail: + +| Ground | Mechanism | Why not one of the others | +|---|---|---| +| The rules changed | a `rules_version` bump | — | +| The evidence is void (court annulment, wrong person) | ADR-0031 suppression — `status` flips to `suppressed`, so the audit reads the current status and treats it as declared | the link is correctly built; only its publication is wrong | +| The input was wrong | `scripts/cacbg/link-corrections.jsonl`, fingerprinted like the suppression list; `load.mjs` flags the key in the pre-wipe snapshot | correcting the input *unbuilds* the link, so a suppression on it matches nothing and trips the unused-entry rail | + +Both non-rules grounds are one-shot, version-controlled, and reviewed in git; neither is silent — the audit +prints every declared removal with the ground that licensed it. An acknowledgement that matches nothing +fails the build, because a stale one would pre-clear a *future* disappearance of that same link. ### 7. Launch gates diff --git a/docs/runbooks/related-persons-suppression.md b/docs/runbooks/related-persons-suppression.md index 2aaa2d97..1fffa8c1 100644 --- a/docs/runbooks/related-persons-suppression.md +++ b/docs/runbooks/related-persons-suppression.md @@ -61,6 +61,38 @@ wrangler d1 execute --remote \ Then do the Standard takedown steps above **in the same shift**. The direct UPDATE is undone by the next ship (which rebuilds the table); only the git list makes it permanent. +## Correcting a wrong INPUT (a different list — read this before reaching for a suppression) + +Suppression keeps a *built* link out of the public surface. It is the wrong tool when the link should never +have been built at all — a misparsed declaration row, a stake attributed to the wrong filing, an entity +that was never declared. There you fix the input, and the link stops existing. + +That removal still has to get past the monotonicity gate ([ADR-0033](../adr/0033-registry-evidence-replaces-name-distinctiveness.md) +decision 6), which hard-fails on a link that was published last run and is not published now. And you +cannot use a suppression to clear it: correcting the input unbuilds the link, so the fingerprint would +match nothing and `load.mjs` fails on the unmatched-entry rail instead. The two removals fail in opposite +directions, which is why there is a second list. + +1. Compute the fingerprint exactly as above (same salt, same function, same `key_version`). +2. Append one line to `scripts/cacbg/link-corrections.jsonl`: + + ```json + {"fp":"","key_version":"1","reason":"the declaration row was misparsed — ","signal_ref":"","corrected_at":"2026-08-11"} + ``` + +3. Land it **in the same PR as the input fix**. The loader flags that key in the pre-wipe snapshot and the + audit reads it as a declared removal — printed, never silent, and attributed to this ground rather than + flattened into "removed". + +**An acknowledgement is one-shot. Delete the line in the next change.** Once the corrected link stops being +published it also stops appearing in the prior set, so the entry matches nothing and the build fails on the +same rail suppressions use. That is deliberate: a stale acknowledgement would sit in the list and silently +pre-clear a *future* disappearance of that same link — the exact regression the gate exists to catch, with +nobody having decided it. + +Do not use this list to clear a removal you do not understand. If a published link vanished and you cannot +name why, that is the gate doing its job; find the cause. + ## Fail-closed behaviour `load.mjs` aborts the build (non-zero exit) on any of three conditions — each would otherwise silently diff --git a/scripts/cacbg/audit.mjs b/scripts/cacbg/audit.mjs index 9c8355f6..5eb889d0 100644 --- a/scripts/cacbg/audit.mjs +++ b/scripts/cacbg/audit.mjs @@ -44,6 +44,17 @@ const published = db ) .all(); +// Every link's CURRENT status, for the monotonicity gate below. A link that left the published set +// because it was SUPPRESSED is still in this table — the takedown is a status flip, not a delete — and +// distinguishing that from a link that simply stopped being built is the difference between a declared +// removal and a silent recall regression. Read here, while the handle is open. +const statusNow = new Map( + db + .prepare('SELECT link_key, status FROM interest_links') + .all() + .map((r) => [r.link_key, r.status]), +); + // The only two evidence rungs that publish (ADR-0033 decision 1). Everything else withholds. const PUBLISHING_EVIDENCE = new Set(['document', 'confirmed']); // The closed vocabulary for a sealed matched_fact lives in evidence.mjs, next to the code that WRITES @@ -189,7 +200,28 @@ try { if (e.code !== 'ENOENT') throw e; } const vanished = (priorPublished ?? []).filter((p) => !publishedNow.has(p.link_key)); -const regressions = vanished.filter((p) => p.rules_version === RULES_VERSION); + +// The three grounds decision 6 sanctions for a removal. Anything else that vanished is a regression. +// +// Getting this set right is what stops the gate from deadlocking the mechanisms it points at. The +// rules bump is the obvious one. The other two are not optional extras — each is the ONLY expressible +// form of a removal the ADR names in prose: +// +// suppressed — the ADR-0031 takedown path, which decision 6 wires the court-annulled entry +// (чл. 29 ЗТРРЮЛНЦ) to by name. It flips status published → suppressed with rules_version +// untouched, so without this branch the one removal the ADR explicitly licenses hard-fails. +// corrected — "a correction of wrong input". Suppression cannot express it: correcting the input +// unbuilds the link, and load.mjs's B3 unused-suppression gate then fails the build for a +// fingerprint that matched nothing. So the two sanctioned removals would fail in OPPOSITE +// directions, leaving a real correction with no path at all. The flag is set by load.mjs from the +// version-controlled, fingerprinted corrections list — a human decision recorded in git, not a +// condition the audit can infer. +// +// Both are declared, both are reviewed, and both are still PRINTED below: a withdrawn public claim is +// never silent, it just is not a build failure. +const declaredRemoval = (p) => + p.rules_version !== RULES_VERSION || p.corrected === true || statusNow.get(p.link_key) === 'suppressed'; +const regressions = vanished.filter((p) => !declaredRemoval(p)); for (const p of regressions) findings.push({ axis: 'D_monotonicity', @@ -214,17 +246,23 @@ for (const [axis, fs] of Object.entries(byAxis)) { if (priorPublished === null) { console.log(`## monotonicity — no prior export at ${SNAPSHOT}; treating this as a first run\n`); } else { - const declared = vanished.filter((p) => p.rules_version !== RULES_VERSION); + const declared = vanished.filter(declaredRemoval); console.log( `## monotonicity — ${priorPublished.length} published last run, ${publishedNow.size} now; ` + `${regressions.length} regression(s), ${declared.length} declared removal(s)\n`, ); - // Declared removals are not findings, but they are never silent: a rules bump is exactly when a - // human should read which named claims it withdrew. - for (const p of declared) - console.log( - ` - ${p.link_key} removed under a rules change (${p.rules_version} → ${RULES_VERSION})`, - ); + // Declared removals are not findings, but they are never silent: withdrawing a named public claim is + // exactly when a human should read WHICH claims went and on what ground. The ground is printed too — + // "removed" alone would flatten a court annulment, a corrected parse and a rules bump into one line. + for (const p of declared) { + const ground = + p.rules_version !== RULES_VERSION + ? `a rules change (${p.rules_version} → ${RULES_VERSION})` + : statusNow.get(p.link_key) === 'suppressed' + ? 'a suppression (ADR-0031 takedown path)' + : 'an acknowledged input correction'; + console.log(` - ${p.link_key} removed under ${ground}`); + } if (declared.length) console.log(''); } diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index 712178c3..ec745bae 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -316,6 +316,62 @@ test('D — no snapshot (a first run) neither fires nor crashes', () => { assert.equal(threw, false); }); +// The gate must also let the two removals ADR-0033 decision 6 actually SANCTIONS through, or it +// deadlocks the mechanisms it points at. Both are declared events with a reviewed paper trail in git; +// neither is a silent recall regression, and the gate exists to tell those apart. + +test('D — a link taken down through the ADR-0031 suppression path is a declared removal, not a regression', () => { + // Decision 6 names the court-annulled entry (чл. 29 ЗТРРЮЛНЦ) and wires it to ADR-0031. That path + // flips status published → suppressed, so the link leaves the published set with rules_version + // unchanged — firing the gate on the ONE removal the ADR explicitly licenses. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + links: [ + ...ONE_LINK.links, + `'il2','p9|200000002','p9','200000002','${K('ВТОРА ФИРМА ЕООД')}','exact_name_key','document','b2','owns',0,1000,'suppressed'`, + ], + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, + ], + }); + assert.equal(threw, false, 'the sanctioned takedown path must not fail the build'); + assert.match(out, /p9\|200000002/, 'but a withdrawn public claim is never silent'); +}); + +test('D — a snapshot entry acknowledged as a corrected input is a declared removal', () => { + // The other sanctioned ground: the link should never have been published because its INPUT was + // wrong. Suppression cannot express it — correcting the input unbuilds the link, and load.mjs's B3 + // gate then fails the build for a suppression that matched nothing. Without this the two sanctioned + // removals fail in opposite directions and there is no way to clear either. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1', corrected: true }, + ], + }); + assert.equal(threw, false, 'an acknowledged correction must not fail the build'); + assert.match(out, /p9\|200000002/); +}); + +test('D — neither escape hatch fires on its own: an unacknowledged, unsuppressed drop still hard-fails', () => { + // The mutation control for the two tests above. A gate that accepted every disappearance would pass + // both of them, and this is the assertion that says it did not. + const { threw } = buildAndAudit({ + ...ONE_LINK, + links: [ + ...ONE_LINK.links, + `'il2','p9|200000002','p9','200000002','${K('ВТОРА ФИРМА ЕООД')}','exact_name_key','document','b2','owns',0,1000,'held'`, + ], + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, + ], + }); + assert.equal(threw, true, 'held is not a sanctioned removal — the evidence simply stopped licensing it'); +}); + test('D positive control — an unchanged published set produces no monotonicity finding', () => { // Without this, a gate that never fires would pass both negatives above. const { threw, out } = buildAndAudit({ diff --git a/scripts/cacbg/link-corrections.jsonl b/scripts/cacbg/link-corrections.jsonl new file mode 100644 index 00000000..e69de29b diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index bcceeb88..e3da7f8a 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -26,7 +26,12 @@ import { RULES_VERSION, } from '../tr/evidence.mjs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; -import { fingerprint, loadSuppressions, SUPPRESSION_KEY_VERSION } from './suppressions.mjs'; +import { + fingerprint, + loadCorrections, + loadSuppressions, + SUPPRESSION_KEY_VERSION, +} from './suppressions.mjs'; import { canonicalInstitution } from './institutions.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); @@ -145,13 +150,56 @@ const priorPublished = (() => { return []; } })(); +// Decision 6's SECOND sanctioned removal: „a correction of wrong input". A link whose input was wrong +// should never have been published, but correcting the input UNBUILDS it — so a suppression on it +// would match no built link and trip the B3 gate above, while doing nothing leaves a permanent hard +// finding. The acknowledgement is therefore recorded here, against the set the gate actually compares: +// each prior-published key whose fingerprint is listed is exported flagged, and audit.mjs reads the +// flag as a declared removal. Fingerprinted for ADR-0031's reason — `pid|eik` in git would record which +// named official was tied to which company for ever. +const CORRECTIONS_LIST = + process.env.CACBG_CORRECTIONS_LIST || path.join(ROOT, 'scripts/cacbg/link-corrections.jsonl'); +const correctedFp = new Set( + loadCorrections(CORRECTIONS_LIST, SUPP_SALT, SUPPRESSION_KEY_VERSION).map((e) => e.fp), +); +const usedCorrections = new Set(); +const snapshot = priorPublished.map((p) => { + if (correctedFp.size === 0) return p; + const fp = fingerprint(p.link_key, SUPP_SALT); + if (!correctedFp.has(fp)) return p; + usedCorrections.add(fp); + return { ...p, corrected: true }; +}); +// The B3 rail, mirrored — and it matters MORE here. A stale suppression silently un-suppresses; a +// stale acknowledgement silently pre-clears a FUTURE disappearance of that same link, which is exactly +// the regression the gate exists to catch, with nobody having decided it. An acknowledgement is +// one-shot by construction: once the corrected link stops being published it also stops appearing in +// the prior set, so the entry must be deleted from the list in the same change that lands the fix. +if (!EMIT_CANDIDATES_ONLY) { + const unusedCorr = [...correctedFp].filter((fp) => !usedCorrections.has(fp)); + if (unusedCorr.length > 0) { + db.close(); + throw new Error( + `${unusedCorr.length} correction(s) matched NO previously published link — a stale acknowledgement ` + + `would clear a future disappearance of that link, which is the regression this gate exists to ` + + `catch. Delete the entry once its fix has shipped (an acknowledgement is one-shot). ` + + `Unmatched fingerprints: ${unusedCorr.map((f) => f.slice(0, 12) + '…').join(', ')}`, + ); + } +} // Not written by the bootstrap pass, which works on a throwaway copy and has no business restating what // the real run is about to record. -if (!EMIT_CANDIDATES_ONLY) - fs.writeFileSync( - path.join(STAGING, 'published-snapshot.json'), - JSON.stringify(priorPublished, null, 2) + '\n', - ); +// +// tmp + rename, not a bare write: a crash mid-write leaves a truncated file, and audit.mjs deliberately +// does NOT swallow a parse failure here (only ENOENT is a legitimate first run). A torn snapshot would +// therefore wedge every subsequent audit until a human cleared the file by hand. The raw deeds already +// land this way; the gate's own input deserves the same. +if (!EMIT_CANDIDATES_ONLY) { + const snapPath = path.join(STAGING, 'published-snapshot.json'); + const snapTmp = `${snapPath}.tmp`; + fs.writeFileSync(snapTmp, JSON.stringify(snapshot, null, 2) + '\n'); + fs.renameSync(snapTmp, snapPath); +} // Full idempotent rebuild that also picks up schema changes: drop the CACBG tables (children first — // FK-safe) and re-apply the migration. Nothing to preserve — suppressions are external now. diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index b2a9dbe7..de7e3d80 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -1307,3 +1307,79 @@ test('the monotonicity gate still sees a prior surface AFTER a bootstrap pass', 'the snapshot went empty — the gate is now vacuous and would never fire again', ); }); + +// ── the corrections list — ADR-0033 decision 6's second sanctioned removal ──────────────────────── +// Decision 6 licenses removal by „a rules-version bump, or a correction of wrong input". Only the +// first was expressible. Suppression cannot carry a correction: correcting the input UNBUILDS the +// link, and the B3 unused-suppression gate above then fails the build for a fingerprint that matched +// nothing — so the two sanctioned removals failed in opposite directions and a real correction had no +// path at all. The list is fingerprinted for the same reason ADR-0031's is: `pid|eik` in git would +// record which named official was linked to which company, for ever. + +test('a corrections entry marks its key in the snapshot, so the audit reads a declared removal', () => { + runLoad(); + const db = open(); + const key = db + .prepare("SELECT link_key FROM interest_links WHERE status='published' LIMIT 1") + .get().link_key; + db.close(); + + const corrFile = path.join(dir, 'corr.jsonl'); + fs.writeFileSync( + corrFile, + JSON.stringify({ + fp: fingerprint(key, SUPP_SALT), + key_version: '1', + reason: 'the declaration row was misparsed; the stake was never declared', + corrected_at: '2026-08-11', + }) + '\n', + ); + runLoad({ CACBG_CORRECTIONS_LIST: corrFile, SUPPRESSION_SALT: SUPP_SALT }); + + const snap = JSON.parse(fs.readFileSync(path.join(STAGING, 'published-snapshot.json'), 'utf8')); + const marked = snap.find((s) => s.link_key === key); + assert.ok(marked, 'the key must still be exported — the gate needs to SEE it leave'); + assert.equal(marked.corrected, true); + // Every other key stays unflagged: an acknowledgement is per-link, never a blanket amnesty. + assert.equal( + snap.filter((s) => s.corrected === true).length, + 1, + 'one acknowledgement must not clear the whole surface', + ); +}); + +test('a corrections entry matching NO previously published link fails the build', () => { + // The B3 rail, mirrored. A stale acknowledgement is worse than a missing one: it sits in the list + // and would clear a FUTURE disappearance of the same link — the exact regression the gate exists to + // catch — with nobody having decided that. + const corrFile = path.join(dir, 'corr-stale.jsonl'); + fs.writeFileSync( + corrFile, + JSON.stringify({ + fp: fingerprint('p-nobody|999999999', SUPP_SALT), + key_version: '1', + reason: 'stale', + corrected_at: '2026-08-11', + }) + '\n', + ); + runLoad(); + assert.throws( + () => runLoad({ CACBG_CORRECTIONS_LIST: corrFile, SUPPRESSION_SALT: SUPP_SALT }), + (err) => + /correction/i.test(String(err.stderr ?? '') + String(err.message ?? '')) && + /matched NO/i.test(String(err.stderr ?? '') + String(err.message ?? '')), + ); +}); + +test('corrections are fail-closed on a missing salt, exactly like suppressions', () => { + const corrFile = path.join(dir, 'corr-nosalt.jsonl'); + fs.writeFileSync( + corrFile, + JSON.stringify({ fp: 'deadbeef', key_version: '1', reason: 'x', corrected_at: '2026-08-11' }) + + '\n', + ); + assert.throws( + () => runLoad({ CACBG_CORRECTIONS_LIST: corrFile, SUPPRESSION_SALT: '' }), + (err) => /SUPPRESSION_SALT is unset/.test(String(err.stderr ?? '') + String(err.message ?? '')), + ); +}); diff --git a/scripts/cacbg/suppressions.mjs b/scripts/cacbg/suppressions.mjs index b1096c60..2c67c93d 100644 --- a/scripts/cacbg/suppressions.mjs +++ b/scripts/cacbg/suppressions.mjs @@ -33,6 +33,25 @@ export function fingerprint(linkKey, salt) { * An absent or empty list needs no salt (nothing to fingerprint), so the common path stays friction-free. */ export function loadSuppressions(listPath, salt, keyVersion = SUPPRESSION_KEY_VERSION) { + return loadFingerprintedList(listPath, salt, keyVersion, 'suppression'); +} + +/** + * The MONOTONICITY-gate twin (ADR-0033 decision 6): links whose removal is licensed because their + * INPUT was wrong, not because the rules changed. + * + * A separate list from the suppressions, because the two are not the same act and cannot share one. + * A suppression keeps a built link OUT of the public surface; a correction says the link should never + * have been built at all — so correcting the input unbuilds it, and a suppression on it would then + * match nothing and trip the B3 unused gate. Same fingerprinting for the same reason: the raw + * `pid|eik` records which named official was tied to which company, which is precisely what must not + * live in git history. Consumed by load.mjs, which flags the matching keys in the pre-wipe snapshot. + */ +export function loadCorrections(listPath, salt, keyVersion = SUPPRESSION_KEY_VERSION) { + return loadFingerprintedList(listPath, salt, keyVersion, 'correction'); +} + +function loadFingerprintedList(listPath, salt, keyVersion, label) { const entries = existsSync(listPath) ? readFileSync(listPath, 'utf8') .split('\n') @@ -42,14 +61,14 @@ export function loadSuppressions(listPath, salt, keyVersion = SUPPRESSION_KEY_VE : []; if (entries.length > 0 && !salt) { throw new Error( - `${entries.length} suppression(s) in ${listPath} but SUPPRESSION_SALT is unset — refusing to build ` + + `${entries.length} ${label}(s) in ${listPath} but SUPPRESSION_SALT is unset — refusing to build ` + `(would silently un-suppress contested links). Set SUPPRESSION_SALT and retry.`, ); } for (const e of entries) { if (String(e.key_version ?? '') !== String(keyVersion)) { throw new Error( - `suppression ${String(e.fp ?? '').slice(0, 12)}… has key_version ${JSON.stringify(e.key_version)} ` + + `${label} ${String(e.fp ?? '').slice(0, 12)}… has key_version ${JSON.stringify(e.key_version)} ` + `but the current SUPPRESSION_KEY_VERSION is ${keyVersion} — refusing to build (a rotated salt ` + `would silently un-suppress it). Re-fingerprint every entry under the current salt + key_version.`, ); From e5e8709b3e6079197261c111b908f9e05b9e2253 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Tue, 11 Aug 2026 17:39:12 +0300 Subject: [PATCH 42/69] fix(ci,tr,db): close the remaining review findings on injection, purge and card claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four independent fixes, all from the deep-pass reviews still open at 9555fbf. related-persons-data.yml — the environment guard read `${{ inputs.environment }}` inline in two `run:` blocks. `type: choice` binds the UI only; a POST to .../dispatches can pass any string, which an inline expression pastes into the shell before `[` evaluates it. Worse, that same value decides whether the production guard fires, so a crafted one could inject AND walk past the guard meant to trip on it. Both now read $SIGMA_SHIP_ENV, already bound one block up, where the value is data rather than code. (deploy.yml:49 has the same shape and is untouched — it belongs to another change.) cache.mjs — the orphan sweep called unlinkSync unguarded, unlike the expired loop above it. It runs AFTER the DB DELETE commits, so a file that vanished between readdir and unlink aborts a half-finished purge: rows gone, files still on disk, run reported as failed, and no way to tell "already gone" from "an orphan still holding third-party names". ENOENT is now tolerated and uncounted; every other error still surfaces, because a purge that cannot delete has left PII behind and reporting success is the failure it exists to prevent. `unlink` is injectable so the race is tested rather than described. conflicts.render.test.tsx — the family fixture inherited evidenceKind 'document' + registryRole 'owner' from the self-link factory, a row the query cannot produce: findPerson searches for the OFFICIAL, and a relative's stake is registered to the relative, so only a seat/ЕИК confirmation is reachable. It now mirrors production, and a new test pins what the old shape hid — a family card must never render „вписано като съдружник/собственик", which would assert the named official is recorded as an owner, on the one card designed to keep the stakeholder anonymous. related-persons.ts — `lookup_date ?? ''` was an unreachable branch (NOT NULL in 0006, and the row only arrives through the seal filter) that would have shipped an empty string as a date if it ever ran. Narrowed like evidenceKind above it. Not taken: the orphan sweep's `known` set was flagged for using raw `r.eik` while the expired loop uses `safeEik(row.eik)`. safeEik VALIDATES rather than normalizes — it throws on a bad shape — so for any storable ЕИК the two are the same string, and a non-canonical one makes the deed unreadable through readDeed, which means the file really is unreachable PII and deleting it is what the privacy rail should do. --- .github/workflows/related-persons-data.yml | 9 ++- apps/web/app/routes/conflict.methodology.tsx | 6 +- apps/web/app/routes/conflicts.render.test.tsx | 21 +++++++ packages/db/src/queries/related-persons.ts | 6 +- scripts/cacbg/audit.mjs | 4 +- scripts/cacbg/audit.test.mjs | 6 +- scripts/tr/cache.mjs | 18 ++++-- scripts/tr/cache.test.mjs | 62 +++++++++++++++++++ 8 files changed, 120 insertions(+), 12 deletions(-) diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index 7058fd32..59db1219 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -128,20 +128,25 @@ jobs: # live in each Environment's SIGMA_D1_NAME var; the ship step additionally requires SIGMA_SHIP_ENV to match # and PROVES the name↔SIGMA_D1_ID pair against Cloudflare before deleting — assertD1TargetAuthorized.) # The slot set is kept in lockstep with PRODUCTION_SLOTS in scripts/ship-related-persons.mjs (#226). + # $SIGMA_SHIP_ENV, never `${{ inputs.environment }}` inline. `type: choice` is enforced by the UI + # only — a POST to .../dispatches can pass an arbitrary string, which an inline expression pastes + # into the shell before `[` ever evaluates it. This block is also the one that decides whether the + # production guard fires, so a crafted value could both inject and walk past the guard it was + # meant to trip. The env binding above passes the same value as data, where it cannot be code. - name: Guard — SIGMA_D1_NAME matches the environment run: | case "${SIGMA_D1_NAME}" in sigma-blue|sigma-green) IS_PROD_SLOT=1 ;; *) IS_PROD_SLOT=0 ;; esac - if [ "${{ inputs.environment || 'staging' }}" = "production" ]; then + if [ "${SIGMA_SHIP_ENV}" = "production" ]; then if [ "$IS_PROD_SLOT" != "1" ]; then echo "::error::production requires SIGMA_D1_NAME to be a prod slot (sigma-blue|sigma-green), got '${SIGMA_D1_NAME}' — refusing to ship." exit 1 fi else if [ "$IS_PROD_SLOT" = "1" ]; then - echo "::error::SIGMA_D1_NAME='${SIGMA_D1_NAME}' (a PRODUCTION slot) on a '${{ inputs.environment || 'staging' }}' run — a rebuild would WIPE prod. Set a non-production SIGMA_D1_NAME variable for this Environment." + echo "::error::SIGMA_D1_NAME='${SIGMA_D1_NAME}' (a PRODUCTION slot) on a '${SIGMA_SHIP_ENV}' run — a rebuild would WIPE prod. Set a non-production SIGMA_D1_NAME variable for this Environment." exit 1 fi fi diff --git a/apps/web/app/routes/conflict.methodology.tsx b/apps/web/app/routes/conflict.methodology.tsx index b1fc692c..152df8a7 100644 --- a/apps/web/app/routes/conflict.methodology.tsx +++ b/apps/web/app/routes/conflict.methodology.tsx @@ -232,9 +232,9 @@ export default function ConflictMethodology() { един и същ запис на акта (две от три не се приема; имена с инициали или с латински букви не се приемат). Седалищата се променят, затова регистрираното седалище се приема само ако е вписано преди декларирания период —{' '} - и само когато този период е известен. Ако от декларациите не може да - се извлече година, проверката няма с какво да се направи и връзката остава скрита. - Заличените записи се пропускат навсякъде. + и само когато този период е известен. Ако от декларациите не може + да се извлече година, проверката няма с какво да се направи и връзката остава + скрита. Заличените записи се пропускат навсякъде.

Справките се опресняват месечно, а датата на справката и номерът на diff --git a/apps/web/app/routes/conflicts.render.test.tsx b/apps/web/app/routes/conflicts.render.test.tsx index 6be5ea6e..07339f05 100644 --- a/apps/web/app/routes/conflicts.render.test.tsx +++ b/apps/web/app/routes/conflicts.render.test.tsx @@ -53,6 +53,13 @@ const familyLink = link({ company: 'ЕВРОСТРОЙ 21 ЕООД', eik: '333', relation: 'related', // a close relative's stake — anonymized + // The shape the QUERY actually yields for a family link, not the base factory's self-link default. + // `findPerson` searches the act for the OFFICIAL; a relative's stake is registered to the relative, + // so the official is never found and no `document` rung is reachable — only a seat/ЕИК confirmation. + // A fixture carrying `document`/`owner` here tested a row that cannot exist and hid the assertion + // below, which is the one that matters: no registry-role claim on an anonymized card. + evidenceKind: 'confirmed' as const, + registryRole: null, ownInstitution: false, contractCount: 1, contractValueEur: 250_000, @@ -270,6 +277,20 @@ describe('Trade Register evidence on the card (#279, ADR-0033)', () => { expect(text).not.toContain('вписано като'); }); + it('a FAMILY card never carries a registry-role claim — the relative is not in the act we read', async () => { + // The production shape for a family link, which the fixture used to contradict: `findPerson` looks + // for the OFFICIAL, and a relative's stake is registered to the relative, so the official is never + // found and the rung can only ever be `confirmed`/`registryRole: null`. A card that said „лицето е + // вписано като съдружник/собственик" here would assert that the named official is recorded in the + // register as an owner of this company — a false, named, libel-shaped claim, on the one card whose + // whole design is that the stakeholder stays anonymous (ADR-0030/0032). + await renderConflicts([familyLink]); + const familyCard = container.querySelector('.conflict-card')!; + expect(familyCard.textContent).toContain('деклариран дял на свързано лице'); + expect(familyCard.textContent).toContain('самоличност, потвърдена по декларирани данни'); + expect(familyCard.textContent).not.toContain('вписано като'); + }); + it('links out to the register so a reader can check the same act we read', async () => { await renderConflicts([link({ eik: '201122335' })]); const hrefs = [...container.querySelectorAll('a')].map((a) => a.getAttribute('href') ?? ''); diff --git a/packages/db/src/queries/related-persons.ts b/packages/db/src/queries/related-persons.ts index 8451b01a..b149ebbd 100644 --- a/packages/db/src/queries/related-persons.ts +++ b/packages/db/src/queries/related-persons.ts @@ -239,7 +239,11 @@ function toLink(r: LinkRow): ConflictLink { r.registry_role === 'owner' || r.registry_role === 'manager' ? r.registry_role : null, registryEntryNumber: r.entry_number, registryEntryDate: r.entry_date, - registryLookupDate: r.lookup_date ?? '', + // Narrowed for the same reason as evidenceKind, not defaulted. `lookup_date` is NOT NULL in + // migration 0006 and the row only reaches here through the seal filter, so `?? ''` was a branch + // that could not run — and if the invariant ever broke it would have shipped an empty string as a + // date, which reads as a valid-but-blank provenance rather than as the contradiction it is. + registryLookupDate: r.lookup_date as string, }; } diff --git a/scripts/cacbg/audit.mjs b/scripts/cacbg/audit.mjs index 5eb889d0..6ca5f3a7 100644 --- a/scripts/cacbg/audit.mjs +++ b/scripts/cacbg/audit.mjs @@ -220,7 +220,9 @@ const vanished = (priorPublished ?? []).filter((p) => !publishedNow.has(p.link_k // Both are declared, both are reviewed, and both are still PRINTED below: a withdrawn public claim is // never silent, it just is not a build failure. const declaredRemoval = (p) => - p.rules_version !== RULES_VERSION || p.corrected === true || statusNow.get(p.link_key) === 'suppressed'; + p.rules_version !== RULES_VERSION || + p.corrected === true || + statusNow.get(p.link_key) === 'suppressed'; const regressions = vanished.filter((p) => !declaredRemoval(p)); for (const p of regressions) findings.push({ diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index ec745bae..267ce4ad 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -369,7 +369,11 @@ test('D — neither escape hatch fires on its own: an unacknowledged, unsuppress { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, ], }); - assert.equal(threw, true, 'held is not a sanctioned removal — the evidence simply stopped licensing it'); + assert.equal( + threw, + true, + 'held is not a sanctioned removal — the evidence simply stopped licensing it', + ); }); test('D positive control — an unchanged published set produces no monotonicity finding', () => { diff --git a/scripts/tr/cache.mjs b/scripts/tr/cache.mjs index 20668718..764c16f2 100644 --- a/scripts/tr/cache.mjs +++ b/scripts/tr/cache.mjs @@ -226,7 +226,7 @@ export function coverage(db, wanted) { export function purgeExpired( db, rawDir, - { retentionDays = RETENTION_DAYS, now = new Date() } = {}, + { retentionDays = RETENTION_DAYS, now = new Date(), unlink = fs.unlinkSync } = {}, ) { const cutoff = new Date(now.getTime() - retentionDays * 86_400_000).toISOString(); const expired = db.prepare('SELECT eik, raw_path FROM deeds WHERE fetched_at < ?').all(cutoff); @@ -236,7 +236,7 @@ export function purgeExpired( if (!row.raw_path) continue; // Resolve through the same safeEik path-traversal rail the writer used, never the stored string. try { - fs.unlinkSync(path.join(rawDir, `${safeEik(row.eik)}.json`)); + unlink(path.join(rawDir, `${safeEik(row.eik)}.json`)); files++; } catch (e) { // Already gone is the goal state, not a failure. Anything else must surface — a purge that @@ -261,8 +261,18 @@ export function purgeExpired( ); for (const name of names) { if (!name.endsWith('.json') || known.has(name)) continue; - fs.unlinkSync(path.join(rawDir, name)); - orphans++; + // The same ENOENT tolerance the expired loop above has, and for a sharper reason here: this loop + // runs AFTER the DB DELETE has committed, so an unguarded throw half-purges — rows gone, files + // still on disk — and reports the whole run as failed. The listing is a snapshot, so a name can + // legitimately be gone by the time we reach it (a concurrent purge, an operator clearing scratch). + // Already gone is the goal state. Anything else still surfaces: a purge that cannot delete has + // left third-party names on disk, and reporting success would be the failure it exists to prevent. + try { + unlink(path.join(rawDir, name)); + orphans++; + } catch (e) { + if (e.code !== 'ENOENT') throw e; + } } return { rows: expired.length, files, orphans }; } diff --git a/scripts/tr/cache.test.mjs b/scripts/tr/cache.test.mjs index 8046c1ad..a7357ec8 100644 --- a/scripts/tr/cache.test.mjs +++ b/scripts/tr/cache.test.mjs @@ -229,3 +229,65 @@ test('purgeExpired defaults to the 35-day window and tolerates an already-missin assert.equal(res.files, 0); assert.ok(readDeed(db, '201122335'), 'the 21-day-old deed is inside the window'); })); + +test('purgeExpired tolerates an orphan that vanished under it, and keeps sweeping', () => { + // The benign race: readdirSync lists a name, and it is gone by the time unlink runs (a concurrent + // purge, an operator clearing scratch/). The expired loop right above has always tolerated this; + // the orphan loop did not, and it throws AFTER the DB DELETE has committed — so the run half-purges, + // reports "purge failed", and leaves the operator unable to tell "already gone" from "an orphan + // still holding third-party names". Injected rather than staged, because the race cannot be timed + // from a test; the injection seam matches the one httpGet/sleep/now already use in this codebase. + return withRaw((db, rawDir) => { + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-08-01T00:00:00Z' })); + writeRaw(rawDir, '115536179'); + writeRaw(rawDir, '204556676'); // orphan 1 — disappears under us + writeRaw(rawDir, '831391124'); // orphan 2 — must still be swept + + const seen = []; + const res = purgeExpired(db, rawDir, { + now: new Date('2026-08-05T00:00:00Z'), + unlink: (p) => { + seen.push(path.basename(p)); + if (p.endsWith('204556676.json')) { + const err = new Error('ENOENT: no such file or directory'); + err.code = 'ENOENT'; + throw err; + } + fs.unlinkSync(p); + }, + }); + + assert.deepEqual(seen.sort(), ['204556676.json', '831391124.json'], 'both orphans attempted'); + assert.equal( + res.orphans, + 1, + 'a file that was already gone was not deleted BY US — do not count it', + ); + assert.equal(fs.existsSync(path.join(rawDir, '831391124.json')), false, 'the sweep continued'); + assert.equal( + fs.existsSync(path.join(rawDir, '115536179.json')), + true, + 'the live deed is untouched', + ); + }); +}); + +test('purgeExpired still refuses loudly on an orphan it could not delete for a REAL reason', () => { + // The other half, and the reason the guard is ENOENT-only. A permission error means retained PII is + // still on disk; reporting success would be the failure mode the purge exists to prevent. + return withRaw((db, rawDir) => { + writeRaw(rawDir, '204556676'); + assert.throws( + () => + purgeExpired(db, rawDir, { + now: new Date('2026-08-05T00:00:00Z'), + unlink: () => { + const err = new Error('EACCES: permission denied'); + err.code = 'EACCES'; + throw err; + }, + }), + /EACCES/, + ); + }); +}); From aee14278fa891328e63ea70c4a58d87850be687b Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Tue, 11 Aug 2026 17:49:16 +0300 Subject: [PATCH 43/69] docs(adr): record the monthly cadence as a decision, not a YAML comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merging the crawl into the decision job forced the schedule from daily to monthly, which contradicts ADR-0033's own consequence line ("decisions daily, registry lookups monthly"). That line is now false, and the truth lived only in a workflow comment — the one place a reader looking for the decision would not go, and nowhere anyone could argue with it. ADR-0034 records it properly: the two constraints that collide (raw deeds must not outlive the runner, because an Actions cache entry sits under a restore-keys chain the 35-day retention cannot reach; and the decision cannot be made without them, because the strongest rung compares the declarant's name against the deed text the index deliberately refuses to store), and both rejected alternatives — persisting the deeds behind someone else's eviction policy, and having the crawl emit per-(link, ЕИК) verdicts so only booleans cross the boundary. The second is the right way back to a daily decision and is named as such rather than left implicit. ADR-0033 is not rewritten (repo convention: an accepted ADR is superseded, not edited) — the stale line carries a pointer to its successor and is scoped to the cadence claim alone. The workflow comment now says it is the operational summary, not the record. Also notes the consequence a monthly rebuild does NOT have: a takedown does not wait for the schedule. It goes through ADR-0031 and, on live exposure, a direct UPDATE on the served D1. --- .github/workflows/related-persons-data.yml | 5 +- ...-evidence-replaces-name-distinctiveness.md | 6 +- ...ups-and-decisions-share-one-monthly-run.md | 63 +++++++++++++++++++ docs/adr/README.md | 1 + 4 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index 59db1219..e352a787 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -21,7 +21,10 @@ on: # its own copy of the credential guards, the D1-target guard and the ship floor — and the copy is the # one nobody exercises. # - # WHY MONTHLY AND NOT DAILY, which is what §9 and ADR-0033 describe. A decision cannot be recomputed + # WHY MONTHLY AND NOT DAILY, which is what §9 and ADR-0033 originally describe. The decision and its + # rejected alternatives are recorded in ADR-0034, which supersedes ADR-0033's cadence line — this + # comment is the operational summary, not the record, so a reader who disagrees has somewhere to argue. + # A decision cannot be recomputed # without the raw deeds — evidenceVerdict's strongest rung matches the declarant's name against the # register's own text — and the raw deeds must not be persisted between runs: they carry the names and # addresses of co-owners and managers who hold no public office, and an Actions cache entry lives on diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md index cfa579ed..0993a41b 100644 --- a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -327,7 +327,11 @@ None of these are follow-ups. function of declarations, cached deeds and contracts, but they no longer run on the 6-hourly ETL cycle (#279 §9 assumes they do; they do not — the loader needs `node:sqlite`, the declaration corpus and the full winner set, none of which exist inside a Worker). Cadence becomes two scheduled workflows: decisions - daily, registry lookups monthly. + daily, registry lookups monthly. **Superseded by [ADR-0034](0034-registry-lookups-and-decisions-share-one-monthly-run.md):** + the split rests on the decision run being able to work from the cache alone, and it cannot — the strongest + rung compares the declarant's name against the deed text itself, which the index deliberately does not + store, so the raw deeds must be present; and they must not survive the runner. One monthly job, therefore. + Nothing else in this ADR is affected. - Two capabilities are deliberately left on the table and become separate changes under new rules versions: as-of evidence via `?entryDate=`, and the §7 „и към днешна дата" labels with their post-period contracts. - The seat rung is structurally capped and we accept the cap: a declared seat is captured **only** from the diff --git a/docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md b/docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md new file mode 100644 index 00000000..c9afd007 --- /dev/null +++ b/docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md @@ -0,0 +1,63 @@ +# ADR-0034 — Справките в ТР и решенията текат в едно месечно задание + +- **Статус:** Прието +- **Дата:** 2026-08-11 +- **Обхват:** конвейерът за свързани лица — `related-persons-data.yml`, `scripts/cacbg/load.mjs`, + `scripts/tr/fetch-deeds.mjs`. Заменя единствено твърдението за каденцията в последствията на + [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md); всичко останало в него стои. + +## Контекст + +ADR-0033 записа каденцията така: **„решения ежедневно, справки в регистъра месечно"**. Разделянето +почиваше на едно предположение — че решаващият ход може да работи само върху кеша, а обхождането е +отделен, по-рядък ход. Прегледът на PR-а показа, че това предположение не се държи. + +Двете страни на противоречието: + +1. **Суровите актове не бива да преживяват изпълнителя.** Кешираният акт носи имена на трети лица — + съдружници и управители без публична длъжност, и адреса на дружеството. Пренасянето му между ходове + изисква хранилище (кеш на Actions или артефакт), чието изгонване е политика за капацитет, не за + задържане: 35-дневният срок по решение 5 на ADR-0033 се изпълнява от `purgeExpired` на изпълнителя и + няма никаква власт над копие в хранилището на GitHub. Това беше находка B3 на прегледа. +2. **Решението не може да се вземе без суровите актове.** Най-силното стъпало сравнява трите имена на + декларатора със самия текст на записа. Индексът на кеша нарочно не пази нито едно име (§9), така че + „кеширай само индекса" премахва точно данните, от които `evidenceVerdict` се нуждае. + +Двете заедно значат: **всеки ход, който решава, трябва и да обхожда.** А ежедневно решение тогава значи +~400 заявки дневно към чужд публичен регистър, което спец. §3.3 („НИКОГА не обхождай ТР масово"; +КЗЛД, CJEU C-200/23) не разрешава да се приема с лека ръка — ограничителят на регистъра е единственият +начин операторът да изрази предпочитание за темп и ние не се настройваме около него емпирично. + +## Решение + +**Едно задание, месечно.** `related-persons-data.yml` изпълнява последователно: зареждане на списъка с +кандидати (`load.mjs --emit-candidates`, върху захвърляемо копие) → обхождане на регистъра → решение и +публикуване (`load.mjs`) → одит (`audit.mjs`) → изтриване на суровите актове в стъпка `if: always()`. +`scratch/tr` **не се кешира никога**; кешира се само `scratch/cacbg/raw` (публични декларации). + +`related-persons-tr-refresh.yml` е премахнат — нямаше какво да разделя. + +**Отхвърлена алтернатива: пренасяне на актовете между ходове** (кеш или артефакт с правило за +жизнен цикъл). Работи технически, но връща B3: срокът на задържане престава да е наш и става политика на +чуждо хранилище. R2 със собствено правило за изтриване остава разумен вариант, ако някога се наложи — +днес не се налага. + +**Отхвърлена алтернатива: ежедневно решение.** Единственият начин да се запази е обхождането да издава +**присъди по (връзка, ЕИК)** — булеви стойности без нито едно име — така че само те да пресичат +границата между ходовете. Това е коректната посока и остава на масата, но е промяна в дизайна на +доказателствения слой, не настройка на график, и не влиза в тази промяна. + +## Последствия + +- Публикуваната повърхност се преизчислява **месечно**, не ежедневно. Практическата цена е по-малка, + отколкото звучи: входът също се движи бавно (декларациите се подават годишно, вписванията в ТР — + рядко), а датата на справката се показва на всяка връзка, така че давността е видима за читателя, а не + подразбираща се. +- **Спешна корекция не чака графика.** Свалянето на връзка минава през ADR-0031 и ръчния ход, а при + жива експозиция — през пряк `UPDATE` върху обслужващата D1; вж. + [ръководството](../runbooks/related-persons-suppression.md). Каденцията определя кога се преоткриват + връзки, не кога може да се махне една. +- Едно задание значи един режим на отказ: ако обхождането спре при 429, гейтът за покритие на `load.mjs` + отказва частичния кеш и ходът не публикува нищо. Това е желаната посока — по-скоро без обновяване, + отколкото с орязана повърхност — и е същият гейт, който вече пазеше разделения вариант. +- #279 §9 продължава да описва ежедневни решения. Този ADR е записът, че не е така, и защо. diff --git a/docs/adr/README.md b/docs/adr/README.md index ecaa107f..f0a8193b 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -41,5 +41,6 @@ | [0031](0031-suppressions-version-controlled-fingerprinted.md) | Свалянията на връзки са версиониран списък с HMAC-отпечатък (salt = CI secret), прилаган при зареждане — не служебна таблица; не изтича сигнала „кой е свален" към прод | Прието | | [0032](0032-family-ownership-published-under-public-interest.md) | Дял на свързано лице се публикува на поименната повърхност наравно със собствения — на основание надделяващ обществен интерес (ЗДОИ чл. 41и; C-184/20 забранява само поименните данни); близкият никога не се назовава, връзката не се твърди; заменя ADR-0030 | Прието | | [0033](0033-registry-evidence-replaces-name-distinctiveness.md) | Връзка се публикува само срещу проверим факт от Търговския регистър — доказателственият ред заменя нивата по отличителност на името; евристиката вече обосновава твърдение (изрично изключение от ADR-0007), барът за акционерни форми е обединение от три сигнала, монотонността е гейт, не хранилище; заменя ADR-0009/0015/0017 | Прието (дизайн) | +| [0034](0034-registry-lookups-and-decisions-share-one-monthly-run.md) | Справките в ТР и решенията текат в едно месечно задание — суровите актове не преживяват изпълнителя, а решението не може да се вземе без тях; заменя твърдението за каденцията в ADR-0033 | Прието | Свързан проектен документ: [spec/related-persons-foundation.md](../spec/related-persons-foundation.md). From 36cdd9d0314124c36053c2fc4b5bf4ddbe33b95a Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Tue, 11 Aug 2026 18:17:13 +0300 Subject: [PATCH 44/69] chore(deps): raise nanoid past GHSA-2v37-7h3g-55p8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Dependency audit went red on a newly published advisory for nanoid 3.3.16 (GHSA-2v37-7h3g-55p8, CVSS 8.2), not on anything this branch changed — it touches neither package.json nor pnpm-lock.yaml, so main fails identically on its next run. Lockfile-only, and confined to nanoid: postcss@8.5.24 already declares `^3.3.16`, so 3.3.18 satisfies the existing range with no manifest edit and no override. `pnpm update nanoid` does not lift a transitive pin; `-r` does. Not suppressed in osv-scanner.toml on purpose — that file's own doctrine reserves entries for advisories whose only fix is unavailable or a major upgrade, and this one is a patch. Verified with the same checksum-pinned osv-scanner 2.4.0 the CI step runs: "No issues found". The package is build-time only (postcss → vite → vitest, all dev) and reaches no Worker bundle, but the chain is exercised anyway — full suite 6/6 packages, lint, typecheck 7/7, and a real `@sigma/web` production build through postcss. --- pnpm-lock.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 304ec682..6cf35954 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1643,8 +1643,8 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - nanoid@3.3.16: - resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} + nanoid@3.3.18: + resolution: {integrity: sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -3369,7 +3369,7 @@ snapshots: ms@2.1.3: {} - nanoid@3.3.16: {} + nanoid@3.3.18: {} node-releases@2.0.45: {} @@ -3406,7 +3406,7 @@ snapshots: postcss@8.5.24: dependencies: - nanoid: 3.3.16 + nanoid: 3.3.18 picocolors: 1.1.1 source-map-js: 1.2.1 From 4d21ba0c6b262ab50922298ebf6ede23a050d250 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:17:16 +0300 Subject: [PATCH 45/69] =?UTF-8?q?fix(cacbg):=20bar=20a=20listed=20=D0=90?= =?UTF-8?q?=D0=94=20whose=20seat=20carries=20neither=20comma=20nor=20dot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `closelyHeldForm` asked whether the name ENDS in a joint-stock form. A declarant who types „ТРЕЙС ГРУП ХОЛД АД София" defeats that: the comma-peel needs a comma and SEAT_MARKER needs a literal dot, so neither strips the seat, the name stops ending in its form, and a listed АД reads as closely-held — the „11 акции на Trace → €88M" trap, reached from the plainest input shape there is. Decide on the LAST form TOKEN instead of an end anchor. Position-independent, so a trailing seat cannot flip the verdict, while „АД" leading („АД ГРУП ООД") or glued („АД-ХОК ЕООД") still isn't the form. JOINT_STOCK itself is untouched — deed.mjs's envelope test reads a registry-clean `fullName` that does end in its form, and deed.test.mjs pins the two patterns to each other. `stripSeatSuffix` gains the same cut for nameDistinctiveness, where the blind spot fails toward PUBLISHING: an uncounted seat token inflates the content-word count to 3 and reads as distinctive. Truncation is anchored on фирма-TERMINATING forms only — ЕТ/СД/КД precede the name, and cutting after them would eat it. Positive controls throughout: a predicate that always returned false would pass every new bar assertion, so each is paired with a name that must stay material. --- scripts/cacbg/classify.mjs | 50 ++++++++++++++++++++++++++------ scripts/cacbg/classify.test.mjs | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 8 deletions(-) diff --git a/scripts/cacbg/classify.mjs b/scripts/cacbg/classify.mjs index a5f9de25..d066a3fa 100644 --- a/scripts/cacbg/classify.mjs +++ b/scripts/cacbg/classify.mjs @@ -33,6 +33,24 @@ const FORM_TOKENS = new Set([ // generic/withhold — the safe side), never fabricates a joint-stock exclusion. Operates on UPPERCASE input. const SEAT_MARKER = /\s+(?:ГР|С|ОБЩ|ОБЛ|Ж\.К)\.\s*\S[^,]*$/u; const hasFormToken = (s) => s.split(/[^А-ЯЁ]+/).some((t) => FORM_TOKENS.has(t)); + +// The legal forms that TERMINATE a фирма (ЗТРРЮЛНЦ writes them as a suffix), so anything after one is a +// seat or a qualifier — never part of the name. ЕТ/СД/КД/КООПЕРАЦИЯ/ФОНДАЦИЯ/СДРУЖЕНИЕ are deliberately +// ABSENT: those PRECEDE the фирма („ЕТ Алекс Петров Димитров"), and truncating after them would eat the +// name itself — turning a distinctive ЕТ into a generic one and withholding a true link. +const SUFFIX_FORMS = new Set(['ЕООД', 'ООД', 'ЕАД', 'АД', 'АДСИЦ', 'КДА', 'ДЗЗД']); + +// Cut everything after the LAST фирма-terminating form token. This is the case the comma-peel and the +// marker strip both miss: „ТРЕЙС ГРУП ХОЛД АД София" has neither a comma nor a „гр." dot, so it survived +// both, stopped ending in its form, and defeated every end-anchored form test downstream. Token-exact +// (never a substring), so „КАДИЕВ ГЛОБАЛ ЕООД" and „АД-ХОК ЕООД" are untouched. +function stripAfterSuffixForm(s) { + const re = /[А-ЯЁ]+/gu; + let last = null; + for (let m = re.exec(s); m !== null; m = re.exec(s)) if (SUFFIX_FORMS.has(m[0])) last = m; + return last === null ? s : s.slice(0, last.index + last[0].length).trim(); +} + function stripSeatSuffix(upper) { let s = String(upper).trim(); // Peel trailing comma-clauses right-to-left while the clause bears no legal form (i.e. it's a seat, not @@ -45,7 +63,7 @@ function stripSeatSuffix(upper) { ) { s = m[1].trim(); } - return s.replace(SEAT_MARKER, '').trim(); + return stripAfterSuffixForm(s.replace(SEAT_MARKER, '').trim()); } /** @@ -81,6 +99,9 @@ const norm = (s) => // directory without closing a cacbg↔tr cycle. A test in deed.test.mjs pins the two identical; change one // and that test fails rather than the two silently diverging on a legal form. export const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; +// The same four forms as whole tokens. Kept in step with JOINT_STOCK above by classify.test.mjs, and with +// deed.mjs's JOINT_SUFFIX twin by deed.test.mjs — three spellings of one rule, all three pinned. +const JOINT_STOCK_FORMS = new Set(['АД', 'ЕАД', 'АДСИЦ', 'КДА']); /** * Materiality by legal form. The public ownership surface is CLOSELY-HELD companies only (ООД/ЕООД/ЕТ/ * КД/СД/ДЗЗД or a form-unspecified name from the closely-held table). Joint-stock forms (АД/ЕАД/АДСИЦ/КДА, @@ -90,13 +111,26 @@ export const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КД * so it withholds rather than fabricates. @returns {boolean} true ⇒ material/closely-held. */ export function closelyHeldForm(name) { - return !JOINT_STOCK.test( - stripSeatSuffix( - String(name ?? '') - .normalize('NFC') - .toUpperCase(), - ), - ); + // LAST-FORM-TOKEN-WINS, not an end anchor. The anchor asked „does the name END in a joint-stock form?", + // which a declarant-typed cell can defeat just by appending a seat — and `stripSeatSuffix` cannot be + // trusted to have removed every shape of one. Asking instead „which legal form is the name's LAST?" + // is position-independent: a trailing seat, a stray qualifier, or nothing at all leaves the verdict + // unchanged, while „АД" leading („АД ГРУП ООД") or glued („АД-ХОК ЕООД") still isn't the form. + // + // This is why the predicate no longer uses JOINT_STOCK directly while deed.mjs's twin still does: that + // twin reads the deed envelope's `fullName` — a REGISTRY-clean name that genuinely ends in its form — + // whereas this one reads a free-text cell a human typed. Same rule, different input hygiene. + const tokens = stripSeatSuffix( + String(name ?? '') + .normalize('NFC') + .toUpperCase(), + ) + .split(/[^А-ЯЁ]+/u) + .filter(Boolean); + const lastForm = tokens.filter((t) => FORM_TOKENS.has(t)).at(-1); + // No form token at all ⇒ nothing says joint-stock ⇒ material, exactly as the anchor behaved. The bar + // only ever fires on an EXPLICIT joint-stock form, so it withholds rather than fabricates. + return lastForm === undefined || !JOINT_STOCK_FORMS.has(lastForm); } // seatConfirmed() and publishTier() lived here until #279. The publish tiers they produced diff --git a/scripts/cacbg/classify.test.mjs b/scripts/cacbg/classify.test.mjs index e0e8b2fa..f0f971ba 100644 --- a/scripts/cacbg/classify.test.mjs +++ b/scripts/cacbg/classify.test.mjs @@ -5,6 +5,7 @@ import { temporalStatus, localityToken, closelyHeldForm, + JOINT_STOCK, } from './classify.mjs'; test('nameDistinctiveness: numbers / Latin / ≥3 words are distinctive; bare 1-2 word Cyrillic is generic', () => { @@ -90,6 +91,56 @@ test('closelyHeldForm: a trailing седалище after the form does not flip assert.equal(closelyHeldForm('СТРОЙ, ИНВЕСТ ООД'), true); // trailing clause has ООД → closely-held }); +test('the token set and the JOINT_STOCK regex name the SAME four forms', () => { + // closelyHeldForm now decides on the last form TOKEN while deed.mjs's envelope test still uses the + // end-anchored REGEX. Two spellings of one rule drift silently — the day one gains a form the other + // lacks, a joint-stock company passes one gate and is barred by the other. Pin them to each other. + // (deed.mjs's third spelling is pinned to the regex by deed.test.mjs:378.) + for (const form of ['АД', 'ЕАД', 'АДСИЦ', 'КДА']) { + assert.equal(JOINT_STOCK.test(`ФИРМА ${form}`), true, `regex misses ${form}`); + assert.equal(closelyHeldForm(`ФИРМА ${form}`), false, `token set misses ${form}`); + // …and the mirror: every form the regex accepts must be one the token set bars, seat or no seat. + assert.equal(closelyHeldForm(`ФИРМА ${form} София`), false, `token set misses ${form} + seat`); + } + // Every FORM_TOKEN that is NOT one of the four must read as closely-held. + for (const form of ['ЕООД', 'ООД', 'ЕТ', 'ДЗЗД', 'КД', 'СД']) { + assert.equal(closelyHeldForm(`ФИРМА ${form}`), true, `${form} wrongly barred`); + } +}); + +test('closelyHeldForm: a seat with NO comma and NO „гр." marker still cannot flip an АД (libel)', () => { + // The gap the marker/comma rules leave open. `SEAT_MARKER` requires a literal dot and the comma-peel + // requires a comma, so „ТРЕЙС ГРУП ХОЛД АД София" — the plainest way a declarant writes it — survives + // both, no longer ENDS in the form, and the end-anchored JOINT_STOCK test misses it. A listed АД then + // reads as closely-held: the „11 акции на Trace → €88M" trap, from the one input shape nothing strips. + assert.equal(closelyHeldForm('ТРЕЙС ГРУП ХОЛД АД София'), false); + assert.equal(closelyHeldForm('ТРЕЙС ГРУП ХОЛД АД СОФИЯ'), false); + assert.equal(closelyHeldForm('Транспроект ЕАД Пловдив'), false); + assert.equal(closelyHeldForm('НЕС АДСИЦ Варна'), false); + assert.equal(closelyHeldForm('АЛФА КДА София'), false); + // POSITIVE CONTROLS — the bar must stay a bound, not become a blanket. A predicate that always returned + // false would pass every assertion above; these are what distinguish the fix from that (ADR-0027). + assert.equal(closelyHeldForm('Вамос ООД Русе'), true); // dot-less seat on a closely-held form + assert.equal(closelyHeldForm('ЕНЕРДЖИ СЪПЛАЙ ЕООД Бургас'), true); + assert.equal(closelyHeldForm('АД ГРУП ООД'), true); // leading „АД" is not the form + assert.equal(closelyHeldForm('АД-ХОК ЕООД'), true); // „АД" glued by a hyphen is not a form token + assert.equal(closelyHeldForm('КДА ГРУП ООД'), true); + assert.equal(closelyHeldForm('КАДИЕВ ГЛОБАЛ ЕООД'), true); // „АД" inside a word + assert.equal(closelyHeldForm('ЕТ Алекс'), true); // ЕТ leads the фирма — nothing after it is a seat +}); + +test('nameDistinctiveness: a dot-less trailing city is not counted as a content word either', () => { + // Same blind spot, and here it fails toward PUBLISHING: an uncounted seat token inflates the content-word + // count to 3 ⇒ 'distinctive'. Since #279 rung 2 gates an uncorroborated „Документ" publish on exactly this + // predicate (ADR-0035), a seat read as a content word is a false company-identity claim, not just noise. + assert.equal(nameDistinctiveness('СТРОЙ ИНВЕСТ ООД София'), 'generic'); + assert.equal(nameDistinctiveness('НИКАС КОМЕРС ЕООД Пловдив'), 'generic'); + // POSITIVE CONTROLS: a genuinely ≥3-content-word фирма stays distinctive, and a leading-form ЕТ name + // keeps its content words — nothing after „ЕТ" is a seat, so the strip must not reach them. + assert.equal(nameDistinctiveness('ХИДРО СТРОЙ МОНТАЖ ЕООД София'), 'distinctive'); + assert.equal(nameDistinctiveness('ЕТ АЛЕКС ПЕТРОВ ДИМИТРОВ'), 'distinctive'); +}); + test('nameDistinctiveness: a trailing city is not counted as a content word (no premature publish)', () => { // The exact over-publish case: 2 real content words + a seat token would read as 3 → distinctive. assert.equal(nameDistinctiveness('СТРОЙ ИНВЕСТ ООД, СОФИЯ'), 'generic'); // was distinctive via +СОФИЯ From 37a3db8655c56a034694495bac0d0449dc94fcde Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:22:06 +0300 Subject: [PATCH 46/69] fix(cacbg): date a filing by its folder when the declared year is unreadable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `if (!Number.isFinite(fy)) continue` looked like the surrounding fail-safe — "never withdraw on missing evidence" — but it is the opposite. An absent filing legitimately yields no horizon; a PRESENT filing that is silently dropped never advances one, so `divested` stays false and a stake the official has since sold keeps naming them on the public surface. Stale claim, real person, no symptom in the output. filings.jsonl already carries `folder` (extract.mjs), so fall back to it. The folder is the PUBLICATION year and runs ahead of the declared year, so the horizon can advance up to a year early — that errs toward withdrawing a claim we are no longer sure of, which is the safe direction on this surface. Documented at the fallback rather than left for a reader to infer. A filing datable by NEITHER field is still ignored: this dates a filing, it does not invent one. Both counts are reported, because a horizon that failed to advance is invisible — without the counter, a corpus-wide date regression would show up only as a surface that quietly stopped withdrawing anything. The RED publishes via the SEAT rung, not rung 2: an owner-matched deed would let §7 reconciliation reverse the divestment and mask the bug. Mutation-checked — removing the fallback fails the test with published/withdrawn. Positive control: an undatable filing leaves its link published. --- scripts/cacbg/load.mjs | 31 +++++++++++- scripts/cacbg/load.test.mjs | 99 ++++++++++++++++++++++++++++++++++--- 2 files changed, 121 insertions(+), 9 deletions(-) diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index e3da7f8a..aecd680a 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -355,14 +355,41 @@ const familyMaterialByTemplate = new Map(); // declare a stake ONLY in the interests declaration). One record per declaration incl. empty / no-material ones, // so a same-type divest-to-ZERO still advances the horizon. An absent/typeless file just yields no match ⇒ the // link is kept (fail-safe: never withdraw on missing evidence). +// +// A filing whose is unreadable falls back to its FOLDER year (#279 §1.3). Dropping the record +// instead — the previous behaviour — is not the fail-safe it resembles: an undated filing that vanishes +// never advances the horizon, so `divested` stays false and a stake the official has since SOLD keeps +// naming them on the public surface. That is a stale claim about a real person, which is the failure this +// surface can least afford. The folder year is an APPROXIMATION — it is the publication year and runs +// ahead of the declared year (migration 0003) — so it can advance the horizon by up to a year early. That +// errs toward WITHDRAWING a claim we are no longer sure of, which is the safe direction here. +// +// A filing datable by NEITHER field is still ignored: the fallback dates a filing, it does not invent one. const filingMaxByPersonType = new Map(); +let filingFolderDated = 0, + filingUndatable = 0; for (const f of readJsonl(path.join(STAGING, 'filings.jsonl'))) { if (!isMatchableKey(companyNameKey(f.person))) continue; - const fy = yr(f.year); - if (!Number.isFinite(fy)) continue; + let fy = yr(f.year); + if (!Number.isFinite(fy)) { + fy = yr(f.folder); + if (!Number.isFinite(fy)) { + // Counted, not silently dropped. A horizon we failed to advance is invisible in the output — the + // link simply stays up — so without this the only symptom of a corpus-wide date regression would be + // a surface that quietly stopped withdrawing anything. + filingUndatable++; + continue; + } + filingFolderDated++; + } const k = `${personId(f.person, f.institution)}|${f.template ?? ''}`; filingMaxByPersonType.set(k, Math.max(filingMaxByPersonType.get(k) ?? fy, fy)); } +if (filingFolderDated > 0 || filingUndatable > 0) { + console.log( + ` filings: ${filingFolderDated} dated by FOLDER (unreadable ), ${filingUndatable} undatable (ignored — no horizon)`, + ); +} db.exec('BEGIN'); for (const h of readJsonl(path.join(STAGING, 'holdings.jsonl'))) { diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index de7e3d80..db60a560 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -170,6 +170,15 @@ before(() => { -- stake at all. The empty filing advances his horizon past 2019 → the 2019 stake is withdrawn. INSERT INTO bidders VALUES ('eik:101010104','ДИВЕСТ ЗЕРО 4 ЕООД','101010104',1,'София'); INSERT INTO contracts VALUES ('c11','t1','eik:101010104','2019-05-01',150000); + -- §1.3 unparseable filing YEAR: Безгодин owns ДИВЕСТ БЕЗГОД (2019), then files a later declaration whose + -- is unreadable while its FOLDER carries 2023. The folder must supply the horizon, or the filing + -- is dropped, the horizon never advances, and a sold stake stays published — a stale public claim. + INSERT INTO bidders VALUES ('eik:212121218','ДИВЕСТ БЕЗГОД 9 ЕООД','212121218',1,'София'); + INSERT INTO contracts VALUES ('c17','t1','eik:212121218','2019-05-01',120000); + -- §1.3 positive control: Дрънкан's later filing is datable by NEITHER NOR folder. It must be + -- ignored, never guessed — an undatable filing is no evidence of a sale, so this stake stays published. + INSERT INTO bidders VALUES ('eik:232323231','ДРЪНКАН ТЕХ 10 ЕООД','232323231',1,'София'); + INSERT INTO contracts VALUES ('c18','t1','eik:232323231','2019-05-01',110000); -- N10 canonicalization: Канонов owns КАНОН ТЕХ 5, filing „МВР" one year and the full ministry name the -- next → ONE identity, ONE link (not a split). Distinctive name (number) → tier B publishable. INSERT INTO bidders VALUES ('eik:131313136','КАНОН ТЕХ 5 ЕООД','131313136',1,'София'); @@ -462,6 +471,43 @@ before(() => { holderRelation: 'self', controlHash: 'H14', }, + // §1.3 unparseable filing year: Безгодин's 2019 stake. His later filing (in filings.jsonl, no holdings + // row) carries an unreadable but a 2023 FOLDER — the fallback that must withdraw this stake. + { + folder: '2020', + xmlFile: 'BG0.xml', + year: '2019', + template: 'assets', + category: '', + institution: 'T3', + person: 'Безгодин Иванов Дивестов', + position: '', + entity: 'ДИВЕСТ БЕЗГОД 9 ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: 'гр. София', + holderRelation: 'self', + controlHash: 'H20', + }, + // §1.3 positive control: Дрънкан's 2019 stake, whose only later filing is undatable (see filings.jsonl). + { + folder: '2020', + xmlFile: 'DR0.xml', + year: '2019', + template: 'assets', + category: '', + institution: 'T4', + person: 'Дрънкан Иванов Тестов', + position: '', + entity: 'ДРЪНКАН ТЕХ 10 ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: '', + holderRelation: 'self', + controlHash: 'H21', + }, // B4 UNKNOWN holder: the holder cell is neither confidently the declarant's own name nor a relative's // (an ambiguous 1-token-different cell). classifyHolder → 'unknown' → this forms NO link (counted // nowhere), so a phantom relative never enters a published family figure. @@ -630,6 +676,27 @@ before(() => { person: 'Интер Иванов Тестов', institution: 'INT', }); + // §1.3: the divesting filing whose is UNREADABLE. `folder` carries 2023 and is the only thing that + // can date it. Dropping the record leaves the horizon at 2019, `divested` false, and a sold stake on the + // public surface — the failure direction that matters here, since the claim names a real official. + filings.push({ + folder: '2023', + xmlFile: 'BG1.xml', + year: 'н/д', + template: 'assets', + person: 'Безгодин Иванов Дивестов', + institution: 'T3', + }); + // POSITIVE CONTROL for the same fallback: a filing datable by NEITHER field must still be ignored, not + // guessed at. Дрън's stake stays published — an undatable filing is no evidence of a sale. + filings.push({ + folder: 'архив', + xmlFile: 'DR1.xml', + year: '', + template: 'assets', + person: 'Дрънкан Иванов Тестов', + institution: 'T4', + }); fs.writeFileSync( path.join(STAGING, 'filings.jsonl'), filings.map((f) => JSON.stringify(f)).join('\n') + '\n', @@ -659,6 +726,11 @@ before(() => { 181818187: { owners: ['АЛФА ИВАНОВ ПАРТНЬОРОВ', 'БЕТА ИВАНОВ ПАРТНЬОРОВ'] }, 121212129: { owners: ['НУЛА ИВАНОВ ТЕСТОВ'] }, 191919199: { owners: ['РОДНИНА ВТОРА'], seat: 'гр. Русе' }, // family + declared seat → confirmed + // Безгодин is ABSENT from the deed (so §7 reconciliation cannot reverse the divestment) but his + // declared seat matches the registered one, so rung 3 says „Потвърдено" and the link PUBLISHES. + // Only the folder-dated divestment withdraws it — making the pre-fix failure the dangerous one. + 212121218: { owners: ['ДРУГ СОБСТВЕНИК СЪВСЕМ'], seat: 'гр. София' }, + 232323231: { owners: ['ДРЪНКАН ИВАНОВ ТЕСТОВ'] }, // still the owner → the undatable filing changes nothing }); }); @@ -855,6 +927,17 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // that asset-declaration silence as a sale and WITHDRAWS the stake; the per-type horizon must not — no later // INTERESTS filing omits the company. This link must stay PUBLISHED. (Guards against dropping a true link: // 13% of holders declare a stake only in the interests declaration.) + // §1.3 unparseable filing YEAR: Безгодин's later declaration has an unreadable ('н/д') but a 2023 + // FOLDER. Dropping that record — the pre-fix behaviour — leaves his horizon at 2019, so `divested` stays + // false and a stake he no longer holds keeps naming him on the public surface. The folder must date it. + const noYear = link('212121218', 'Безгодин Иванов Дивестов'); + assert.equal(noYear.interest_class, 'private_ownership'); + assert.equal(noYear.status, 'withdrawn'); + // POSITIVE CONTROL: datable by NEITHER field ⇒ ignored, not guessed. The fallback must not become a + // licence to invent a horizon — an undatable filing is no evidence of a sale, so this link stays up. + const noDate = link('232323231', 'Дрънкан Иванов Тестов'); + assert.equal(noDate.status, 'published'); + const crossType = link('161616163', 'Интер Иванов Тестов'); assert.equal(crossType.interest_class, 'private_ownership'); assert.equal(crossType.status, 'published'); // NOT withdrawn — the later asset filing is a different type @@ -917,15 +1000,17 @@ test('re-run is idempotent and honors the suppression list (contested link stays 'suppressed', ); // idempotent: still exactly the same number of links + persons after a clean rebuild. - // 17 links: 12 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero - // 'withdrawn' + Интер's per-type-kept published link) + 2 family (Кмет's, now held for want of - // registry evidence, and Кметица's seat-confirmed one) + Канонов's canonicalized single link + + // 19 links: 14 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero + // 'withdrawn' + Безгодин's folder-dated 'withdrawn' + Дрънкан's undatable-filing 'published' + + // Интер's per-type-kept published link) + 2 family (Кмет's, now held for want of registry evidence, + // and Кметица's seat-confirmed one) + Канонов's canonicalized single link + // Алфа & Бета (two officials on one winner, ПАРТНЬОРИ 5); Мария (quarantined), Акционер (securities), // Двусмислен (unknown holder) & Безинст (empty institution) none. - assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 17); - // 20 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, - // zero-contract Нула and the two ПАРТНЬОРИ co-owners; Канонов's two institution-variant filings fold to ONE. - assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 20); + assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 19); + // 22 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, + // zero-contract Нула, the two ПАРТНЬОРИ co-owners and the two §1.3 filing-date cases; Канонов's two + // institution-variant filings fold to ONE. + assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 22); db.close(); }); From 8e6f1dc876cd7219152cc3694458c4c4f3228889 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:24:00 +0300 Subject: [PATCH 47/69] fix(cacbg): do not read an unresolvable holder column as an own stake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `colNum` always returns a fallback, so a table that renumbers the holder column resolves it to a column that is not in the row. `by[cHolder]` is then undefined, the caller reads '' and `classifyHolder` maps a blank cell to 'self' — its documented meaning. A RELATIVE's stake becomes the official's own, enters the OWN-only refutation path and publishes as their private_ownership. The distinction the parser needs is column RESOLVABLE-BUT-EMPTY (self, a blank cell does mean the declarant) versus column NOT RESOLVABLE (unknown — we never read the holder, so we make no claim). `colNumOrNull` supplies it for the holder column alone; the other five call sites keep `colNum` and its fallback verbatim. Scope: parse.mjs was byte-identical to main. It is touched here because #279 §5 item 4 scheduled §1.4 for this PR, and this defect fails toward publishing a stake against a person who did not declare it as theirs. --- scripts/cacbg/parse.mjs | 21 +++++++++++++++++---- scripts/cacbg/parse.test.mjs | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/scripts/cacbg/parse.mjs b/scripts/cacbg/parse.mjs index bf95f154..6e89e242 100644 --- a/scripts/cacbg/parse.mjs +++ b/scripts/cacbg/parse.mjs @@ -82,10 +82,18 @@ function cellsByNum(row) { } // find the @_Num of the first column whose @_Description matches `re` (labels live on the header row) function colNum(firstRow, re, fallback) { + const found = colNumOrNull(firstRow, re); + return found ?? fallback; +} +// The same resolution WITHOUT the fallback, for the one column where "I could not find it" and "I found it +// and it was blank" must stay distinguishable. `colNum` collapses them: an unresolvable column resolves to +// a Num that is not in the row, `by[col]` is undefined, and the caller reads an empty cell it never found. +// For the HOLDER column those two readings differ by who owns the stake (§1.4) — see its call site. +function colNumOrNull(firstRow, re) { for (const c of asArray(firstRow?.Cell)) { if (c?.['@_Num'] && re.test(String(c?.['@_Description'] ?? ''))) return c['@_Num']; } - return fallback; + return null; } const year4 = (s) => String(s ?? '').match(/\b(20\d{2})\b/)?.[1] ?? null; @@ -159,15 +167,20 @@ function parseAssets(pp) { ? colNum(rows[0], /наименование.*дружеств|фирма/i, '4') : colNum(rows[0], /емитент/i, '6'); const cSeat = colNum(rows[0], /седалище/i, '5'); - const cHolder = colNum(rows[0], /собствено.*фамил/i, isOod ? '7' : '8'); + // NO fallback here, unlike every other column (§1.4). A blank holder cell MEANS something — the stake is + // the declarant's own (classifyHolder) — so a column we failed to resolve must not imitate one. With a + // fallback, a renumbered table resolves the holder to a missing column, reads '' and calls a RELATIVE's + // stake the official's own, which then publishes as their private_ownership. Unresolvable ⇒ 'unknown', + // which forms no link at all: we did not read the holder, so we make no claim about them. + const cHolder = colNumOrNull(rows[0], /собствено.*фамил/i); const cEgn = colNum(rows[0], /^егн$/i, isOod ? '8' : '9'); for (const row of rows) { const by = cellsByNum(row); const company = by[cCompany] ?? ''; if (!company) continue; if ((by[cEgn] ?? '').length > 0) egnPresent = true; - const holder = by[cHolder] ?? ''; - const holderRelation = classifyHolder(holder, declarant); + const holderRelation = + cHolder === null ? 'unknown' : classifyHolder(by[cHolder] ?? '', declarant); if (holderRelation === 'related') familyHoldingCount += 1; const seat = isOod ? (by[cSeat] ?? '') : ''; interests.push({ diff --git a/scripts/cacbg/parse.test.mjs b/scripts/cacbg/parse.test.mjs index 30495481..6a9b835e 100644 --- a/scripts/cacbg/parse.test.mjs +++ b/scripts/cacbg/parse.test.mjs @@ -249,3 +249,35 @@ test('parseList: only values shaped like a declaration filename are announced', assert.equal(shape('Уведомление'), 0, 'a title slotted into the filename field'); assert.equal(shape('AAAA.pdf'), 0, 'not a declaration document'); }); + +test('§1.4: an UNRESOLVABLE holder column is unknown, not an own stake', () => { + // `colNum` always returns a fallback, so a table that renumbers the holder column away resolves it to a + // column that is not there: `by[cHolder]` is undefined → '' → classifyHolder('') → 'self'. A RELATIVE's + // stake then enters the OWN-only path and publishes as the official's private_ownership — the surface's + // worst failure, since the official did not declare that stake as theirs. + // + // The required distinction is column RESOLVABLE-BUT-EMPTY (→ self, a blank cell means the declarant) + // versus column NOT RESOLVABLE (→ unknown, we have not read the holder at all). + const renumbered = ` + 1 + "РЕНОМЕР" ЕООД + Мария Спасова Роднинска`; + const d = parseDeclaration(assetDecl({ rows: renumbered }), 'REN.xml'); + const it = d.interests.find((i) => i.entity === '"РЕНОМЕР" ЕООД'); + assert.equal(it.holderRelation, 'unknown'); + + // POSITIVE CONTROL 1 — the column resolves and the cell is genuinely EMPTY: still the declarant's own + // stake. Collapsing both cases to 'unknown' would silently drop every stake declared this way. + const blankHolder = ` + 1 + "ПРАЗНА" ЕООД + `; + const blank = parseDeclaration(assetDecl({ rows: blankHolder }), 'BLK.xml'); + assert.equal(blank.interests[0].holderRelation, 'self'); + + // POSITIVE CONTROL 2 — a resolvable column still classifies a relative and the declarant correctly, so + // the change bounds one case rather than blanketing the parser. + const ok = parseDeclaration(assetDecl({ rows: selfRow + familyRow }), 'OK.xml'); + assert.equal(ok.interests.find((i) => i.entity === '"ТЕСТ АГРО" ЕООД').holderRelation, 'self'); + assert.equal(ok.interests.find((i) => i.entity === '"ФАМИЛНА" ЕООД').holderRelation, 'related'); +}); From ce5b2ada4df81c2639ba4549b8cb2e7e73ed1afc Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:32:21 +0300 Subject: [PATCH 48/69] fix(tr): require the registry evidence to establish the COMPANY, not only the person MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0033 held that rung 2 needs no gate because "the register named this person in THIS company". That holds only if THIS company is the one the official declared, and it is not guaranteed. `resolveEntity` maps a declared фирма to the sole ЕИК among PROCUREMENT WINNERS, and `nameGloballyUnique` ranges over bidders — never the register at large. So an official who owns a same-named company that never bid resolves to an unrelated winner, and a three-token homonym in that winner's deed completes a link false in both halves. Two coincidences, neither rare in Bulgaria. The ladder moved the name-collision risk into the Register instead of closing it, which is the premise #279 rests on. Rung 2 now asks for a reason to believe the company is the declared one: a declared ЕИК (the identifier resolves it outright), a declared seat matching the registered one, or a фирма distinctive enough that a national twin is improbable. Anything else returns a new withholding kind `document_uncorroborated` — never publishing, carrying neither role nor matched_fact, since asserting either would leak the claim the rung just refused to make. It is a distinct kind rather than a fall-through to `unknown` because "matched a person, could not establish the company" and "matched nothing" are different facts, and the seal exists on held links precisely to keep that queue reviewable. It is counted in the run summary: the third corroborator is a bound, not a proof, and that number is what F8 reads to decide whether this tightens to strict ЕИК/seat corroboration. Strict was the alternative; declared seats come only from the ООД/ЕООД table of asset declarations, so its recall cost cannot be known before that measurement. `companyNameDistinctive` defaults to FALSE — this gates the primary publishing rung, so a caller that forgets it must withhold. Rungs 2 and 3 now share one `matchDeclaredSeat`, R10 included, so they cannot drift about what a seat match means. Mutation-checked three ways: never-withhold fails 4 tests, always-withhold fails 7 (the positive controls hold it to a bound), and dropping R10 from the shared helper fails 2. Covered end to end in load.test.mjs, not only as a pure function. ADR-0035 records it and supersedes ADR-0033 decision 2 on this point alone; the methodology page discloses the rule verbatim per ADR-0021 E10. --- apps/web/app/routes/conflict.methodology.tsx | 12 ++- ...-evidence-replaces-name-distinctiveness.md | 7 ++ ...vidence-must-also-establish-the-company.md | 81 ++++++++++++++ docs/adr/README.md | 1 + scripts/cacbg/load-collision.test.mjs | 20 +++- scripts/cacbg/load.mjs | 28 ++++- scripts/cacbg/load.test.mjs | 60 +++++++++-- scripts/tr/evidence.mjs | 96 ++++++++++++----- scripts/tr/evidence.test.mjs | 101 ++++++++++++++++++ 9 files changed, 367 insertions(+), 39 deletions(-) create mode 100644 docs/adr/0035-registry-evidence-must-also-establish-the-company.md diff --git a/apps/web/app/routes/conflict.methodology.tsx b/apps/web/app/routes/conflict.methodology.tsx index 152df8a7..209f11f9 100644 --- a/apps/web/app/routes/conflict.methodology.tsx +++ b/apps/web/app/routes/conflict.methodology.tsx @@ -201,7 +201,17 @@ export default function ConflictMethodology() {

  • Документ. И трите имена на лицето се срещат в живо поле на акта — управители, съдружници, едноличен собственик или физическо лице-търговец. Пазим - ролята (съдружник/собственик или само управител), защото тя не е едно и също. + ролята (съдружник/собственик или само управител), защото тя не е едно и също.{' '} + + Съвпадението по име само по себе си не стига: изисква се и нещо извън фирменото + наименование да установи, че дружеството е декларираното + {' '} + — деклариран ЕИК, декларирано седалище, съвпадащо с вписаното, или достатъчно + отличително фирмено наименование. Иначе връзката остава скрита. Причината е, че + дружеството се разпознава сред изпълнителите по обществени поръчки: ако + лицето притежава едноименно дружество, което никога не е кандидатствало, търсенето + би посочило чуждото — и съименник в неговия акт би „потвърдил" връзка, невярна и в + двете си части.
  • Потвърдено. Лицето не е намерено в акта, но декларирано от него diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md index 0993a41b..82633fb7 100644 --- a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -138,6 +138,13 @@ Rung 2 asserts on a name match. We accept it, on these grounds and no wider: ships a libel. - **The company identity behind it passed preventive control.** The candidate ЕИК comes from the deterministic exact-name resolution of ADR-0008/#226, not from fuzzy matching. + + > **Superseded on this point by [ADR-0035](0035-registry-evidence-must-also-establish-the-company.md).** + > Deterministic is not the same as correct: that resolution ranges over PROCUREMENT WINNERS only, so an + > official whose real company never bid resolves to a same-named winner, and a homonym in the winner's + > deed completes a link false in both halves. Rung 2 now requires a corroborator for the COMPANY — + > declared ЕИК, a matching declared seat, or a distinctive фирма — and the uncorroborated remainder is + > withheld as `document_uncorroborated` and counted. - **It is corroborated, and that is the whole difference.** A bare TR name is spec §4's *weakest* join — no ЕГН, no birthdate, not even ADR-0026's `(name, ведомство)` grain. The registry-graph spike that explored this API concluded, for its own use, that no person-derived edge may reach a user-visible diff --git a/docs/adr/0035-registry-evidence-must-also-establish-the-company.md b/docs/adr/0035-registry-evidence-must-also-establish-the-company.md new file mode 100644 index 00000000..4ba01f70 --- /dev/null +++ b/docs/adr/0035-registry-evidence-must-also-establish-the-company.md @@ -0,0 +1,81 @@ +# ADR-0035 — Регистровото доказателство трябва да установи и ДРУЖЕСТВОТО, не само лицето + +- **Статус:** Прието +- **Дата:** 2026-08-12 +- **Обхват:** доказателственото стъпало 2 („Документ") — `scripts/tr/evidence.mjs`, `scripts/cacbg/load.mjs`. + Заменя единствено твърдението в решение 2 на + [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md), че стъпало 2 не носи допълнителен + гейт; всичко останало в него стои. + +## Контекст + +ADR-0033 записа за стъпало 2: „по-силното стъпало не е гейтнато — регистърът е назовал това лице в ТОВА +дружество, което прави ключа по име без значение". Прегледът на PR-а показа, че изводът важи само ако +„ТОВА дружество" е дружеството, което лицето е декларирало. Не е гарантирано. + +Веригата, по която се стига до акта: + +1. `resolveEntity` (`load.mjs`) свежда декларираното фирмено наименование до **едно ЕИК на ПЕЧЕЛИВШ** — + участник в обществена поръчка. Дружества, които никога не са участвали, не съществуват за резолвера. +2. `nameGloballyUnique` проверява уникалност **само в множеството на участниците**, не в Търговския + регистър. „Единствено" тук значи „единственото сред печелившите". + +Оттук следва сценарият, който прегледът описа и който възпроизведохме: + +- Длъжностното лице действително притежава «АЛФА ООД» с ЕИК **A**, което никога не е участвало в поръчка. +- Друго, несвързано «АЛФА ООД» с ЕИК **B** е печеливш и единственият участник с това наименование. + Резолверът избира **B**. +- Актът на **B** съдържа **съименник** на декларатора — същите три имена, което в България не е рядкост. + Стъпало 2 съвпада и публикува връзка, **невярна и в двете си половини**. + +Регистровото доказателство доказва, че *някой с това име* е съдружник в **B** — не че *това лице* е. +Тоест стъпалото **премести** риска от съвпадение на имена в регистъра, вместо да го затвори, а точно +затварянето му е основанието на #279. Съществуващият карантинен механизъм за двусмислени ключове покрива +само сблъсък печеливш-срещу-печеливш; случаят печеливш-срещу-непечеливш не беше нито обработен, нито +записан като приет остатъчен риск. + +## Решение + +**Преди стъпало 2 да твърди, нещо извън фирменото наименование трябва да каже, че дружеството е +декларираното.** Съвпадението по име остава необходимо, но вече не е достатъчно само по себе си. + +Приема се, ако е изпълнено **поне едно** от: + +| Потвърдител | Защо стига | +|---|---| +| деклариран **ЕИК** | националният идентификатор по ЗТРРЮЛНЦ решава дружеството еднозначно, независимо от споделена фирма (ADR-0028) | +| декларирано **седалище**, съвпадащо с вписаното | близнак в друго населено място отпада; същият факт, на който публикува стъпало 3, включително темпоралната гаранция R10 | +| **отличително** фирмено наименование (`nameDistinctiveness`) | национален близнак е малко вероятен изобщо да съществува | + +Съвпадение по име, което не мине гейта, връща **нов, отделен въздържащ се резултат +`document_uncorroborated`** — никога не публикува, не носи роля и не носи `matched_fact`. + +**Защо отделен вид, а не пропадане към `unknown`.** „Съвпаднахме лице, но не установихме дружеството" и +„не съвпаднахме нищо" са различни факти за една връзка. Печатът се пише за всяка връзка, включително +задържаните, именно за да е прегледаема опашката (migration 0006); сливането им я заслепява точно там, +където решението има нужда от числото. + +**Третият потвърдител е граница, не доказателство.** Затова се **брои** (`document_uncorroborated` в +обобщението на зареждането) — това е числото, което F8 чете, за да реши дали гейтът да се стегне до +първите два. + +**Отхвърлена алтернатива: строго потвърждаване** (само ЕИК или седалище). По-висока точност, но +декларирано седалище се извлича само от таблицата ООД/ЕООД на **имуществена** декларация — никога от +декларация за интереси — така че цената в покритие е неизвестна преди F8 и можеше да свали повърхността +под прага за публикуване. Остава на масата и е точно решението, което измереният остатък ще информира. + +## Последствия + +- Стъпало 2 вече е **гейтнато**, което ADR-0033 решение 2 изрично казваше, че не е. Това е единственото + твърдение оттам, което този ADR заменя; наборът от стъпала, редът им и правилото „първото съвпадение + печели" стоят. +- Гейтът се проверява по подразбиране **затворено**: `companyNameDistinctive` е `false`, ако не бъде + подадено. За разлика от `nameGloballyUnique` — чието разрешаващо подразбиране е ограничено до + най-слабото стъпало — този гейт пази основното публикуващо стъпало. +- Стъпала 2 и 3 споделят **една** реализация на съвпадението по седалище (`matchDeclaredSeat`), заедно с + R10. Две копия на „какво брои за съвпадение по седалище" рано или късно се разминават за това кои връзки + могат да се публикуват. +- Покритието пада с неизмерена величина — по построение точно върху връзките, за които дружеството е + най-слабо установено. Това е приетата посока: по-скоро задържана вярна връзка, отколкото публикувана + невярна, назоваваща действително лице. +- Методологичната страница оповестява правилото дословно (ADR-0021 E10). diff --git a/docs/adr/README.md b/docs/adr/README.md index f0a8193b..0d63413f 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -42,5 +42,6 @@ | [0032](0032-family-ownership-published-under-public-interest.md) | Дял на свързано лице се публикува на поименната повърхност наравно със собствения — на основание надделяващ обществен интерес (ЗДОИ чл. 41и; C-184/20 забранява само поименните данни); близкият никога не се назовава, връзката не се твърди; заменя ADR-0030 | Прието | | [0033](0033-registry-evidence-replaces-name-distinctiveness.md) | Връзка се публикува само срещу проверим факт от Търговския регистър — доказателственият ред заменя нивата по отличителност на името; евристиката вече обосновава твърдение (изрично изключение от ADR-0007), барът за акционерни форми е обединение от три сигнала, монотонността е гейт, не хранилище; заменя ADR-0009/0015/0017 | Прието (дизайн) | | [0034](0034-registry-lookups-and-decisions-share-one-monthly-run.md) | Справките в ТР и решенията текат в едно месечно задание — суровите актове не преживяват изпълнителя, а решението не може да се вземе без тях; заменя твърдението за каденцията в ADR-0033 | Прието | +| [0035](0035-registry-evidence-must-also-establish-the-company.md) | Стъпало 2 („Документ") публикува само ако извън фирменото наименование нещо установи, че дружеството е декларираното — деклариран ЕИК, съвпадащо седалище или отличителна фирма; иначе `document_uncorroborated`, което се брои; заменя твърдението в ADR-0033, че стъпало 2 не е гейтнато | Прието | Свързан проектен документ: [spec/related-persons-foundation.md](../spec/related-persons-foundation.md). diff --git a/scripts/cacbg/load-collision.test.mjs b/scripts/cacbg/load-collision.test.mjs index 8d709ec2..43fb1df4 100644 --- a/scripts/cacbg/load-collision.test.mjs +++ b/scripts/cacbg/load-collision.test.mjs @@ -49,7 +49,8 @@ function buildTrCache(owners) { raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); - for (const [eik, name] of Object.entries(owners)) { + for (const [eik, spec] of Object.entries(owners)) { + const { name, seat } = spec; const deed = { uic: eik, fullName: '"ФИКС" ЕООД', @@ -67,6 +68,17 @@ function buildTrCache(owners) { fieldEntryNumber: '20110502101007', fieldEntryDate: '2011-05-02T00:00:00', }, + // The REGISTERED seat, matching what this official declared. Both фирми here are + // generic („КОМПАНИЯ ЕДНО/ДВЕ" — two content words), so under ADR-0035 a name match + // alone cannot establish which company was declared; the agreeing seat is what does. + // Without it this fixture would exercise the withholding path instead of the + // cross-folder attribution it exists to test. + { + nameCode: 'CR_F_5_L', + htmlData: `

    Държава: БЪЛГАРИЯ
    Населено място: гр. ${seat}

    `, + fieldEntryNumber: '20110502101008', + fieldEntryDate: '2011-05-02T00:00:00', + }, ], }, ], @@ -111,7 +123,7 @@ before(() => { CREATE TABLE contracts(id TEXT PRIMARY KEY, tender_id TEXT, bidder_id TEXT, signed_at TEXT, amount_eur REAL); INSERT INTO authorities VALUES ('auth:1','ВЕДОМСТВО ТЕСТ'); INSERT INTO tenders VALUES ('t1','auth:1'),('t2','auth:1'); - -- Two distinct seat-confirmed single-ЕИК winners → both publish (A_seat), no ambiguity. + -- Two distinct single-ЕИК winners, each named in its own deed AND seat-corroborated → both publish. INSERT INTO bidders VALUES ('eik:100000001','КОМПАНИЯ ЕДНО ЕООД','100000001',1,'София'); INSERT INTO bidders VALUES ('eik:200000002','КОМПАНИЯ ДВЕ ЕООД','200000002',1,'Пловдив'); INSERT INTO contracts VALUES ('c1','t1','eik:100000001','2021-05-01',50000); @@ -160,8 +172,8 @@ before(() => { fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); buildTrCache({ - 100000001: 'ИВАН ПЪРВИ ТЕСТОВ', - 200000002: 'ПЕТЪР ВТОРИ ПРОБЕН', + 100000001: { name: 'ИВАН ПЪРВИ ТЕСТОВ', seat: 'София' }, + 200000002: { name: 'ПЕТЪР ВТОРИ ПРОБЕН', seat: 'Пловдив' }, }); }); diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index aecd680a..31e627c9 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -16,7 +16,12 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; // nameDistinctiveness is deliberately NOT imported: the evidence ladder replaced it in the publish // path (ADR-0033). It survives in classify.mjs for the review queue, not for a publishing decision. -import { temporalStatus, localityToken, closelyHeldForm } from './classify.mjs'; +import { + temporalStatus, + localityToken, + closelyHeldForm, + nameDistinctiveness, +} from './classify.mjs'; import { openCache, readDeed, coverage } from '../tr/cache.mjs'; import { TR_DB, TR_RAW, deedPath } from '../tr/paths.mjs'; import { @@ -769,6 +774,12 @@ for (const rec of agg.values()) { firstDeclaredYear: declYears.length ? Math.min(...declYears) : null, scope: rec.scope, nameGloballyUnique: nameUnique, + // ADR-0035. The resolver picked this winner by фирма; `nameGloballyUnique` above only says no OTHER + // WINNER shares that name, which says nothing about the register at large. So an uncorroborated + // rung 2 additionally asks whether the фирма is one a national twin is unlikely to share. + // `rec.key` is the WINNER's registered name — the canonical spelling of the company we actually + // looked up, which is the identity in question. + companyNameDistinctive: nameDistinctiveness(rec.key) === 'distinctive', }); } catch (err) { // A deed we cannot parse is a deed we cannot reason about — so this link withholds, exactly as an @@ -1039,6 +1050,21 @@ const S = { r.n, ]), ), + // Every evidence rung's count, publishing and withholding alike (ADR-0033 decision 1). `publish_tier` + // IS the verdict kind since #279. + by_evidence_kind: Object.fromEntries( + q('SELECT publish_tier, COUNT(*) n FROM interest_links GROUP BY publish_tier').map((r) => [ + r.publish_tier, + r.n, + ]), + ), + // ADR-0035's residual: links where the register named this person in the resolved company but nothing + // established that the company is the declared one. This is the number F8's hand-labelled sample reads + // to decide whether the distinctiveness gate tightens to strict ЕИК/seat corroboration. Reported + // separately from the map above because it is a DECISION input, not just a tally. + document_uncorroborated: one( + "SELECT COUNT(*) n FROM interest_links WHERE publish_tier='document_uncorroborated'", + ).n, ambiguous_name_keys: ambiguousKeys.length, ambiguous_name_key_examples: ambiguousKeys, noMatch, diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index db60a560..468f11fa 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -179,6 +179,13 @@ before(() => { -- ignored, never guessed — an undatable filing is no evidence of a sale, so this stake stays published. INSERT INTO bidders VALUES ('eik:232323231','ДРЪНКАН ТЕХ 10 ЕООД','232323231',1,'София'); INSERT INTO contracts VALUES ('c18','t1','eik:232323231','2019-05-01',110000); + -- ADR-0035 winner-vs-non-winner homonym: „ХОМОНИМ ТРЕЙД" is a GENERIC фирма (two content words) and the + -- sole WINNER holding it. Хомоним Иванов Тестов declared a stake in a company of that name — but the one + -- he owns never bid, so the resolver lands on this winner instead. This winner's deed happens to name a + -- HOMONYM (identical three tokens), which under a name-only rung 2 „proves" a link false in both halves. + -- Nothing corroborates the company (no declared ЕИК, no declared seat), so it must be withheld. + INSERT INTO bidders VALUES ('eik:242424248','ХОМОНИМ ТРЕЙД ЕООД','242424248',1,'София'); + INSERT INTO contracts VALUES ('c19','t1','eik:242424248','2023-05-01',95000); -- N10 canonicalization: Канонов owns КАНОН ТЕХ 5, filing „МВР" one year and the full ministry name the -- next → ONE identity, ONE link (not a split). Distinctive name (number) → tier B publishable. INSERT INTO bidders VALUES ('eik:131313136','КАНОН ТЕХ 5 ЕООД','131313136',1,'София'); @@ -508,6 +515,24 @@ before(() => { holderRelation: 'self', controlHash: 'H21', }, + // ADR-0035: Хомоним's declared stake in a generic-named company, with NO ЕИК and NO seat declared. + { + folder: '2024', + xmlFile: 'HOM.xml', + year: '2023', + template: 'assets', + category: '', + institution: 'T5', + person: 'Хомоним Иванов Тестов', + position: '', + entity: 'ХОМОНИМ ТРЕЙД ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: '', + holderRelation: 'self', + controlHash: 'H22', + }, // B4 UNKNOWN holder: the holder cell is neither confidently the declarant's own name nor a relative's // (an ambiguous 1-token-different cell). classifyHolder → 'unknown' → this forms NO link (counted // nowhere), so a phantom relative never enters a published family figure. @@ -731,6 +756,9 @@ before(() => { // Only the folder-dated divestment withdraws it — making the pre-fix failure the dangerous one. 212121218: { owners: ['ДРУГ СОБСТВЕНИК СЪВСЕМ'], seat: 'гр. София' }, 232323231: { owners: ['ДРЪНКАН ИВАНОВ ТЕСТОВ'] }, // still the owner → the undatable filing changes nothing + // The homonym: the deed names someone with Хомоним's exact three tokens. Rung 2 matches — and must + // still withhold, because nothing says this is the company he declared. + 242424248: { owners: ['ХОМОНИМ ИВАНОВ ТЕСТОВ'] }, }); }); @@ -927,6 +955,24 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // that asset-declaration silence as a sale and WITHDRAWS the stake; the per-type horizon must not — no later // INTERESTS filing omits the company. This link must stay PUBLISHED. (Guards against dropping a true link: // 13% of holders declare a stake only in the interests declaration.) + // ADR-0035 — the CRITICAL, end to end. Хомоним declared a stake in „ХОМОНИМ ТРЕЙД ЕООД"; the company he + // actually owns never bid, so `resolveEntity` resolved his declaration to the same-named WINNER, whose + // deed names a person with his exact three tokens. Rung 2 matches. It must NOT publish: the register + // proves someone of that name owns THIS company, not that this is the company he declared. `nameGlobally- + // Unique` cannot catch it — it ranges over bidders, and this winner is the only bidder with the name. + const homonym = link('242424248', 'Хомоним Иванов Тестов'); + assert.equal(homonym.publish_tier, 'document_uncorroborated'); + assert.notEqual(homonym.status, 'published'); + // The seal must record WHY it was withheld, and must not carry the role the rung refused to assert. + const homonymSeal = db + .prepare( + 'SELECT evidence_kind, registry_role, matched_fact FROM interest_link_evidence WHERE link_key=?', + ) + .get(homonym.link_key); + assert.equal(homonymSeal.evidence_kind, 'document_uncorroborated'); + assert.equal(homonymSeal.registry_role, null); + assert.equal(homonymSeal.matched_fact, null); + // §1.3 unparseable filing YEAR: Безгодин's later declaration has an unreadable ('н/д') but a 2023 // FOLDER. Dropping that record — the pre-fix behaviour — leaves his horizon at 2019, so `divested` stays // false and a stake he no longer holds keeps naming him on the public surface. The folder must date it. @@ -1000,17 +1046,17 @@ test('re-run is idempotent and honors the suppression list (contested link stays 'suppressed', ); // idempotent: still exactly the same number of links + persons after a clean rebuild. - // 19 links: 14 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero + // 20 links: 15 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero // 'withdrawn' + Безгодин's folder-dated 'withdrawn' + Дрънкан's undatable-filing 'published' + - // Интер's per-type-kept published link) + 2 family (Кмет's, now held for want of registry evidence, + // Хомоним's ADR-0035 'document_uncorroborated' hold + Интер's per-type-kept published link) + 2 family (Кмет's, now held for want of registry evidence, // and Кметица's seat-confirmed one) + Канонов's canonicalized single link + // Алфа & Бета (two officials on one winner, ПАРТНЬОРИ 5); Мария (quarantined), Акционер (securities), // Двусмислен (unknown holder) & Безинст (empty institution) none. - assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 19); - // 22 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, - // zero-contract Нула, the two ПАРТНЬОРИ co-owners and the two §1.3 filing-date cases; Канонов's two - // institution-variant filings fold to ONE. - assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 22); + assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 20); + // 23 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, + // zero-contract Нула, the two ПАРТНЬОРИ co-owners, the two §1.3 filing-date cases and Хомоним; + // Канонов's two institution-variant filings fold to ONE. + assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 23); db.close(); }); diff --git a/scripts/tr/evidence.mjs b/scripts/tr/evidence.mjs index 51c4c770..56bc01ef 100644 --- a/scripts/tr/evidence.mjs +++ b/scripts/tr/evidence.mjs @@ -89,6 +89,32 @@ function findPerson(deed, name, nameCodes) { return null; } +/** + * The registered seat, when it matches one THIS person declared for THIS company and was in force for the + * declared period. Shared by rung 3 (which publishes on it) and rung 2's company-identity corroborator. + * + * R10: seats move. A company that relocated INTO the declared settlement afterwards would confirm falsely, + * so the registered seat must predate the period. + * + * A null `firstDeclaredYear` FAILS the guard rather than skipping it. `load.mjs` passes null whenever no + * history row carried a parseable year, and an unknown year is not a satisfied temporal test — it is the + * absence of one. Reading it as „covers the period" made the weakest rung the only one with no temporal + * check, on exactly the links where we know least, and rung 4 already refuses to run without a year on the + * same ground. The undated-SEAT leg is different and stays: a seat with no entry date is the ordinary shape + * for a company that never moved, and it is checkable — a known year is still on the other side. + * + * @returns {{settlement:string, entryDate:string|null}|null} + */ +function matchDeclaredSeat(deed, declaredSeats, firstDeclaredYear) { + const seat = registrySeat(deed); + // Empty NEVER matches — otherwise every link with no seat data on either side rubber-stamps itself. + if (seat.settlement === '') return null; + if (firstDeclaredYear == null) return null; + if (seat.entryDate != null && seat.entryDate > `${firstDeclaredYear}-12-31`) return null; + const declared = declaredSeats.map(normalizeSettlement).filter((s) => s !== ''); + return declared.includes(seat.settlement) ? seat : null; +} + /** * Decide the evidence for one link. * @@ -104,6 +130,9 @@ function findPerson(deed, name, nameCodes) { * @param {number|null} [input.firstDeclaredYear] * @param {'self'|'family'} [input.scope] * @param {boolean} [input.nameGloballyUnique] AND-gate on the WEAKEST rung only + * @param {boolean} [input.companyNameDistinctive] the declared фирма is unlikely to have a national + * twin. Gates an UNCORROBORATED rung 2 (ADR-0035). + * Defaults to FALSE: a caller that forgets it withholds. * @returns {{kind:string, publishable:boolean, registryRole:string|null, matchedFact:string|null, * entryNumber:string|null, entryDate:string|null, rulesVersion:string, * shortName:boolean, latinInName:boolean}} @@ -118,6 +147,11 @@ export function evidenceVerdict(input) { firstDeclaredYear = null, scope = 'self', nameGloballyUnique = true, + // Fail-CLOSED, unlike `nameGloballyUnique` above. That one's permissive default is bounded — it gates + // only the weakest rung. This one gates the PRIMARY publishing rung, so a caller that forgets to pass + // it must withhold rather than publish a claim naming a real person against a company we did not + // establish. There is exactly one production caller (load.mjs) and it passes it explicitly. + companyNameDistinctive = false, } = input; const tokens = personTokens(declarantName); @@ -152,12 +186,43 @@ export function evidenceVerdict(input) { if (form.verdict === 'joint_stock') return verdict('bar_joint_stock', false); if (form.verdict === 'unknown') return verdict('unknown', false); + // The registered seat, matched against what THIS person declared for THIS company, with R10's temporal + // guard applied. Computed once and consumed by two rungs: rung 3 publishes „Потвърдено" on it, and rung 2 + // uses it as a COMPANY-IDENTITY corroborator. One implementation, because two copies of "what counts as a + // seat match" would eventually disagree about which links may be published. + const matchedSeat = matchDeclaredSeat(deed, declaredSeats, firstDeclaredYear); + // ── rung 2 ────────────────────────────────────────────────────────────────── // Only a full three-token name may assert. A Latin homoglyph makes the name a non-match rather than // a false match — company-name-key.ts's posture, applied to people. + // + // The company gate (ADR-0035). A name match proves someone with these three tokens is registered in the + // company we LOOKED UP — never that this is the company the official declared. `resolveEntity` picks the + // sole WINNER holding the declared name and `nameGloballyUnique` ranges over bidders only, so an official + // whose real company never bid resolves to a same-named winner, and a homonym in that winner's deed + // completes a link false in both halves. Before rung 2 may assert, something other than the фирма must + // say the company is the declared one: + // • the declarant wrote the ЕИК — the national identifier resolves it outright (ADR-0028); or + // • the declared seat matches the registered one — a twin in another town is excluded; or + // • the фирма is distinctive enough that a national twin is improbable in the first place. + // The third is a bound, not a proof, and it is COUNTED (`documentUncorroborated`) so F8 can decide from + // the measured residual whether to tighten to the first two. Strict corroboration was the alternative; + // declared seats exist only on the ООД/ЕООД table of asset declarations, so its recall cost cannot be + // known before that measurement. + const companyCorroborated = declaredEik || matchedSeat != null; const eligibleForDocument = !telemetry.shortName && !telemetry.latinInName; if (eligibleForDocument) { const owner = findPerson(deed, declarantName, OWNERSHIP_FIELDS); + const manager = owner ? null : findPerson(deed, declarantName, [MANAGER_FIELD]); + const hit = owner ?? manager; + if (hit && !companyCorroborated && !companyNameDistinctive) { + // A DISTINCT withholding kind, not a fall-through to `unknown`. „We matched a person but could not + // establish the company" and „we matched nothing" are different facts about a link, and the review + // queue (which is sealed for held links precisely to be reviewable) has to be able to tell them + // apart. It never publishes, and it carries no role or fact — asserting either would leak the very + // claim the rung just refused to make. + return verdict('document_uncorroborated', false); + } if (owner) { return verdict('document', true, { registryRole: 'owner', @@ -166,7 +231,6 @@ export function evidenceVerdict(input) { entryDate: owner.entryDate, }); } - const manager = findPerson(deed, declarantName, [MANAGER_FIELD]); if (manager) { return verdict('document', true, { registryRole: 'manager', @@ -190,31 +254,11 @@ export function evidenceVerdict(input) { // The SEAT leg is name-gated, and only this one. ADR-0017's holding carried forward: a name shared by // two ЕИК cannot support a name-derived identity claim. The seat still rescues a GENERIC name — that // is the whole point of the rung — it just cannot rescue a NATIONALLY SHARED one. - if (nameGloballyUnique) { - const seat = registrySeat(deed); - // Empty NEVER confirms — otherwise every link with no seat data on either side rubber-stamps. - if (seat.settlement !== '') { - const declared = declaredSeats.map(normalizeSettlement).filter((s) => s !== ''); - // R10: the registered seat must have been in force for the declared period. Seats move, and a - // company that relocated INTO the declared settlement afterwards would otherwise confirm falsely. - // - // A null `firstDeclaredYear` FAILS the guard rather than skipping it. `load.mjs` passes null - // whenever no history row carried a parseable year, and an unknown year is not a satisfied - // temporal test — it is the absence of one. Reading it as „covers the period" made the weakest - // rung the only one with no temporal check, on exactly the links where we know least, and rung 4 - // below already refuses to run without a year (`firstDeclaredYear != null`) on the same ground. - // The undated-seat leg is different and stays: a seat with no entry date is the ordinary shape - // for a company that never moved, and it is checkable — a known year is still on the other side. - const seatCoversPeriod = - firstDeclaredYear != null && - (seat.entryDate == null || seat.entryDate <= `${firstDeclaredYear}-12-31`); - if (seatCoversPeriod && declared.includes(seat.settlement)) { - return verdict('confirmed', true, { - matchedFact: `seat:${seat.settlement}`, - entryDate: seat.entryDate, - }); - } - } + if (nameGloballyUnique && matchedSeat != null) { + return verdict('confirmed', true, { + matchedFact: `seat:${matchedSeat.settlement}`, + entryDate: matchedSeat.entryDate, + }); } // ── rung 4 ────────────────────────────────────────────────────────────────── diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs index 4de42a93..3010698c 100644 --- a/scripts/tr/evidence.test.mjs +++ b/scripts/tr/evidence.test.mjs @@ -41,6 +41,9 @@ const base = { firstDeclaredYear: 2021, scope: 'self', nameGloballyUnique: true, + // The company was resolved by a name unlikely to have a national twin. The rung-2 tests below are about + // NAME matching inside a deed, so they hold this dimension fixed; the gate itself is tested separately. + companyNameDistinctive: true, }; test('RULES_VERSION is a stable, non-empty identifier — §8 hangs off it', () => { @@ -270,6 +273,104 @@ test('rung 3 — name uniqueness does NOT gate the stronger „Документ" ); }); +// ── rung 2's company gate: the winner-vs-non-winner homonym (ADR-0035) ──────── +// +// The ladder proves a person with these three tokens is registered in the company we LOOKED UP. It cannot +// prove that company is the one the official declared. `resolveEntity` maps a declared name to the sole +// WINNER holding it, and `nameGloballyUnique` ranges over procurement bidders only — never the whole +// register. So when an official owns a same-named company that never bid, we resolve to the winner, and a +// three-token homonym in the winner's deed „proves" a link that is false in both halves. +// +// Two coincidences, and neither is rare in Bulgaria: a shared фирма and a shared three-part name. The gate +// asks for a reason to believe the COMPANY is the declared one before rung 2 may assert. +test('rung 2 — a GENERIC company name with no corroboration cannot publish on a name match alone', () => { + const v = evidenceVerdict({ ...base, companyNameDistinctive: false }); + assert.equal( + v.kind, + 'document_uncorroborated', + 'the deed names SOMEONE with this name in THIS company — not that this is the declared company', + ); + assert.equal(v.publishable, false); + // It must not fall through to `unknown`: the residual is the input to F8's decision on whether this gate + // tightens, and a rung-2 match withheld for want of company identity is a different fact from no match. + assert.equal(v.registryRole, null); + assert.equal(v.matchedFact, null); +}); + +test('rung 2 — a declared ЕИК corroborates the company, so the name match publishes', () => { + // POSITIVE CONTROL. The ЕИК IS the identity (ЗТРРЮЛНЦ, ADR-0028) — it resolves the company behind any + // shared фирма, which is exactly the collision the gate is about. + const v = evidenceVerdict({ ...base, companyNameDistinctive: false, declaredEik: true }); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); + assert.equal(v.registryRole, 'owner'); +}); + +test('rung 2 — a declared seat matching the registered seat corroborates the company', () => { + // POSITIVE CONTROL. The declarant put this company in гр. Пловдив and the register agrees; a national + // twin in another town is excluded by the same fact rung 3 publishes on. + const v = evidenceVerdict({ + ...base, + companyNameDistinctive: false, + declaredSeats: ['гр. Пловдив'], + }); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); +}); + +test('rung 2 — a DISTINCTIVE company name publishes uncorroborated (the gate is not a blanket)', () => { + // POSITIVE CONTROL, and the one that distinguishes this fix from disabling rung 2. A predicate that + // always withheld would satisfy the bar above; this is what it must NOT do. + const v = evidenceVerdict({ ...base, companyNameDistinctive: true }); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); +}); + +test('rung 2 — a seat that does NOT match cannot corroborate a generic name', () => { + // The corroborator has to actually corroborate. A declared seat in another town is evidence AGAINST the + // company being the declared one, so it certainly cannot rescue the rung. + const v = evidenceVerdict({ + ...base, + companyNameDistinctive: false, + declaredSeats: ['гр. Бургас'], // the deed says Пловдив + }); + assert.equal(v.kind, 'document_uncorroborated'); + assert.equal(v.publishable, false); +}); + +test('rung 2 — the seat corroborator carries the SAME temporal guard as rung 3 (R10)', () => { + // A seat registered after the declared period cannot corroborate anything: the company may have moved + // INTO that town afterwards. Rung 3 already refuses it; rungs 2 and 3 share one implementation so they + // cannot drift into disagreeing about what a seat match means. + const moved = deed([ + fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ')), + fld( + 'CR_F_5_L', + container('Държава: БЪЛГАРИЯ
    Населено място: гр. Пловдив, п.к. 4000'), + '2023-07-01T00:00:00', + ), + ]); + const v = evidenceVerdict({ + ...base, + deed: moved, + companyNameDistinctive: false, + declaredSeats: ['гр. Пловдив'], + firstDeclaredYear: 2021, + }); + assert.equal(v.kind, 'document_uncorroborated'); +}); + +test('rung 2 — the company gate never rescues a link rung 1 has barred', () => { + // Ordering: a joint-stock bar outranks everything, corroborated or not. The gate adds a way to WITHHOLD, + // never a way to publish something a stronger rung refused. + const ad = deed([fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))], { + legalForm: 5, + fullName: '"ГАМА ИНВЕСТ" АД', + }); + const v = evidenceVerdict({ ...base, deed: ad, companyNameDistinctive: true, declaredEik: true }); + assert.equal(v.kind, 'bar_joint_stock'); +}); + // ── rung 4: „Оборена" ───────────────────────────────────────────────────────── test('rung 4 — absent from a deed whose ownership predates the declaration refutes the link', () => { const older = deed([ From e9c17b980fb95052b07b3000dc61ce94239b6b92 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:42:28 +0300 Subject: [PATCH 49/69] feat(db): constrain the publishing-gate enums and fix the declaration natural key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #279 §2 asked for CHECK constraints and a non-null control_hash. The enums are done as asked; the control_hash half is not, and the reason matters. Enums. `status` and `interest_class` (0003) and `evidence_kind`/`registry_role`/ `live_status` (0006) were plain TEXT while the surface reads them as enums. A value that only LOOKS like a gate value — 'published ' with a trailing space, the case the ticket names — passes every writer and then fails `status = 'published'` silently. A CHECK binds every writer at once, including a hand-run UPDATE during an incident, which is exactly when it gets typed. 0006 is unshipped in this PR, so its constraints go on directly. control_hash NOT NULL is REJECTED, and the underlying defect fixed differently. The field is genuinely optional at the source — the register omits and parse.mjs carries that through as null — so NOT NULL converts a missing optional field into a run-stopping loader failure, and a fabricated placeholder would assert an integrity check nobody performed. The real defect is that `UNIQUE (xml_file, control_hash)` never constrained re-import: SQLite counts NULLs as DISTINCT, so every hashless declaration was mutually unique and re-imported as a fresh row each run, double-counting its stakes. Replaced by a unique index over (xml_file, folder_year, COALESCE(control_hash,'')) — the same natural key `id` already encodes, since the register reuses basenames across folders. 0007 retrofits both onto a deployed D1. `CREATE TABLE IF NOT EXISTS` never revisits an existing table and ship-related-persons wipes ROWS, not definitions, so a live database would otherwise keep the unconstrained 0003 shape forever however often the data reloads. interest_links needs a rebuild (SQLite cannot add a CHECK in place) under defer_foreign_keys, since interest_link_evidence references it and D1 enforces FKs; declarations needs only the corrected index. Both converge on re-application, which the bare `d1 execute --file` path requires, and both workflows apply it. Tests cover the trailing-space case by name, the ADR-0035 kind, the hashless duplicate that used to slip through, and a pre-0007 database that keeps its rows through the retrofit and survives a second application. Positive controls throughout: every real status and class still inserts, and the natural key still admits the same basename in a different folder and a corrected re-filing under a new hash. --- .github/workflows/deploy.yml | 8 + .github/workflows/related-persons-data.yml | 8 + .../0003_related_persons_foundation.sql | 33 ++- .../0006_interest_link_evidence.sql | 17 +- .../0007_publishing_gate_constraints.sql | 103 ++++++++++ packages/db/src/migrations.test.ts | 188 ++++++++++++++++++ 6 files changed, 349 insertions(+), 8 deletions(-) create mode 100644 packages/db/migrations/0007_publishing_gate_constraints.sql diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f52927c9..e05a6fd3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -151,6 +151,14 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0006_interest_link_evidence.sql + # 0007 retrofits the publishing-gate CHECK constraints and control_hash NOT NULL onto an + # ALREADY-PROVISIONED D1 (#279 §2). 0003 declares them now, but `CREATE TABLE IF NOT EXISTS` + # never revisits an existing table and the ship step wipes ROWS, not definitions — so without + # this a live database keeps the unconstrained shape no matter how often the data reloads. + # Rebuild-based and therefore idempotent: re-applying copies an already-constrained table. + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0007_publishing_gate_constraints.sql # The base schema was created out-of-band with `d1 execute --file`, so wrangler's migration # ledger is empty and `d1 migrations apply` would collide on 0000. Probe the actual table # instead. SQLite has no `ADD COLUMN IF NOT EXISTS`; a completion-marker table is created only diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index e352a787..8472b96f 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -309,6 +309,14 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0006_interest_link_evidence.sql + # 0007 retrofits the publishing-gate CHECK constraints and control_hash NOT NULL onto an + # ALREADY-PROVISIONED D1 (#279 §2). 0003 declares them now, but `CREATE TABLE IF NOT EXISTS` + # never revisits an existing table and the ship step wipes ROWS, not definitions — so without + # this a live database keeps the unconstrained shape no matter how often the data reloads. + # Rebuild-based and therefore idempotent: re-applying copies an already-constrained table. + pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0007_publishing_gate_constraints.sql - name: Ship свързани-лица tables → D1 # ship-related-persons.mjs shells out to `wrangler d1 execute` with cwd=apps/web. Two things it diff --git a/packages/db/migrations/0003_related_persons_foundation.sql b/packages/db/migrations/0003_related_persons_foundation.sql index 6ad3be97..58a85878 100644 --- a/packages/db/migrations/0003_related_persons_foundation.sql +++ b/packages/db/migrations/0003_related_persons_foundation.sql @@ -29,9 +29,24 @@ CREATE TABLE IF NOT EXISTS declarations ( category TEXT, -- CACBG category (e.g. Народни представители) institution TEXT, -- the body the official serves in position TEXT, -- declared position - source_url TEXT NOT NULL, -- register.cacbg.bg// — provenance - UNIQUE (xml_file, control_hash) + source_url TEXT NOT NULL -- register.cacbg.bg// — provenance ); +-- Re-import idempotence (#279 §2). This REPLACES a table-level `UNIQUE (xml_file, control_hash)`, which +-- did not constrain anything it was meant to: +-- 1. SQLite counts NULLs as DISTINCT, and `control_hash` is genuinely optional at the source — the +-- register omits on some declarations and parse.mjs carries that through as null. So +-- every hashless declaration was mutually unique and re-imported as a fresh row each run, +-- double-counting the stakes it carries. COALESCE folds them to one value. +-- 2. `xml_file` is not unique ACROSS FOLDERS — the register splits years into suffixed folders and +-- reuses basenames — which load.mjs already knows, since it namespaces the declaration id by folder +-- for exactly this reason. Without folder_year here the two keys disagree, and the narrower one +-- would reject a second official's genuinely different declaration. +-- +-- `control_hash` deliberately stays NULLABLE. NOT NULL was the other candidate, but it converts an +-- optional source field into a run-stopping loader failure, and fabricating a hash to satisfy it would +-- assert an integrity check we never performed. +CREATE UNIQUE INDEX IF NOT EXISTS idx_declarations_natural_key + ON declarations(xml_file, folder_year, COALESCE(control_hash, '')); -- Per-person lookup: audit.mjs scans a person's declarations, and LINK_SELECT's source_url subquery -- filters declarations by person_id — without this it's a full scan of the table. CREATE INDEX IF NOT EXISTS idx_declarations_person ON declarations(person_id); @@ -75,7 +90,12 @@ CREATE TABLE IF NOT EXISTS interest_links ( -- management_role — self, relation manages, a single declarant (ambiguous: private manager or small board) -- Materiality: only CLOSELY-HELD forms (ООД/ЕООД/ЕТ/…) reach ownership classes; listed АД/ЕАД securities -- and management-only roles never become private_ownership/family_ownership (ADR-0022). - interest_class TEXT NOT NULL DEFAULT 'management_role', -- fail-closed: a NON-surfaced class, so a writer that sets status but forgets the class can't leak to the public surface (load.mjs always sets it explicitly) + -- fail-closed: a NON-surfaced class, so a writer that sets status but forgets the class can't leak to + -- the public surface (load.mjs always sets it explicitly). CHECKed for the same reason as `status`: + -- the surface reads this as an enum, so a typo'd class is a gate failure, not a cosmetic one. + interest_class TEXT NOT NULL DEFAULT 'management_role' + CHECK (interest_class IN ('private_ownership','family_ownership', + 'ex_officio_board','management_role')), contemporaneous INTEGER NOT NULL DEFAULT 0, own_institution TEXT NOT NULL DEFAULT 'none', -- exact (deterministic) | locality (heuristic) | none evidence_count INTEGER NOT NULL DEFAULT 1, -- # declared_interests supporting this link @@ -87,7 +107,12 @@ CREATE TABLE IF NOT EXISTS interest_links ( contract_value_eur REAL, -- SUM(contracts.amount_eur); NULL if none summable first_contract_year TEXT, last_contract_year TEXT, - status TEXT NOT NULL DEFAULT 'held', -- published (public surface: private/family ownership) | internal (non-surfaced class) | held | withdrawn | suppressed + -- CHECKed, not merely documented (#279 §2). The public surface is `status = 'published'`, so a value + -- that only LOOKS like one — 'published ' with a trailing space is the case the ticket names — passes + -- every writer and then fails the gate silently, or slips a LIKE. A CHECK binds every writer at once, + -- including a hand-run UPDATE during an incident, which is exactly when this is most likely typed. + status TEXT NOT NULL DEFAULT 'held' + CHECK (status IN ('published','internal','held','withdrawn','suppressed')), verified_by TEXT, verified_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')) diff --git a/packages/db/migrations/0006_interest_link_evidence.sql b/packages/db/migrations/0006_interest_link_evidence.sql index a7b83033..e91869d1 100644 --- a/packages/db/migrations/0006_interest_link_evidence.sql +++ b/packages/db/migrations/0006_interest_link_evidence.sql @@ -20,9 +20,16 @@ CREATE TABLE IF NOT EXISTS interest_link_evidence ( link_key TEXT PRIMARY KEY REFERENCES interest_links(link_key), - -- document | confirmed | refuted | bar_joint_stock | unknown | outside_tr (ADR-0033 decision 1) - evidence_kind TEXT NOT NULL, - registry_role TEXT, -- owner | manager | NULL — only meaningful for evidence_kind='document' + -- ADR-0033 decision 1, plus document_uncorroborated (ADR-0035: a name match whose COMPANY nothing + -- corroborated). CHECKed because the read gate filters on this column — SURFACED_OWNERSHIP admits + -- exactly 'document' and 'confirmed' — so an unlisted value is either a link that silently stops + -- surfacing or, if it collides with a publishing name, one that surfaces unproven. + evidence_kind TEXT NOT NULL + CHECK (evidence_kind IN ('document','confirmed','document_uncorroborated','refuted', + 'bar_joint_stock','unknown','outside_tr')), + -- owner | manager | NULL — only meaningful for evidence_kind='document'. The card renders this as + -- „вписан като …", so a stray value becomes a public claim about a named person's registry role. + registry_role TEXT CHECK (registry_role IS NULL OR registry_role IN ('owner','manager')), matched_fact TEXT, -- the closed vocabulary above. NEVER a name. -- TEXT, not INTEGER: a fieldEntryNumber like 20130716101007 is already 14 digits and exceeds the -- exact-integer range once it round-trips through JSON/JS. @@ -33,7 +40,9 @@ CREATE TABLE IF NOT EXISTS interest_link_evidence ( -- live | terminated_owner_still | terminated_manager_still | terminated. -- RE-DERIVED on every run and deliberately NOT part of the seal's permanence: it asserts a present -- tense about a named person, and its freshness is bounded by the cache refresh cycle (ADR-0033 R3). - live_status TEXT NOT NULL, + live_status TEXT NOT NULL + CHECK (live_status IN ('live','terminated_owner_still','terminated_manager_still', + 'terminated')), sealed_at TEXT NOT NULL DEFAULT (datetime('now')) ); diff --git a/packages/db/migrations/0007_publishing_gate_constraints.sql b/packages/db/migrations/0007_publishing_gate_constraints.sql new file mode 100644 index 00000000..ac682308 --- /dev/null +++ b/packages/db/migrations/0007_publishing_gate_constraints.sql @@ -0,0 +1,103 @@ +-- Retrofit the publishing-gate constraints onto an ALREADY-DEPLOYED database (#279 §2). +-- +-- WHY THIS EXISTS AT ALL, given 0003 and 0006 now declare the same constraints: +-- `CREATE TABLE IF NOT EXISTS` is a no-op against a table that already exists, and +-- `scripts/ship-related-persons.mjs` wipes ROWS (`DELETE FROM`, WIPE_ORDER) and never table +-- definitions. So every environment that applied 0003 before today keeps the unconstrained shape +-- forever, no matter how many times the data is reloaded. Only a rebuild can change it. +-- +-- WHY IT IS SAFE TO RE-RUN: migrations here are applied by a bare `wrangler d1 execute --file` with no +-- applied-migrations tracking (see related-persons-data.yml / deploy.yml), so re-application MUST be a +-- no-op. Both blocks below are written to converge: the index block is `IF NOT EXISTS` over a de-duplicated +-- table, and the rebuild block copies the CURRENT table into a constrained one — run a second time it +-- copies an already-constrained table into an identical one. Idempotent by construction, no version flag. +-- +-- WHY A REBUILD FOR interest_links AND NOT `ALTER TABLE`: SQLite cannot add a CHECK in place, so +-- create-copy-drop-rename is the documented route. `declarations` needs no rebuild — its defect was a +-- constraint that failed to constrain, which a correct index fixes without touching the table. +-- +-- FOREIGN KEYS: `interest_link_evidence` references interest_links(link_key), and D1 enforces foreign +-- keys, so dropping the parent mid-rebuild would be refused. `defer_foreign_keys` postpones the check to +-- the end of the transaction — the documented way to rebuild a referenced table — by which point the +-- renamed table satisfies it. D1 runs each file in one implicit transaction, so a failure rolls the whole +-- file back rather than leaving a half-swapped table. +PRAGMA defer_foreign_keys = true; + +-- ── declarations ──────────────────────────────────────────────────────────────────────────────────── +-- The table-level `UNIQUE (xml_file, control_hash)` did not constrain what it was written for. SQLite +-- counts NULLs as DISTINCT and `control_hash` is genuinely optional at the source (the register omits +-- on some declarations), so every hashless declaration was mutually unique and re-imported +-- as a NEW row on each run, double-counting the stakes it carries. `xml_file` is also not unique across +-- folders — the register reuses basenames per year — which load.mjs already handles by namespacing the +-- declaration id with the folder. +-- +-- Replaced by an expression index over COALESCE(control_hash, '') plus folder_year: NULL-proof, and the +-- same natural key `id` already encodes. The column stays NULLABLE on purpose — NOT NULL would convert an +-- optional source field into a run-stopping loader failure, and a fabricated placeholder hash would +-- assert an integrity check nobody performed. +-- +-- No table rebuild is needed for this one: dropping the old constraint means dropping the table it lives +-- on, but the same effect is had by creating the correct index — and the stale UNIQUE, being strictly +-- weaker, rejects nothing the new index accepts. Duplicates ALREADY stored under the old non-constraint +-- would block the new index, so they are collapsed first, keeping the earliest row per natural key. +DELETE FROM declarations WHERE id NOT IN ( + SELECT MIN(id) FROM declarations GROUP BY xml_file, folder_year, COALESCE(control_hash, '') +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_declarations_natural_key + ON declarations(xml_file, folder_year, COALESCE(control_hash, '')); + +-- ── interest_links ────────────────────────────────────────────────────────────────────────────────── +-- `status` and `interest_class` ARE the publishing gate. The public surface is `status = 'published'` +-- AND a surfaced `interest_class`, so a value that merely resembles one — 'published ' with a trailing +-- space, the case #279 §2 names — passes every writer and then fails the gate silently. A CHECK binds +-- every writer at once, including a hand-run UPDATE during an incident, which is when it is most likely +-- to be typed. +-- +-- Rows violating the new CHECKs are dropped by INSERT OR IGNORE rather than rewritten: a status we cannot +-- interpret is not a link we should publish or guess at, and the loader rebuilds the table on every run. +CREATE TABLE IF NOT EXISTS interest_links_0007 ( + id TEXT PRIMARY KEY, + link_key TEXT NOT NULL UNIQUE, + person_id TEXT NOT NULL REFERENCES persons(id), + bidder_id TEXT NOT NULL, + eik TEXT NOT NULL, + entity_key TEXT NOT NULL, + match_method TEXT, + matcher_version TEXT NOT NULL, + publish_tier TEXT NOT NULL, + relation TEXT NOT NULL, + interest_class TEXT NOT NULL DEFAULT 'management_role' + CHECK (interest_class IN ('private_ownership','family_ownership', + 'ex_officio_board','management_role')), + contemporaneous INTEGER NOT NULL DEFAULT 0, + own_institution TEXT NOT NULL DEFAULT 'none', + evidence_count INTEGER NOT NULL DEFAULT 1, + first_declared_year TEXT, + last_declared_year TEXT, + contract_count INTEGER NOT NULL DEFAULT 0, + contract_value_eur REAL, + first_contract_year TEXT, + last_contract_year TEXT, + status TEXT NOT NULL DEFAULT 'held' + CHECK (status IN ('published','internal','held','withdrawn','suppressed')), + verified_by TEXT, + verified_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); +INSERT OR IGNORE INTO interest_links_0007 + (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, + relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, + last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, + status, verified_by, verified_at, created_at) +SELECT id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, + relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, + last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, + status, verified_by, verified_at, created_at +FROM interest_links +WHERE status IN ('published','internal','held','withdrawn','suppressed') + AND interest_class IN ('private_ownership','family_ownership','ex_officio_board','management_role'); +DROP TABLE interest_links; +ALTER TABLE interest_links_0007 RENAME TO interest_links; +CREATE INDEX IF NOT EXISTS idx_interest_links_eik ON interest_links(eik); +CREATE INDEX IF NOT EXISTS idx_interest_links_person ON interest_links(person_id); +CREATE INDEX IF NOT EXISTS idx_interest_links_status ON interest_links(status); diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index fed25dda..1997bbd2 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -12,6 +12,7 @@ const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_ const migration2 = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration7 = resolve(root, 'packages/db/migrations/0007_publishing_gate_constraints.sql'); const backfill = resolve(root, 'scripts/backfill-current-value-currency.sql'); const precompute = resolve(root, 'scripts/precompute.sql'); @@ -205,4 +206,191 @@ describe('served migrations', () => { rmSync(dir, { recursive: true, force: true }); } }); + + // #279 §2: the publishing gate reads these columns as enums, but the schema let them hold anything. + // A value that merely LOOKS like a gate value — the trailing space in 'published ' §2 names — passes + // every writer and then silently fails `status = 'published'` (or worse, passes a LIKE somewhere). + // A CHECK is the only place this can be enforced for every writer at once, including a hand-run UPDATE. + it('constrains the publishing-gate enums and the declaration identity key', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + for (const m of [migration0, migration3, migration6, migration7]) readScript(dbPath, m); + + const seedLink = (status: string) => + sqlite( + dbPath, + `INSERT INTO persons(id, name) VALUES ('person:a', 'А') ON CONFLICT DO NOTHING; + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:${status}', '${status}', 'person:a', 'eik:201122335', '201122335', + 'АЛФА СТРОЙ ООД', 'test', 'document', 'owns', 'private_ownership', '${status}');`, + ); + + // §2's named case: 'published ' is not 'published'. It must be rejected, not stored. + expect(() => seedLink('published ')).toThrow(/CHECK/i); + expect(() => seedLink('publushed')).toThrow(/CHECK/i); + // POSITIVE CONTROL — every real status still inserts. A CHECK that rejected everything would pass + // the assertions above while breaking the loader. + for (const s of ['published', 'held', 'withdrawn', 'suppressed', 'internal']) + expect(() => seedLink(s)).not.toThrow(); + + // interest_class gates the surface just as hard: a non-surfaced class is what keeps a management + // role off the public page, so a typo'd class is a leak in the same way. + expect(() => + sqlite( + dbPath, + `INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:c', 'c', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership ', 'held');`, + ), + ).toThrow(/CHECK/i); + + // evidence_kind decides whether a link may be read at all (SURFACED_OWNERSHIP). ADR-0035's + // document_uncorroborated must be a legal value — and a misspelling must not be. + const seal = (kind: string) => + sqlite( + dbPath, + `INSERT INTO interest_link_evidence + (link_key, evidence_kind, lookup_date, rules_version, live_status) + VALUES ('${kind}', '${kind}', '2026-08-12', 'tr-rules-1', 'live');`, + ); + expect(() => seal('document_uncorroberated')).toThrow(/CHECK|FOREIGN KEY/i); + for (const k of ['document', 'confirmed', 'document_uncorroborated', 'refuted']) + expect(() => { + sqlite( + dbPath, + `INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:${k}', '${k}', 'person:a', 'eik:1', '1', 'X', 't', '${k}', 'owns', + 'private_ownership', 'held'); + INSERT INTO interest_link_evidence + (link_key, evidence_kind, lookup_date, rules_version, live_status) + VALUES ('${k}', '${k}', '2026-08-12', 'tr-rules-1', 'live');`, + ); + }).not.toThrow(); + + // §2: `UNIQUE (xml_file, control_hash)` did not constrain re-import. SQLite counts NULLs as + // DISTINCT and control_hash is genuinely optional at the source, so two hashless imports of the + // SAME declaration both inserted and double-counted the stakes it carries. + const decl = (id: string, hash: string | null, folder = '2023') => + sqlite( + dbPath, + `INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('${id}', 'person:a', 'A.xml', ${hash === null ? 'NULL' : `'${hash}'`}, + '${folder}', 'assets', 'https://x/A.xml');`, + ); + expect(() => decl('d:n1', null)).not.toThrow(); + expect(() => decl('d:n2', null)).toThrow(/UNIQUE/i); // the case that used to slip through + expect(() => decl('d:1', 'H1')).not.toThrow(); + expect(() => decl('d:2', 'H1')).toThrow(/UNIQUE/i); + // POSITIVE CONTROLS — the key must not over-constrain. The register reuses basenames across + // FOLDERS, so the same xml_file in a different folder is a DIFFERENT declaration (load.mjs + // namespaces its id for this reason), and a genuinely different hash is a corrected re-filing. + expect(() => decl('d:f2', null, '2024')).not.toThrow(); + expect(() => decl('d:3', 'H2')).not.toThrow(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + // The retrofit has to be safe on a LIVE D1: ship-related-persons wipes rows, never table definitions, + // so a deployed database keeps its unconstrained 0003 shape until this migration rebuilds it. + it('0007 retrofits a pre-existing database, keeps its rows, and re-applies as a no-op', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + for (const m of [migration0, migration3, migration6]) readScript(dbPath, m); + // Simulate a database provisioned BEFORE today: 0003 now declares the constraints, so the legacy + // shape has to be recreated explicitly. This is the state every already-deployed environment is in + // — `CREATE TABLE IF NOT EXISTS` never revisited it and ship-related-persons only wipes rows. + sqlite( + dbPath, + `DROP TABLE declarations; + CREATE TABLE declarations ( + id TEXT PRIMARY KEY, person_id TEXT NOT NULL REFERENCES persons(id), xml_file TEXT NOT NULL, + control_hash TEXT, folder_year TEXT NOT NULL, declared_year TEXT, template TEXT NOT NULL, + category TEXT, institution TEXT, position TEXT, source_url TEXT NOT NULL, + UNIQUE (xml_file, control_hash)); + DROP TABLE interest_links; + CREATE TABLE interest_links ( + id TEXT PRIMARY KEY, link_key TEXT NOT NULL UNIQUE, + person_id TEXT NOT NULL REFERENCES persons(id), bidder_id TEXT NOT NULL, eik TEXT NOT NULL, + entity_key TEXT NOT NULL, match_method TEXT, matcher_version TEXT NOT NULL, + publish_tier TEXT NOT NULL, relation TEXT NOT NULL, + interest_class TEXT NOT NULL DEFAULT 'management_role', contemporaneous INTEGER NOT NULL DEFAULT 0, + own_institution TEXT NOT NULL DEFAULT 'none', evidence_count INTEGER NOT NULL DEFAULT 1, + first_declared_year TEXT, last_declared_year TEXT, contract_count INTEGER NOT NULL DEFAULT 0, + contract_value_eur REAL, first_contract_year TEXT, last_contract_year TEXT, + status TEXT NOT NULL DEFAULT 'held', verified_by TEXT, verified_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')));`, + ); + // The rows the retrofit must preserve — plus the two shapes it must not. + sqlite( + dbPath, + `INSERT INTO persons(id, name) VALUES ('person:a', 'А'); + INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('d:keep', 'person:a', 'K.xml', 'HK', '2023', 'assets', 'https://x/K.xml'); + INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('d:drop', 'person:a', 'D.xml', NULL, '2023', 'assets', 'https://x/D.xml'); + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:k', 'k', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership', 'published'); + -- The uninterpretable status §2 is about, already stored because nothing rejected it. + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:bad', 'bad', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership', 'published ');`, + ); + + readScript(dbPath, migration7); + // The valid rows survive the rebuild… + expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( + 'd:keep', + ); + expect(sqlite(dbPath, "SELECT link_key FROM interest_links WHERE link_key='k';").trim()).toBe( + 'k', + ); + // …while the trailing-space row cannot: a status we cannot interpret is not a link to publish or + // guess at, and the loader rebuilds the table on the next run. + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('1'); + // The hashless row SURVIVES — it is a real declaration and NOT NULL was rejected for that reason — + // but it is now covered by the natural key, so re-importing it is refused rather than duplicated. + expect(sqlite(dbPath, "SELECT COUNT(*) FROM declarations WHERE id='d:drop';").trim()).toBe( + '1', + ); + expect(() => + sqlite( + dbPath, + `INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('d:dup', 'person:a', 'D.xml', NULL, '2023', 'assets', 'https://x/D.xml');`, + ), + ).toThrow(/UNIQUE/i); + // …and the constraints are actually in force afterwards. + expect(() => + sqlite(dbPath, "UPDATE interest_links SET status='published ' WHERE link_key='k';"), + ).toThrow(/CHECK/i); + + // Migrations here are applied by a bare `d1 execute --file` with no applied-migrations tracking, + // so a second application MUST be a no-op rather than an error or a data loss. + readScript(dbPath, migration7); + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('1'); + expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( + 'd:keep', + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); From 021da5a1f37797b4ee9b26662b5cd84e98533f53 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:45:31 +0300 Subject: [PATCH 50/69] fix(db): gate the company-search badge on the evidence seal, and bind all four copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The detail-page gate is belt-and-braces — SURFACED_OWNERSHIP requires `status='published'` AND a publishing seal. The company-search badge checked status alone, so a published row whose seal was missing or withholding (a legacy row, a partial run, a loader bug) advertised a свързани-лица claim about a named official to every searcher, while the page behind the badge correctly showed nothing. Search is the wider surface of the two: it is what a reader sees before deciding to look. The table probe now requires interest_links AND interest_link_evidence. The join reads both, so an env with 0003 but not 0006 would 500 every search on every kind — the exact failure the probe was written to prevent. Requiring both also gives the right answer there: with no seal table nothing is provably sealed, so nothing should badge. The predicate now has four copies, and nothing but a test can bind SQL across three languages and two file types. related-persons-sql.test.ts asserts all four agree on both the EXISTS shape and the rung list, whitespace-insensitively. refresh-slice.sql is in that set for its own reason: it runs on the 6-hourly cron, so drift there silently keeps officials in the search index whose links no longer surface. Mutation-checked: adding a rung to refresh-slice.sql alone fails both binding tests. The badge fixture carries an unsealed published link and a withholding-sealed one, each of which must not badge, plus sealed links on both publishing rungs that must. --- packages/db/src/queries/search.ts | 45 +++++++++++----- packages/db/src/related-persons-sql.test.ts | 36 ++++++++++++- packages/db/src/search-sql.test.ts | 59 ++++++++++++++++++--- 3 files changed, 120 insertions(+), 20 deletions(-) diff --git a/packages/db/src/queries/search.ts b/packages/db/src/queries/search.ts index 7a19f01e..c3f3f02d 100644 --- a/packages/db/src/queries/search.ts +++ b/packages/db/src/queries/search.ts @@ -112,15 +112,23 @@ interface HitRow { // One group's ranked hits. Company rows additionally carry a свързани-лица flag: a LEFT JOIN against the // published conflict links keyed on the winner's ЕИК (= the company row's `ident`). Published, self OR family -// stake, AND the winner must have LIVE contracts (the same read-time N9 gate LINK_SELECT applies), so search -// flags exactly the companies the /conflicts surface shows and never one whose page would 404. A family-only -// winner badges — the /conflicts page already publishes that link by name, so the badge discloses nothing the -// surface doesn't. Binds: kind, match, limit. +// stake, backed by a Trade Register evidence SEAL, AND the winner must have LIVE contracts (the same +// read-time N9 gate LINK_SELECT applies), so search flags exactly the companies the /conflicts surface shows +// and never one whose page would 404. A family-only winner badges — the /conflicts page already publishes +// that link by name, so the badge discloses nothing the surface doesn't. Binds: kind, match, limit. +// +// The seal EXISTS clause is the same one SURFACED_OWNERSHIP applies (related-persons.ts), and it belongs +// here for a sharper reason than symmetry: search is the WIDER surface. A published row whose seal is +// missing or withholding — a legacy row, a partial run, a loader bug — would badge a свързани-лица claim +// about a named official to every searcher, while the page behind the badge correctly showed nothing. +// Four copies of this predicate now exist (here, SURFACED_OWNERSHIP, precompute.sql, refresh-slice.sql); +// related-persons-sql.test.ts pins them to each other. // Exported so search-sql.test runs the EXACT SQL (not a copy). -// Built with or without the interest_links conflict join. The join is on the свързани-лица migration (0003); -// on an env where it has not been applied yet the join would make the ENTIRE search 500 with „no such table: -// interest_links". search() detects the table once per request and picks the no-conflict variant when absent, -// so search degrades to has_conflict=0 rather than breaking (ADR-0031 robustness ask). +// Built with or without the conflict join. The join reads BOTH свързани-лица migrations — interest_links +// (0003) and interest_link_evidence (0006) — and on an env where either is unapplied it would make the +// ENTIRE search 500 with „no such table". search() detects both tables once per request and picks the +// no-conflict variant when either is absent, so search degrades to has_conflict=0 rather than breaking +// (ADR-0031 robustness ask). const hitsSql = (withConflict: boolean): string => `SELECT search_index.ref, search_index.title, search_index.ident, search_index.subtitle, search_index.amount, rank, ct.kind AS entity_kind, ct.ownership_kind, ct.eik_valid, @@ -133,6 +141,8 @@ const hitsSql = (withConflict: boolean): string => `SELECT search_index.ref, sea ? `LEFT JOIN ( SELECT DISTINCT il.eik FROM interest_links il WHERE il.status = 'published' AND il.interest_class IN ('private_ownership', 'family_ownership') + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed')) AND EXISTS (SELECT 1 FROM contracts cc JOIN bidders bb ON bb.id = cc.bidder_id WHERE bb.eik_normalized = il.eik) ) cf ON search_index.kind = 'company' AND cf.eik = search_index.ident` @@ -161,12 +171,21 @@ export async function search(db: D1Database, rawQuery: string): Promise(); const counts = new Map(countRows.results.map((r) => [r.kind, r.n])); - // Pick the hits SQL once: on an env where the свързани-лица migration (0003) is not applied, the conflict - // join would 500 the whole search — fall back to the no-conflict variant (has_conflict=0) instead. + // Pick the hits SQL once: on an env where the свързани-лица migrations are not applied, the conflict join + // would 500 the whole search — fall back to the no-conflict variant (has_conflict=0) instead. + // + // BOTH tables are required, not just interest_links: the join now also reads interest_link_evidence + // (0006), so an env with 0003 but not 0006 would break every search on every kind. Requiring both also + // gives the right answer on such an env — with no seal table nothing is provably sealed, so nothing + // should badge. const hasConflictTable = - (await db - .prepare("SELECT 1 AS n FROM sqlite_master WHERE type='table' AND name='interest_links'") - .first<{ n: number }>()) != null; + ( + await db + .prepare( + "SELECT COUNT(*) AS n FROM sqlite_master WHERE type='table' AND name IN ('interest_links','interest_link_evidence')", + ) + .first<{ n: number }>() + )?.n === 2; const hitsSql = hasConflictTable ? SEARCH_HITS_SQL : SEARCH_HITS_SQL_NO_CONFLICT; const built = await Promise.all( diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index 512da590..1501f16d 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -1,6 +1,6 @@ /// import { execFileSync } from 'node:child_process'; -import { mkdtempSync, rmSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -11,7 +11,9 @@ import { LINK_CONTRACTS_LIMIT, LINK_CONTRACTS_SQL, OFFICIAL_SQL, + SURFACED_OWNERSHIP, } from './queries/related-persons'; +import { SEARCH_HITS_SQL } from './queries/search'; // Integration test for the свързани-лица SQL. The query layer's unit tests (queries/related-persons.test) // use a fake D1 and never run the aggregation; this runs the EXACT exported SQL against a real SQLite @@ -629,3 +631,35 @@ describe('свързани-лица SQL (real SQLite)', () => { }); }); }); + +// The evidence-seal gate has FOUR copies, and they must say the same thing. Reviewers found the fourth +// (the company-search badge) checking `status='published'` alone, so search advertised a свързани-лица +// claim about a named official that the detail page correctly withheld. `refresh-slice.sql` matters for a +// different reason: it runs on the 6-hourly cron, so drift there silently keeps officials in the search +// index whose links no longer surface. Nothing but a test binds SQL in three languages and two file types. +describe('the evidence-seal gate is identical in all four places it is written', () => { + const sources: [string, string][] = [ + ['related-persons.ts (SURFACED_OWNERSHIP)', SURFACED_OWNERSHIP], + ['search.ts (company badge)', SEARCH_HITS_SQL], + ['precompute.sql', readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8')], + ['refresh-slice.sql', readFileSync(resolve(root, 'scripts/refresh-slice.sql'), 'utf8')], + ]; + + it.each(sources)('%s requires a publishing evidence seal', (_label, sql) => { + // Whitespace-insensitive: the four are formatted for their own file, and a line break is not drift. + const flat = sql.replace(/\s+/g, ' '); + expect(flat).toMatch( + /EXISTS \( ?SELECT 1 FROM interest_link_evidence e WHERE e\.link_key = il\.link_key AND e\.evidence_kind IN \('document','confirmed'\) ?\)/, + ); + }); + + it.each(sources)('%s admits exactly the two publishing rungs', (_label, sql) => { + // The rung list is the gate. A future rung added to one copy and not the others either leaks an + // unproven claim or silently drops a proven one, depending on which copy gained it. + const kinds = [...sql.matchAll(/evidence_kind IN \(([^)]*)\)/g)].map((m) => + m[1]!.replace(/\s|'/g, ''), + ); + expect(kinds.length).toBeGreaterThan(0); + for (const k of kinds) expect(k).toBe('document,confirmed'); + }); +}); diff --git a/packages/db/src/search-sql.test.ts b/packages/db/src/search-sql.test.ts index 58c25205..f371cd1d 100644 --- a/packages/db/src/search-sql.test.ts +++ b/packages/db/src/search-sql.test.ts @@ -16,6 +16,7 @@ import { SEARCH_HITS_SQL, SEARCH_HITS_SQL_NO_CONFLICT } from './queries/search'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); function readScript(dbPath: string, path: string): void { execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${path}\n`, stdio: 'pipe' }); @@ -44,17 +45,26 @@ INSERT INTO bidders (id, name, eik_normalized, eik_valid, kind) VALUES ('eik:333','ГАМА ООД','333333333',1,'company'), ('eik:444','ДЕЛТА ООД','444444444',1,'company'), ('eik:555','ЕПСИЛОН ООД','555555555',1,'company'), - ('eik:666','ЗЕТА ООД','666666666',1,'company'); + ('eik:666','ЗЕТА ООД','666666666',1,'company'), + -- #279: two winners with a PUBLISHED link that must not badge. ЙОТА's link carries NO evidence seal at + -- all (a legacy row, a partial run, or a loader bug); КАПА's seal is a WITHHOLDING rung. The detail page + -- refuses both via SURFACED_OWNERSHIP; the badge has to agree, or search advertises an unproven claim + -- about a named official on a page that then shows nothing. + ('eik:777','ЙОТА ООД','777777777',1,'company'), + ('eik:888','КАПА ООД','888888888',1,'company'); INSERT INTO company_totals (bidder_id, name, kind, eik, eik_valid, won_eur, contracts, authorities) VALUES ('eik:111','АЛФА ООД','company','111111111',1,1000000,1,1), ('eik:222','БЕТА ООД','company','222222222',1,500000,1,1), ('eik:333','ГАМА ООД','company','333333333',1,200000,1,1), ('eik:444','ДЕЛТА ООД','company','444444444',1,300000,1,1), ('eik:555','ЕПСИЛОН ООД','company','555555555',1,700000,1,1), - ('eik:666','ЗЕТА ООД','company','666666666',1,900000,1,1); + ('eik:666','ЗЕТА ООД','company','666666666',1,900000,1,1), + ('eik:777','ЙОТА ООД','company','777777777',1,400000,1,1), + ('eik:888','КАПА ООД','company','888888888',1,600000,1,1); INSERT INTO persons (id, name) VALUES ('person:ИВАН МИНЕВ','Иван Минев'),('person:ГЕОРГИ ПЕТРОВ','Георги Петров'), - ('person:ДАНА ФАМ','Дана Фам'),('person:БОРИС БОРД','Борис Борд'),('person:ДВОЕН ТЕСТ','Двоен Тест'); + ('person:ДАНА ФАМ','Дана Фам'),('person:БОРИС БОРД','Борис Борд'),('person:ДВОЕН ТЕСТ','Двоен Тест'), + ('person:БЕЗ ПЕЧАТ','Без Печат'),('person:ЗАДЪРЖАН ПЕЧАТ','Задържан Печат'); INSERT INTO declarations (id, person_id, xml_file, control_hash, folder_year, declared_year, template, category, institution, position, source_url) VALUES ('decl:i','person:ИВАН МИНЕВ','i.xml','H1','2024','2023','assets','','ОБЩИНА РУСЕ','', 'https://register.cacbg.bg/2024/i.xml'), ('decl:g','person:ГЕОРГИ ПЕТРОВ','g.xml','H2','2024','2023','assets','','МИНИСТЕРСТВО Х','', 'https://register.cacbg.bg/2024/g.xml'), @@ -75,15 +85,30 @@ INSERT INTO interest_links -- so the redundant-family collapse (ADR-0032) DROPS the family link from the index; the winner's €50k counts -- ONCE, not €100k — no de-anonymization vector, no double-count. ('il:ds','person:ДВОЕН ТЕСТ|666','person:ДВОЕН ТЕСТ','eik:666','666666666','ЗЕТА ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',0,'none',1,'2020','2023',1,50000,'2021','2021','published'), - ('il:dfam','person:ДВОЕН ТЕСТ|666|family','person:ДВОЕН ТЕСТ','eik:666','666666666','ЗЕТА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2023',1,50000,'2021','2021','published'); + ('il:dfam','person:ДВОЕН ТЕСТ|666|family','person:ДВОЕН ТЕСТ','eik:666','666666666','ЗЕТА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2023',1,50000,'2021','2021','published'), + -- published, ownership class, live contracts — and NO seal. Everything the badge used to check, passed. + ('il:no','person:БЕЗ ПЕЧАТ|777','person:БЕЗ ПЕЧАТ','eik:777','777777777','ЙОТА ООД','exact_name_key','v1','document','owns','private_ownership',0,'none',1,'2020','2023',1,400000,'2021','2021','published'), + -- published, and sealed with a rung that WITHHOLDS (ADR-0035). A seal existing is not a seal permitting. + ('il:un','person:ЗАДЪРЖАН ПЕЧАТ|888','person:ЗАДЪРЖАН ПЕЧАТ','eik:888','888888888','КАПА ООД','exact_name_key','v1','document_uncorroborated','owns','private_ownership',0,'none',1,'2020','2023',1,600000,'2021','2021','published'); +INSERT INTO interest_link_evidence (link_key, evidence_kind, lookup_date, rules_version, live_status) VALUES + ('person:ИВАН МИНЕВ|111','document','2026-08-12','tr-rules-1','live'), + ('person:ИВАН МИНЕВ|333','confirmed','2026-08-12','tr-rules-1','live'), + ('person:ДАНА ФАМ|444','confirmed','2026-08-12','tr-rules-1','live'), + ('person:БОРИС БОРД|555','document','2026-08-12','tr-rules-1','live'), + ('person:ДВОЕН ТЕСТ|666','document','2026-08-12','tr-rules-1','live'), + ('person:ДВОЕН ТЕСТ|666|family','document','2026-08-12','tr-rules-1','live'), + ('person:ЗАДЪРЖАН ПЕЧАТ|888','document_uncorroborated','2026-08-12','tr-rules-1','live'); INSERT INTO authorities (id, name) VALUES ('a:1','ВЕДОМСТВО ТЕСТ'); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES - ('t:1','s1','Т1','a:1','open'),('t:3','s3','Т3','a:1','open'),('t:4','s4','Т4','a:1','open'),('t:6','s6','Т6','a:1','open'); + ('t:1','s1','Т1','a:1','open'),('t:3','s3','Т3','a:1','open'),('t:4','s4','Т4','a:1','open'), + ('t:6','s6','Т6','a:1','open'),('t:7','s7','Т7','a:1','open'),('t:8','s8','Т8','a:1','open'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES ('c:1','t:1','eik:111',1000000,'EUR','2021-05-01','N1',1000000), ('c:3','t:3','eik:333',200000,'EUR','2021-05-01','N3',200000), ('c:4','t:4','eik:444',300000,'EUR','2021-05-01','N4',300000), - ('c:6','t:6','eik:666',50000,'EUR','2021-05-01','N6',50000); + ('c:6','t:6','eik:666',50000,'EUR','2021-05-01','N6',50000), + ('c:7','t:7','eik:777',400000,'EUR','2021-05-01','N7',400000), + ('c:8','t:8','eik:888',600000,'EUR','2021-05-01','N8',600000); `; // Search-index population — a STRUCTURAL proxy for scripts/precompute.sql's officials block: it exercises the @@ -116,6 +141,7 @@ function withDb(fn: (dbPath: string) => void): void { try { readScript(dbPath, migration0); readScript(dbPath, migration2); + readScript(dbPath, migration6); exec(dbPath, FIXTURE); exec(dbPath, POPULATE_INDEX); fn(dbPath); @@ -151,6 +177,27 @@ describe('search свързани-лица SQL', () => { }); }); + it('the badge requires an evidence SEAL, not merely status=published (#279)', () => { + // The detail page gate is belt-and-braces — status AND a publishing seal (SURFACED_OWNERSHIP). The + // badge checked status alone, so an evidence-less published row advertised a свързани-лица claim on a + // named official in search while the page it links to correctly withheld it. Search is the wider + // surface of the two: it is what a reader sees before deciding to look. + withDb((dbPath) => { + // NO seal at all — the legacy/partial-run/loader-bug shape. + const iota = rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'йота*', 10)); + expect(iota).toHaveLength(1); + expect(iota[0]!.has_conflict).toBe(0); + // Sealed, but with a rung that WITHHOLDS. A seal existing is not a seal permitting. + const kapa = rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'капа*', 10)); + expect(kapa).toHaveLength(1); + expect(kapa[0]!.has_conflict).toBe(0); + // POSITIVE CONTROL — a properly sealed link still badges, on both publishing rungs. Without this a + // gate that rejected everything would satisfy the two assertions above. + expect(rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'алфа*', 10))[0]!.has_conflict).toBe(1); + expect(rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'гама*', 10))[0]!.has_conflict).toBe(1); + }); + }); + it('indexes family_ownership (ADR-0032) but NEVER management_role/ex-officio (interest_class gate)', () => { withDb((dbPath) => { // family_ownership now reaches the index identically to self (ADR-0032): Дана (a relative's stake) IS From b8b4332442fab0dc13db107bde890a963964b942 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:48:01 +0300 Subject: [PATCH 51/69] fix(web,ci): make the page prose family-aware and default the data job to staging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §2.6 made the CARD labels tense-neutral and family-aware but left the surrounding page copy asserting „собствен дял" — an OWN stake — directly above cards that correctly read „деклариран дял на свързано лице". On a family-only page that is a false claim about the named official, and it is exactly the second source of truth the card-label fix set out to remove. `declaredStakeNoun` derives the wording from the links the page is actually rendering, so a page cannot describe a set it isn't showing. Family-AWARE rather than family-blind: an own-stake page still says „собствен дял" (a positive control covers this — deleting the claim everywhere would pass the family assertion while making the page vaguer than the data warrants), and a mixed set gets the only phrasing true of every card. The ledes also stopped claiming „точно съвпадение по фирмено име" — since #279 identity rests on a Trade Register fact, not on the name. related-persons-data.yml offered `default: dev` while no `dev` environment exists, every fallback in the file is `|| 'staging'`, and the note at the top says the job targets staging. A manual dispatch therefore went to an environment with no secrets and failed at the credential guard rather than running where intended. Dropped from the options too, so it cannot be picked deliberately either. Methodology: „премахва незабавно" now distinguishes the D1 write (immediate) from what a reader sees, which the page's own publicCache(3600) can hold for up to an hour. The promise was not wrong about the action, only about when it becomes visible. --- .github/workflows/related-persons-data.yml | 7 ++++-- apps/web/app/lib/conflicts.ts | 17 ++++++++++++++ apps/web/app/routes/conflict.company.tsx | 4 ++-- apps/web/app/routes/conflict.methodology.tsx | 12 ++++++---- apps/web/app/routes/conflict.official.tsx | 14 +++++++---- .../app/routes/conflict.pages.render.test.tsx | 23 +++++++++++++++++++ 6 files changed, 63 insertions(+), 14 deletions(-) diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index 8472b96f..02a0fa56 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -56,8 +56,11 @@ on: environment: description: Target D1 environment type: choice - options: [dev, staging, production] - default: dev + # No `dev` environment exists — every fallback in this file is `|| 'staging'` and the note above + # says the job targets staging. A `dev` default sent a manual dispatch to an environment with no + # secrets, so it failed at the credential guard rather than running where it was meant to. + options: [staging, production] + default: staging full_crawl: description: Crawl the CACBG corpus (slow). RESUMES from the cached raw XML — fetch.mjs skips files already on disk, so this fills gaps rather than re-fetching from scratch (clear the cache for a true rebuild). Off = extract from the existing raw cache without crawling. type: boolean diff --git a/apps/web/app/lib/conflicts.ts b/apps/web/app/lib/conflicts.ts index 5b99180e..7e0908c1 100644 --- a/apps/web/app/lib/conflicts.ts +++ b/apps/web/app/lib/conflicts.ts @@ -46,6 +46,23 @@ export function relationLabel(relation: string): string { return RELATION_LABEL[relation] ?? relation; } +/** + * How to describe a PAGE's set of links in prose (#279 §2.6). The card labels above are already + * family-aware; the surrounding page copy was not, and asserted „собствен дял" — an OWN stake — above + * cards that correctly read „свързано лице". On a family-only page that is a false claim about the named + * official, and it is the second source of truth the card-label fix set out to remove. + * + * Derived from the links themselves rather than passed in, so a page cannot describe a set it isn't + * rendering. Mixed sets get the neutral wording: it is the only phrasing true of every card. + */ +export function declaredStakeNoun(links: { relation: string }[]): string { + const anyFamily = links.some((l) => l.relation === 'related'); + const anySelf = links.some((l) => l.relation !== 'related'); + if (anyFamily && !anySelf) return 'дял на свързано лице'; + if (anyFamily && anySelf) return 'деклариран дял — собствен или на свързано лице'; + return 'собствен дял'; +} + // Defense in depth: the slug is base64url and the ЕИК numeric today (so encoding is a no-op), but if either // assumption ever drifts, an un-escaped `/`, `?` or `#` would break routing and the cache key. Escape the // dynamic segments unconditionally (ydimitrof #226, conflicts.ts). diff --git a/apps/web/app/routes/conflict.company.tsx b/apps/web/app/routes/conflict.company.tsx index 9dd28595..5de8e12e 100644 --- a/apps/web/app/routes/conflict.company.tsx +++ b/apps/web/app/routes/conflict.company.tsx @@ -9,7 +9,7 @@ import { ConflictCards } from '../components/ConflictCards'; import { publicCache } from '../lib/cache'; import { withDbRetry } from '../lib/retry'; import { seoMeta } from '../lib/meta'; -import { companyProfileHref } from '../lib/conflicts'; +import { companyProfileHref, declaredStakeNoun } from '../lib/conflicts'; // Officials with a published declared interest in one winner (by ЕИК). Reads interest_links only. 404 when // no official has a published link to this company — never an empty page under a company's name. @@ -59,7 +59,7 @@ export default function ConflictCompany({ loaderData }: Route.ComponentProps) { } title={company} - lede={`Длъжностни лица, декларирали собствен дял в това дружество пред КПКОНПИ. ${count(links.length)} ${plural(links.length, 'връзка', 'връзки')} — всяка е точно съвпадение по фирмено име.`} + lede={`Длъжностни лица, декларирали ${declaredStakeNoun(links)} в това дружество пред КПКОНПИ. ${count(links.length)} ${plural(links.length, 'връзка', 'връзки')} — всяка почива на проверим факт от Търговския регистър.`} /> diff --git a/apps/web/app/routes/conflict.methodology.tsx b/apps/web/app/routes/conflict.methodology.tsx index 209f11f9..d7b1f9d4 100644 --- a/apps/web/app/routes/conflict.methodology.tsx +++ b/apps/web/app/routes/conflict.methodology.tsx @@ -437,11 +437,13 @@ export default function ConflictMethodology() {

    Разглеждаме сигнала в срок до 7 работни дни. При основателен сигнал - връзката се премахва незабавно и се записва в списък за премахване, - който оцелява при всяко следващо обновяване — веднъж премахната - връзка не се възстановява при повторно зареждане на данните. Премахваме връзка при - доказана неточност; не заличаваме коректни записи по искане на изпълнител или - възложител — данните идват от публични източници и остават публични. + връзката се премахва веднага след решението и се записва в списък + за премахване, който оцелява при всяко следващо обновяване — веднъж + премахната връзка не се възстановява при повторно зареждане на данните. Премахването + се прилага незабавно в базата; страницата може да се вижда в кеш{' '} + до един час след това. Премахваме връзка при доказана неточност; не + заличаваме коректни записи по искане на изпълнител или възложител — данните идват от + публични източници и остават публични.

    • diff --git a/apps/web/app/routes/conflict.official.tsx b/apps/web/app/routes/conflict.official.tsx index 8ef91879..01710bf5 100644 --- a/apps/web/app/routes/conflict.official.tsx +++ b/apps/web/app/routes/conflict.official.tsx @@ -8,6 +8,7 @@ import { ConflictCards } from '../components/ConflictCards'; import { publicCache } from '../lib/cache'; import { withDbRetry } from '../lib/retry'; import { seoMeta } from '../lib/meta'; +import { declaredStakeNoun } from '../lib/conflicts'; // One office-holder's declared ownership links. Reads private-ownership interest_links only. 404 (not an // empty page) when the person has no published link — a bare page under someone's name reads as an @@ -39,6 +40,9 @@ export async function loader({ params, context }: Route.LoaderArgs) { export default function ConflictOfficial({ loaderData }: Route.ComponentProps) { const { official, links } = loaderData; + // Family-AWARE, not family-blind: the page must not assert an own stake above cards that say „свързано + // лице", and must not go vague where the stake really is the official's own (§2.6). + const stake = declaredStakeNoun(links); return ( <>

    Данните са от собствените декларации на лицето (публичен регистър на КПКОНПИ), - съпоставени точно с регистъра на изпълнителите. Показваме само 100% съвпадения и само - собствен деклариран дял. Сигнал за неточност:{' '} - Методология → Поправки. + съпоставени точно с регистъра на изпълнителите. Показваме само 100% съвпадения, и само + когато самоличността на дружеството е потвърдена от Търговския регистър. Сигнал за + неточност: Методология → Поправки.

    { expect(card.textContent).not.toContain('Кмет Тестов'); }); + it('a family-only page never asserts the official owns the stake (§2.6)', async () => { + // The CARD labels were made tense-neutral and family-aware, but the page lede and the section hint + // still read „декларирало собствен дял" — a second source of truth on the very page that renders a + // relative's stake. On a family-only page that is a false claim about the named official, printed + // above a card that correctly says „свързано лице". + await mount(ConflictOfficial as never, { + official: 'Кмет Тестов', + links: [link({ relation: 'related', company: 'ЕВРОСТРОЙ 21 ЕООД', eik: '333' })], + }); + expect(text()).not.toContain('собствен дял'); + expect(text()).toContain('деклариран дял на свързано лице'); + }); + + it('an own-stake page still says so — the wording is family-AWARE, not family-blind', async () => { + // POSITIVE CONTROL. Removing the claim everywhere would satisfy the assertion above while making the + // page vaguer than the data warrants: a self stake IS the official's own and should read that way. + await mount(ConflictOfficial as never, { + official: 'Кмет Тестов', + links: [link({ relation: 'owns', company: 'ЕВРОСТРОЙ 21 ЕООД', eik: '333' })], + }); + expect(text()).toContain('собствен дял'); + }); + it('meta() names the person in the title and marks the page noindex', () => { const tags = officialMeta({ data: { official: 'Иван Петров', links: [] }, From 2039b9f7ee1d472248f5d212bef2d1b0cfaae1c9 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:50:13 +0300 Subject: [PATCH 52/69] test: cover the four unexercised audit axes, every ownership field, and the rule disclosure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three gaps were named by reviewers, and none had a bug behind it — but each guards a path where failure is silent, which is the case for testing them rather than trusting them. Audit axes A_key_missing, A_eik_mismatch, B_bidder_eik and B_eik_invalid fired in no test. The audit is the last gate before a named claim ships, so an axis that quietly stopped firing would be invisible: A_eik_mismatch is the libel case in its purest form (a name resolving to one winner, the link published against another), and B_eik_invalid is the only rail keeping a checksum-invalid ЕИК off the surface. Mutation-checked — disabling B_eik_invalid fails its test — and paired with a positive control, since four assertions that something fails prove nothing unless the clean shape passes. Every rung-2 owner test used CR_F_19_L, leaving CR_F_18_L (едноличен собственик) and CR_F_23_L (ФЛ-търговец) unexercised. A dropped entry there would withhold an entire ownership shape — a sole owner, the commonest ЕООД form — as „Неизвестна", a recall hole with no symptom. Controlled by a non-ownership field code that must NOT match. The methodology page's rule disclosure is a launch condition under ADR-0033 decision 7, not a nicety, and nothing kept it in step with the ladder. The rule could change in evidence.mjs and leave the page describing a system that no longer exists — worse than not disclosing it. Now pinned: every rung by name, the joint-stock bar, the three-names/ one-record rule, ADR-0035's company gate, both halves of R10, the no-ЕГН homonym limit, and the company-identity-not-ownership distinction the surface rests on. --- .../app/routes/conflict.pages.render.test.tsx | 29 +++++++ scripts/cacbg/audit.test.mjs | 86 +++++++++++++++++++ scripts/tr/evidence.test.mjs | 18 ++++ 3 files changed, 133 insertions(+) diff --git a/apps/web/app/routes/conflict.pages.render.test.tsx b/apps/web/app/routes/conflict.pages.render.test.tsx index d94f4567..3178fed7 100644 --- a/apps/web/app/routes/conflict.pages.render.test.tsx +++ b/apps/web/app/routes/conflict.pages.render.test.tsx @@ -162,6 +162,35 @@ describe('/conflicts/company/:eik — render', () => { }); describe('/conflicts/methodology — render', () => { + it('discloses the matching rule verbatim — every rung, and what each may conclude', async () => { + // ADR-0021 E10 makes this page the disclosure of the rule, and ADR-0033 decision 7 makes it a LAUNCH + // CONDITION rather than a follow-up: a heuristic that asserts something about a named person is only + // defensible if the reader can see exactly what was asserted and why. Nothing but a test keeps the + // page in step with the ladder — the rule can change in evidence.mjs and leave the page describing a + // system that no longer exists, which is worse than not disclosing it at all. + await mount(ConflictMethodology as never, {}); + const t = text(); + // every rung of the ladder, by the name the seal and the card use + for (const rung of ['Документ', 'Потвърдено', 'Оборена', 'Неизвестна']) + expect(t).toContain(rung); + // rung 1 — the joint-stock bar and its reason (the „11 акции" trap) + expect(t).toContain('Акционерна форма'); + expect(t).toContain('не е публична'); + // rung 2 — all three names, one record, and the two refusals + expect(t).toContain('пълно съвпадение и на трите имена'); + expect(t).toContain('един и същ запис'); + // ADR-0035 — the company gate, the part a reader most needs to judge the claim + expect(t).toContain('Съвпадението по име само по себе си не стига'); + // R10 — the seat's temporal guard, both halves + expect(t).toContain('вписано преди декларирания период'); + expect(t).toContain('когато този период е известен'); + // the honest limit: no ЕГН, so a homonym is possible + expect(t).toContain('не съдържа ЕГН'); + expect(t).toContain('съименник'); + // what the register proves and what it does not — the distinction the whole surface rests on + expect(t).toContain('самоличността на дружеството'); + }); + it('states the three libel rails in plain language', async () => { await mount(ConflictMethodology as never, {}); const t = text(); diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index 267ce4ad..e0f8af4b 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -385,3 +385,89 @@ test('D positive control — an unchanged published set produces no monotonicity assert.equal(threw, false); assert.doesNotMatch(out, /D_monotonicity/); }); + +// The four axes below fire on shapes that no test exercised. Each is a hard finding — the audit is the +// last gate before a named claim ships — so an axis that silently stopped firing would be invisible. + +test('A_key_missing: a link whose entity_key is in NO live bidder → hard finding', () => { + // The key resolved when the link was built and does not now: the winner was renamed, re-keyed or + // dropped from the corpus. The link still names an official against a company we can no longer find, + // so it must stop the run rather than ship pointing at nothing. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('ИЗЧЕЗНАЛ ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'an unresolvable entity_key must fail the gate'); + assert.equal(/A_key_missing/.test(out), true); +}); + +test('A_eik_mismatch: a name-resolved key pointing at a DIFFERENT ЕИК than it resolves to → hard', () => { + // The libel case in its purest form: the name resolves to exactly one winner, and the link published a + // different company against it. Everything on the card — contracts, money, the ЕИК link — would be the + // wrong company's, under a real official's name. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`, `'b2','ДРУГ ЕООД','200000002',1`], + links: [ + `'il1','p1|200000002','p1','200000002','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b2','owns',0,1000,'published'`, + ], + seals: [ + `'p1|200000002','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true); + assert.equal(/A_eik_mismatch/.test(out), true); +}); + +test('B_bidder_eik: the stored bidder row disagrees with the link ЕИК → hard finding', () => { + // Row integrity. The card renders the BIDDER's name and money but links out on the link's ЕИК, so a + // disagreement means the reader is shown one company and sent to another. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`, `'b2','ДРУГ ЕООД','200000002',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b2','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true); + assert.equal(/B_bidder_eik/.test(out), true); +}); + +test('B_eik_invalid: publishing against a checksum-INVALID ЕИК → hard finding', () => { + // eik_valid=0 means the ЕИК failed its control digit, so it identifies no company at all. This axis is + // the rail that keeps such a row off the public surface, and nothing else re-checks it downstream. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',0`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'an invalid-ЕИК publish must fail the gate'); + assert.equal(/B_eik_invalid/.test(out), true); +}); + +test('the four axes above are a BOUND: a clean, valid, sealed link passes them all', () => { + // POSITIVE CONTROL for the whole block. Four assertions that something fails prove nothing unless the + // correct shape passes — an audit that flagged everything would satisfy every test above. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, false, `a clean link must pass: ${out}`); + for (const axis of ['A_key_missing', 'A_eik_mismatch', 'B_bidder_eik', 'B_eik_invalid']) + assert.equal(new RegExp(axis).test(out), false, `${axis} must not fire on a clean link`); +}); diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs index 3010698c..32123070 100644 --- a/scripts/tr/evidence.test.mjs +++ b/scripts/tr/evidence.test.mjs @@ -273,6 +273,24 @@ test('rung 3 — name uniqueness does NOT gate the stronger „Документ" ); }); +test('rung 2 — every OWNERSHIP field code can carry the match, not just CR_F_19_L', () => { + // OWNERSHIP_FIELDS is ['CR_F_18_L','CR_F_19_L','CR_F_23_L'] — едноличен собственик, съдружници, and + // ФЛ-търговец. Every owner test used CR_F_19_L, so a typo or a dropped entry in the other two would + // have silently withheld an entire ownership shape: a sole owner (the commonest ЕООД form) publishing + // as „Неизвестна" is a recall hole with no symptom. + for (const code of ['CR_F_18_L', 'CR_F_19_L', 'CR_F_23_L']) { + const d = deed([fld(code, container('ИВАН ПЕТРОВ ТЕСТОВ'))]); + const v = evidenceVerdict({ ...base, deed: d }); + assert.equal(v.kind, 'document', `${code} must carry an ownership match`); + assert.equal(v.registryRole, 'owner', `${code} is an OWNERSHIP field, not management`); + assert.equal(v.matchedFact, `role:owner:${code}`); + } + // POSITIVE CONTROL — a field that is NOT an ownership or manager field must not match at all, or the + // loop above would pass for a reason other than the one it claims. + const other = deed([fld('CR_F_99_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))]); + assert.notEqual(evidenceVerdict({ ...base, deed: other }).kind, 'document'); +}); + // ── rung 2's company gate: the winner-vs-non-winner homonym (ADR-0035) ──────── // // The ladder proves a person with these three tokens is registered in the company we LOOKED UP. It cannot From da80eafbabdd537e0b2e01d33e77b7ccc5ae42b2 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 17:00:55 +0300 Subject: [PATCH 53/69] fix(db): enforce the retrofit constraints with triggers, not a table rebuild MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `wrangler d1 migrations apply` rejected the create-copy-drop-rename rebuild: interest_link_evidence references interest_links(link_key), D1 enforces foreign keys, and `PRAGMA defer_foreign_keys` does not survive the statement-by-statement execution wrangler performs — the migration aborted with SQLITE_CONSTRAINT_FOREIGNKEY and the Durable Object rolled back. The sqlite3-CLI test passed because that CLI has foreign keys OFF by default, so only a real local apply surfaced it. Dropping the child first would work mechanically and is the wrong trade: it strips every evidence seal, and since the read gate REQUIRES a seal, that empties the public surface until the next monthly data run. BEFORE INSERT/UPDATE triggers enforce the identical invariant for every writer — including the hand-run UPDATE during an incident that no application-level validation covers — with no rebuild, no FK exposure and no seal loss. Fresh databases still get true CHECK constraints from 0003/0006; this is only the retrofit path for databases that already exist. Verified against real local D1: applies clean, and a second application is a no-op. The test now also covers UPDATE, and asserts that a pre-existing bad row SURVIVES — a trigger constrains future writes and cannot retroactively reject stored data, and deleting a real link on a guess would be worse than leaving one the read gate already refuses to show. --- .../0007_publishing_gate_constraints.sql | 128 +++++++++--------- packages/db/src/migrations.test.ts | 32 ++++- 2 files changed, 87 insertions(+), 73 deletions(-) diff --git a/packages/db/migrations/0007_publishing_gate_constraints.sql b/packages/db/migrations/0007_publishing_gate_constraints.sql index ac682308..7720ac45 100644 --- a/packages/db/migrations/0007_publishing_gate_constraints.sql +++ b/packages/db/migrations/0007_publishing_gate_constraints.sql @@ -8,20 +8,20 @@ -- -- WHY IT IS SAFE TO RE-RUN: migrations here are applied by a bare `wrangler d1 execute --file` with no -- applied-migrations tracking (see related-persons-data.yml / deploy.yml), so re-application MUST be a --- no-op. Both blocks below are written to converge: the index block is `IF NOT EXISTS` over a de-duplicated --- table, and the rebuild block copies the CURRENT table into a constrained one — run a second time it --- copies an already-constrained table into an identical one. Idempotent by construction, no version flag. +-- no-op. Everything below is `IF NOT EXISTS` over statements that converge. -- --- WHY A REBUILD FOR interest_links AND NOT `ALTER TABLE`: SQLite cannot add a CHECK in place, so --- create-copy-drop-rename is the documented route. `declarations` needs no rebuild — its defect was a --- constraint that failed to constrain, which a correct index fixes without touching the table. +-- WHY TRIGGERS AND NOT A TABLE REBUILD: SQLite cannot add a CHECK in place, so the textbook route is +-- create-copy-drop-rename. That route is WRONG here, and a real `wrangler d1 migrations apply` proved it: +-- `interest_link_evidence` references interest_links(link_key), D1 enforces foreign keys, and +-- `PRAGMA defer_foreign_keys` does not survive the statement-by-statement execution wrangler performs — +-- the rebuild aborts with SQLITE_CONSTRAINT_FOREIGNKEY and the Durable Object rolls back. Dropping the +-- child first would work mechanically but would strip every evidence seal, and since the read gate +-- REQUIRES a seal that empties the public surface until the next monthly data run. -- --- FOREIGN KEYS: `interest_link_evidence` references interest_links(link_key), and D1 enforces foreign --- keys, so dropping the parent mid-rebuild would be refused. `defer_foreign_keys` postpones the check to --- the end of the transaction — the documented way to rebuild a referenced table — by which point the --- renamed table satisfies it. D1 runs each file in one implicit transaction, so a failure rolls the whole --- file back rather than leaving a half-swapped table. -PRAGMA defer_foreign_keys = true; +-- A BEFORE INSERT/UPDATE trigger that RAISEs enforces the identical invariant for every writer — +-- including a hand-run UPDATE during an incident, which is when a bad value is most likely typed — with +-- no rebuild, no FK exposure and no seal loss. Freshly created databases still get true CHECK constraints +-- from 0003/0006; this is the retrofit path for databases that already exist. -- ── declarations ──────────────────────────────────────────────────────────────────────────────────── -- The table-level `UNIQUE (xml_file, control_hash)` did not constrain what it was written for. SQLite @@ -46,58 +46,54 @@ DELETE FROM declarations WHERE id NOT IN ( CREATE UNIQUE INDEX IF NOT EXISTS idx_declarations_natural_key ON declarations(xml_file, folder_year, COALESCE(control_hash, '')); --- ── interest_links ────────────────────────────────────────────────────────────────────────────────── --- `status` and `interest_class` ARE the publishing gate. The public surface is `status = 'published'` --- AND a surfaced `interest_class`, so a value that merely resembles one — 'published ' with a trailing --- space, the case #279 §2 names — passes every writer and then fails the gate silently. A CHECK binds --- every writer at once, including a hand-run UPDATE during an incident, which is when it is most likely --- to be typed. +-- ── interest_links: status / interest_class ───────────────────────────────────────────────────────── +-- These two ARE the publishing gate: the surface is `status = 'published'` AND a surfaced +-- `interest_class`. A value that merely resembles one — 'published ' with a trailing space, the case +-- #279 §2 names — passes every writer and then fails the gate silently, hiding a link that should show. -- --- Rows violating the new CHECKs are dropped by INSERT OR IGNORE rather than rewritten: a status we cannot --- interpret is not a link we should publish or guess at, and the loader rebuilds the table on every run. -CREATE TABLE IF NOT EXISTS interest_links_0007 ( - id TEXT PRIMARY KEY, - link_key TEXT NOT NULL UNIQUE, - person_id TEXT NOT NULL REFERENCES persons(id), - bidder_id TEXT NOT NULL, - eik TEXT NOT NULL, - entity_key TEXT NOT NULL, - match_method TEXT, - matcher_version TEXT NOT NULL, - publish_tier TEXT NOT NULL, - relation TEXT NOT NULL, - interest_class TEXT NOT NULL DEFAULT 'management_role' - CHECK (interest_class IN ('private_ownership','family_ownership', - 'ex_officio_board','management_role')), - contemporaneous INTEGER NOT NULL DEFAULT 0, - own_institution TEXT NOT NULL DEFAULT 'none', - evidence_count INTEGER NOT NULL DEFAULT 1, - first_declared_year TEXT, - last_declared_year TEXT, - contract_count INTEGER NOT NULL DEFAULT 0, - contract_value_eur REAL, - first_contract_year TEXT, - last_contract_year TEXT, - status TEXT NOT NULL DEFAULT 'held' - CHECK (status IN ('published','internal','held','withdrawn','suppressed')), - verified_by TEXT, - verified_at TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')) -); -INSERT OR IGNORE INTO interest_links_0007 - (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, - relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, - last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, - status, verified_by, verified_at, created_at) -SELECT id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, - relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, - last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, - status, verified_by, verified_at, created_at -FROM interest_links -WHERE status IN ('published','internal','held','withdrawn','suppressed') - AND interest_class IN ('private_ownership','family_ownership','ex_officio_board','management_role'); -DROP TABLE interest_links; -ALTER TABLE interest_links_0007 RENAME TO interest_links; -CREATE INDEX IF NOT EXISTS idx_interest_links_eik ON interest_links(eik); -CREATE INDEX IF NOT EXISTS idx_interest_links_person ON interest_links(person_id); -CREATE INDEX IF NOT EXISTS idx_interest_links_status ON interest_links(status); +-- One trigger per operation, since SQLite triggers are per-event. `RAISE(ABORT)` rolls back the +-- statement, so a bad write fails loudly at its source instead of surfacing later as a missing link. +DROP TRIGGER IF EXISTS trg_interest_links_status_ins; +CREATE TRIGGER trg_interest_links_status_ins +BEFORE INSERT ON interest_links +WHEN NEW.status NOT IN ('published','internal','held','withdrawn','suppressed') + OR NEW.interest_class NOT IN ('private_ownership','family_ownership','ex_officio_board','management_role') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_links.status/interest_class outside the publishing-gate enum'); +END; + +DROP TRIGGER IF EXISTS trg_interest_links_status_upd; +CREATE TRIGGER trg_interest_links_status_upd +BEFORE UPDATE ON interest_links +WHEN NEW.status NOT IN ('published','internal','held','withdrawn','suppressed') + OR NEW.interest_class NOT IN ('private_ownership','family_ownership','ex_officio_board','management_role') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_links.status/interest_class outside the publishing-gate enum'); +END; + +-- ── interest_link_evidence: evidence_kind ─────────────────────────────────────────────────────────── +-- The read gate filters on this column — SURFACED_OWNERSHIP admits exactly 'document' and 'confirmed' — +-- so an unlisted value either silently stops a link surfacing or, if it collides with a publishing name, +-- surfaces one that was never proven. 0006 declares this as a CHECK for new databases; the trigger is +-- the same rule for those provisioned before it. +DROP TRIGGER IF EXISTS trg_ile_kind_ins; +CREATE TRIGGER trg_ile_kind_ins +BEFORE INSERT ON interest_link_evidence +WHEN NEW.evidence_kind NOT IN ('document','confirmed','document_uncorroborated','refuted', + 'bar_joint_stock','unknown','outside_tr') + OR (NEW.registry_role IS NOT NULL AND NEW.registry_role NOT IN ('owner','manager')) + OR NEW.live_status NOT IN ('live','terminated_owner_still','terminated_manager_still','terminated') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_link_evidence enum outside the ADR-0033/0035 vocabulary'); +END; + +DROP TRIGGER IF EXISTS trg_ile_kind_upd; +CREATE TRIGGER trg_ile_kind_upd +BEFORE UPDATE ON interest_link_evidence +WHEN NEW.evidence_kind NOT IN ('document','confirmed','document_uncorroborated','refuted', + 'bar_joint_stock','unknown','outside_tr') + OR (NEW.registry_role IS NOT NULL AND NEW.registry_role NOT IN ('owner','manager')) + OR NEW.live_status NOT IN ('live','terminated_owner_still','terminated_manager_still','terminated') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_link_evidence enum outside the ADR-0033/0035 vocabulary'); +END; diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index 1997bbd2..bc5d9a22 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -354,16 +354,18 @@ describe('served migrations', () => { ); readScript(dbPath, migration7); - // The valid rows survive the rebuild… + // The valid rows survive — nothing is rebuilt, so nothing can be lost… expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( 'd:keep', ); expect(sqlite(dbPath, "SELECT link_key FROM interest_links WHERE link_key='k';").trim()).toBe( 'k', ); - // …while the trailing-space row cannot: a status we cannot interpret is not a link to publish or - // guess at, and the loader rebuilds the table on the next run. - expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('1'); + // …including the pre-existing trailing-space row. A trigger constrains FUTURE writes; it cannot + // retroactively reject a row already stored, and deleting one would mean dropping a real link on a + // guess. The loader rewrites the table wholesale on the next run, which is what corrects it — and + // the read gate never showed it anyway, since 'published ' is not 'published'. + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('2'); // The hashless row SURVIVES — it is a real declaration and NOT NULL was rejected for that reason — // but it is now covered by the natural key, so re-importing it is refused rather than duplicated. expect(sqlite(dbPath, "SELECT COUNT(*) FROM declarations WHERE id='d:drop';").trim()).toBe( @@ -377,15 +379,31 @@ describe('served migrations', () => { VALUES ('d:dup', 'person:a', 'D.xml', NULL, '2023', 'assets', 'https://x/D.xml');`, ), ).toThrow(/UNIQUE/i); - // …and the constraints are actually in force afterwards. + // …and the constraint is in force for every writer afterwards, on INSERT and on UPDATE alike. + // UPDATE is the one that matters most: a hand-run status change during an incident is exactly when + // a stray character gets typed, and it is the path no application-level validation covers. expect(() => sqlite(dbPath, "UPDATE interest_links SET status='published ' WHERE link_key='k';"), - ).toThrow(/CHECK/i); + ).toThrow(/CHECK failed/i); + expect(() => + sqlite( + dbPath, + `INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:x', 'x', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership', 'publushed');`, + ), + ).toThrow(/CHECK failed/i); + // POSITIVE CONTROL: a legitimate status change still goes through — the trigger is a bound. + expect(() => + sqlite(dbPath, "UPDATE interest_links SET status='withdrawn' WHERE link_key='k';"), + ).not.toThrow(); // Migrations here are applied by a bare `d1 execute --file` with no applied-migrations tracking, // so a second application MUST be a no-op rather than an error or a data loss. readScript(dbPath, migration7); - expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('1'); + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('2'); expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( 'd:keep', ); From 013b1773b6111a575c5325061628f7ec2531e531 Mon Sep 17 00:00:00 2001 From: Stefan Angelov Date: Wed, 12 Aug 2026 22:11:25 +0300 Subject: [PATCH 54/69] =?UTF-8?q?fix(etl):=20=D1=81=D0=B2=D1=8A=D1=80?= =?UTF-8?q?=D0=B6=D0=B8=20OCDS=20=D0=B0=D0=BD=D0=B5=D0=BA=D1=81=D0=B8?= =?UTF-8?q?=D1=82=D0=B5=20=D1=81=20=D0=B4=D0=BE=D0=B3=D0=BE=D0=B2=D0=BE?= =?UTF-8?q?=D1=80=D0=B8=D1=82=D0=B5=20=D0=BF=D1=80=D0=B5=D0=B7=20=D0=BC?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D0=B0=20tender.id=20=E2=86=92=20=D0=A3=D0=9D?= =?UTF-8?q?=D0=9F=20(#302)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OCDS анексите стейджваха OCID в `unp` вместо УНП, тъй че нито един не се връзваше с договор. УНП липсва в емисията; онова, което release-ът носи, е `tender.id` - същото пространство като `tenderId` на EOP. Поправката е в три части, неразделни: ingest хваща `tender_ext_id` и записва стойността от release-а като `value_before` с празно `value_after` (OCDS не може да знае стойността СЛЕД анекса и не бива да движи `current_value`); мост в derive-amendments.sql и refresh-slice.sql възстановява УНП през `tender_ext_id`, с отказ при двусмислие; и prefer-EOP dedup маха близнаците, за да не се удвои `annex_count`. По инкременталния път dedup-ът се сверява и с кумулативната сервирана таблица в двете посоки. Остатъчното подценяване е наблюдаемо през две диагностики, а инвариантът е закован от нова integrity проверка `amendment-twin-dedup`. --- docs/README.md | 1 + docs/etl.md | 8 + .../286-ocds-amendment-unp.md | 179 +++++++++++++ .../db/src/amendments-bridge-lockstep.test.ts | 42 ++++ packages/db/src/amendments-ocds-link.test.ts | 235 ++++++++++++++++++ packages/db/src/refresh-slice.test.ts | 183 ++++++++++++++ packages/ingest/src/ocds.test.ts | 42 ++++ packages/ingest/src/ocds.ts | 23 +- scripts/derive-amendments.sql | 107 +++++++- scripts/integrity-checks.mjs | 33 +++ scripts/refresh-slice.sql | 76 ++++++ 11 files changed, 924 insertions(+), 5 deletions(-) create mode 100644 docs/implementation-plans/286-ocds-amendment-unp.md create mode 100644 packages/db/src/amendments-bridge-lockstep.test.ts create mode 100644 packages/db/src/amendments-ocds-link.test.ts diff --git a/docs/README.md b/docs/README.md index 0a3bf6e0..408599db 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ - [`etl-pipeline-state.md`](etl-pipeline-state.md) — анализ на текущото състояние на ETL pipeline-а. - [`etl-architecture.md`](etl-architecture.md) — целевата ETL архитектура (RFC): предложение за състоянието и реда на изпълнение. - [`v1-implementation-plan.md`](v1-implementation-plan.md) — precompute слоят и пагинацията (защо rollup-и и keyset вместо per-request GROUP BY / OFFSET). +- [`implementation-plans/286-ocds-amendment-unp.md`](implementation-plans/286-ocds-amendment-unp.md) — защо OCDS анексите не се свързват с договор (OCID вместо УНП) и планът за поправка през bridge-а `tender.id → УНП` + prefer-EOP dedup (#286). - [`integrity-gate.md`](integrity-gate.md) — reconciliation gate-ът: hard asserts върху тоталите при import/CI. - [`anomaly-report.md`](anomaly-report.md) — cross-row аномалии при опресняване: какво `value_flag` не хваща на ниво отделен договор. - [`deploy.md`](deploy.md) — деплой към Cloudflare: двата Worker-а (`sigma`, `sigma-etl`) и споделеният D1 per environment. diff --git a/docs/etl.md b/docs/etl.md index c0f5a9a7..318ce288 100644 --- a/docs/etl.md +++ b/docs/etl.md @@ -230,6 +230,14 @@ CZK). `normalize-raw.sql` пази нативната записана стой присвоява verdict `value_flag` и безопасната за сумиране `amount_eur` на всеки доменен договор. Всички прагове се сравняват в евро след FX нормализация. +> **Изключение — prefer-EOP dedup на анексите (#286).** `derive-amendments.sql` / `refresh-slice.sql` +> **изтриват** OCDS редове от `raw_amendments`, когато EOP близнак покрива същия `(unp, contract_number)`. +> Това е единственият пазач срещу двойно броене, защото промоцията в `amendments` е безусловна — затова +> DELETE-ът, а не read-time филтър. Инвариантът е записан и се налага от integrity проверката +> `amendment-twin-dedup` (`scripts/integrity-checks.mjs`): нито една двойка `(unp, contract_number)` не +> носи едновременно EOP и OCDS сервиран ред. По slice пътя близнакът може да се раздели между прозорци, +> затова там dedup-ът сверява и кумулативната сервирана таблица (виж коментарите в `refresh-slice.sql`). + Verdict-ът се определя от две прогнозни стойности, като всеки флаг ползва тази, която избягва *собствената* си посока на false positive: diff --git a/docs/implementation-plans/286-ocds-amendment-unp.md b/docs/implementation-plans/286-ocds-amendment-unp.md new file mode 100644 index 00000000..e21755d4 --- /dev/null +++ b/docs/implementation-plans/286-ocds-amendment-unp.md @@ -0,0 +1,179 @@ +# Implementation Plan: #286 — OCDS amendments don't link to any contract (OCID instead of УНП) + +## Executive Summary + +| Field | Value | +|---|---| +| Ticket | [midt-bg/sigma#286](https://github.com/midt-bg/sigma/issues/286) — labels `data-quality`, `etl`, `priority: high` | +| Problem | OCDS-sourced rows in `amendments` store the **OCID** (`ocds-e82gsb-245534`) in `unp` instead of the **УНП** (`00044-2022-0146`), so **none** of them join to a contract. Measured live on `sigma-dev`: 4,800 OCDS rows, **0 linked**. | +| Root cause | `packages/ingest/src/ocds.ts` sets `unp: rel.ocid`. The УНП is **not present anywhere structured** in the OCDS feed — it must be recovered by bridging OCDS `tender.id` → EOP `tenderId` (`raw_tenders.tender_id` / `tenders.eop_tender_id`) → УНП (`source_id`). | +| Approach | (1) Ingest captures `tender_ext_id` + fixes value semantics; (2) SQL bridge rewrites the OCDS `unp` from the existing lots-bridge pattern; (3) prefer-EOP dedup so twins don't double-count and the value trap can't fire. | +| Complexity | Medium — small code, but the correctness traps (value semantics + cross-source dedup) are the whole point. | +| Risk | Medium — touches the value pipeline that feeds headline `current_value`; a naive fix understates contracts by millions (issue's €5.8M case). Mitigated by parts 2+3 shipping together and a real-corpus before/after. | +| Status | **Draft — validated against the live `sigma-dev` corpus.** | + +> **The two halves are inseparable.** Fixing the join key *without* fixing the OCDS value semantics is worse than the bug: once linked, the stale OCDS "before" value competes with the correct EOP value on the same `published_at` and the tiebreak can pick it. Key-alignment **and** value-correctness land in the same PR. + +--- + +## 1. Problem, verified on real data + +The УНП is genuinely absent from the OCDS releases. Pulling the real feed +`storage.eop.bg/open-data-2026-03-05/…OCDS.json` and walking all 28 +`contractAmendment` releases: the `NNNNN-YYYY-NNNN` УНП pattern appears **zero +times** as a structured field (once, incidentally, in free-text `rationale`). A +real release carries only: + +- `ocid: "ocds-e82gsb-425867"` and `tender.id: "425867"` (the EOP internal + procedure id = the OCID suffix) +- `contracts[].id: "180821"` (the contract number) +- `buyer.identifier.id: "000024948"` (authority EIK) +- `contracts[].value` — the **pre-amendment** value + +The EOP base "договори" feed, by contrast, carries **both** join anchors: +`uniqueProcurementNumber` (→ `unp`, the УНП) **and** `tenderId` (→ +`tender_ext_id`, the same id space as OCDS `tender.id`). + +### Reproduction on `sigma-dev` (live) + +| check | count | +|---|---| +| `amendments` rows with `source LIKE 'ocds:%'` | 4,800 | +| …with `unp LIKE 'ocds-%'` | 4,800 (all) | +| …that link via `contracts.tender_id = 't:' \|\| unp` | **0** | +| `eop:%` rows that link | 174,635 | + +### The value trap (contract 90029, live) + +| source | unp | value_before | value_after | published_at | +|---|---|---|---|---| +| `eop:annexes` | `00044-2022-0146` | 21,602,081.98 | **27,435,415.31** | 2026-03-05 | +| `ocds:` | `ocds-e82gsb-245534` | null | **21,602,081.98** | 2026-03-05 | + +`ocds.ts:378-379` writes `value_after = c.value.amount` (the *before* value) and +`value_before = null`. `derive-amendments.sql:42-50` selects `current_value` as +the latest non-null `value_after` with `ORDER BY published_at DESC, natural_key +DESC`. Same date ⇒ the tiebreak can pick OCDS's 21.6M and drop €5.8M. Twin +samples confirm OCDS values are generally stale/partial (e.g. contract 188980: +OCDS 111,163 vs EOP 217,416), so the OCDS value is never trustworthy as an +"after". + +--- + +## 2. Why the bridge is the right recovery (proven in-repo + on data) + +The exact bridge already exists for OCDS **lots** — +`scripts/normalize-raw.sql:1110-1133`: + +``` +-- The bridge is OCDS tender.id -> EOP tenderId (raw_tenders.tender_id) -> UNP -> domain lots. +-- ocid is a surrogate and is never treated as the UNP. +JOIN raw_tenders rt ON rt.tender_id = rl.tender_id +``` + +OCDS **amendments** never got the same treatment: `releaseToAmendments` doesn't +read `rel.tender.id`, even though `raw_amendments.tender_ext_id` already exists +as a column (`work-staging-schema.sql:183`) and EOP amendments populate it +(`base.ts:300`). + +Read-only validation of the bridge against `sigma-dev` (all OCDS `unp` are +`ocds-e82gsb-`; `tenders.eop_tender_id` maps to `source_id`/УНП): + +| step | count | +|---|---| +| OCDS amendments whose `tender.id` bridges to a tender (УНП recovered) | 4,797 / 4,800 | +| …that link to a contract with matching `contract_number` | **4,782** | +| …that have an **EOP twin** for the same `(unp, contract_number)` | 4,741 (~99%) | +| genuinely **OCDS-only** annexes (net-new) | ~41 | + +So ~99% of OCDS amendments duplicate an existing EOP annex. Linking them without +dedup would ~double `annex_count` on those contracts and arm the value trap. +Per-annex twin matching is unreliable (twin dates and values differ), so dedup is +done at the contract level: **prefer EOP; use OCDS only where EOP has no annex.** + +--- + +## 3. The fix (three parts, one PR) + +> **Shipped implementation — where it diverged from the plan below (this section is the original +> planning snapshot).** Part 2's bridge landed in `scripts/derive-amendments.sql` (mirrored in +> `scripts/refresh-slice.sql`), **not** `normalize-raw.sql` — `derive-amendments.sql` is the *first* +> consumer of `raw_amendments` on every path, so the rewrite has to happen there or its own rollup still +> joins on the OCID. Part 3's dedup shipped as a `DELETE FROM raw_amendments` (not an inline +> `WHERE NOT EXISTS` filter), and `promote-amendments.sql` is unchanged. On the incremental path the +> slice dedup additionally reconciles against the cumulative served `amendments` table (full path rebuilds +> it wholesale, so it needs no equivalent). See the review thread on the PR for the reasoning. + +### Part 1 — Ingest (`packages/ingest/src/ocds.ts`, `apps/etl` re-exports) +- Add `tender_ext_id` to `AmendmentStagingRow` and `AMENDMENT_STAGING_COLS`. +- In `releaseToAmendments`, set `tender_ext_id: clean(rel.tender?.id)`. +- Value semantics: store the release value as `value_before`; leave + `value_after = null`. OCDS cannot know the after-value, so it must never drive + `current_value`. (Verify `contractUpdate` separately before treating it the + same as `contractAmendment`; default to the conservative before-only mapping.) +- Unit tests updated in `packages/ingest/src/ocds.test.ts`. + +### Part 2 — SQL bridge (`scripts/normalize-raw.sql`) +- Before `derive-amendments.sql` / `promote-amendments.sql`, rewrite OCDS + amendment `unp` from the bridge, mirroring the lots block: + ```sql + UPDATE raw_amendments + SET unp = ( + SELECT rt.unp FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL + ) + WHERE source LIKE 'ocds:%' + AND tender_ext_id IS NOT NULL + AND EXISTS (SELECT 1 FROM raw_tenders rt WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL); + ``` + Fallback to `raw_contracts.tender_ext_id` where a tender row is absent. `ocid` + stays only as a surrogate, never a key. Mirror the same step in + `scripts/refresh-slice.sql` (the scoped slice path). + +### Part 3 — Prefer-EOP dedup (`derive-amendments.sql` + `promote-amendments.sql`) +- Include an OCDS amendment only when no EOP amendment exists for the same + `(unp, contract_number)`: + ```sql + AND NOT (source LIKE 'ocds:%' AND EXISTS ( + SELECT 1 FROM raw_amendments e + WHERE e.source LIKE 'eop:%' AND e.unp = raw_amendments.unp + AND e.contract_number = raw_amendments.contract_number)) + ``` + This kills the double-count and the value trap in one move while still + surfacing the ~41 OCDS-only annexes. OCDS-only rows keep `value_after = null`, + so they increment `annex_count` and become visible/linked without inventing a + `current_value`. + +--- + +## 4. Testing — proving we solved the real problem + +1. **Unit** (`packages/ingest/src/ocds.test.ts`): realistic fixture with + `tender.id` + value asserts `tender_ext_id` captured, `value_before` set, + `value_after` null. +2. **End-to-end SQL** (`packages/db/src/refresh-slice.test.ts` harness runs the + *actual* `normalize-raw → derive-amendments → promote-amendments` scripts via + the sqlite3 CLI). New case seeds: + - an EOP tender+contract (УНП `X`, `tenderId T`) with one EOP annex, + - an OCDS twin (`ocds-…-T`, same `contract_number`), + - an OCDS-only annex (`tenderId T2`, no EOP annex). + Assert: OCDS-only links & counts once; twin is dropped; the 90029 scenario + keeps `current_value = 27.4M` (never 21.6M). +3. **Repro regression**: encode the issue's SQL (`ocds:%` linked rows > 0, + no contract understated) as assertions. +4. **Real-corpus before/after**: rebuild a local slice from the public feeds and + run the repro queries; optionally dry-run-validate the bridge against + `sigma-dev` read-only. Expected: linked OCDS rows 0 → ~41 net (4,782 matched, + 4,741 deduped), zero contracts understated. + +--- + +## 5. Scope boundaries + +- **Out of scope:** OCDS *contracts* also store the OCID in `unp` + (`ocds.ts:323`); whether they reach the served domain at all is a separate + question — note it, don't fold it in. +- **Adjacent:** #248 (annex plausibility) and PR #285 (`value_delta` sign) touch + the same table but are independent defects. +- **Migration:** none — `raw_amendments.tender_ext_id` already exists; the change + is ingest + ETL SQL + tests only. diff --git a/packages/db/src/amendments-bridge-lockstep.test.ts b/packages/db/src/amendments-bridge-lockstep.test.ts new file mode 100644 index 00000000..2f99e840 --- /dev/null +++ b/packages/db/src/amendments-bridge-lockstep.test.ts @@ -0,0 +1,42 @@ +// Issue #286 — the tender.id → УНП bridge UPDATE is duplicated in the full path (derive-amendments.sql) +// and the incremental path (refresh-slice.sql). Both must recover the SAME УНП, or a daily slice refresh +// on the production Worker would silently regress to the #286 bug while every functional test stayed green +// (the Worker runs refresh-slice.sql, not derive-amendments.sql). The prefer-EOP dedup DELETE deliberately +// diverges — the slice path additionally reconciles against the cumulative served `amendments` — so only +// the marked bridge block is held byte-identical here. This is the repo's established drift-guard form +// (see search-sql.test.ts / precompute-cohort.test.ts). +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); + +const START = '-- @bridge-lockstep start'; +const END = '-- @bridge-lockstep end'; + +function bridgeBlock(file: string): string { + const sql = readFileSync(resolve(root, file), 'utf8'); + const start = sql.indexOf(START); + const end = sql.indexOf(END, start); + expect(start, `no "${START}" marker in ${file}`).toBeGreaterThanOrEqual(0); + expect(end, `no "${END}" marker in ${file}`).toBeGreaterThan(start); + return sql.slice(start + START.length, end).trim(); +} + +describe('OCDS amendment bridge lockstep (issue #286)', () => { + it('the bridge UPDATE is byte-identical between derive-amendments.sql and refresh-slice.sql', () => { + const derive = bridgeBlock('scripts/derive-amendments.sql'); + const slice = bridgeBlock('scripts/refresh-slice.sql'); + + // Sanity: the extracted span really is the bridge UPDATE, not an empty/misplaced marker range. + expect(derive).toContain('UPDATE raw_amendments'); + expect(derive).toContain('raw_amendments.tender_ext_id'); + expect(derive).toContain('ORDER BY rt.unp LIMIT 1'); + expect(derive).toContain('ORDER BY rc.unp LIMIT 1'); + + // The guard: exact byte-equality. Deleting the block from refresh-slice.sql fails on the missing + // marker above; changing the recovery in either file fails here. + expect(slice).toBe(derive); + }); +}); diff --git a/packages/db/src/amendments-ocds-link.test.ts b/packages/db/src/amendments-ocds-link.test.ts new file mode 100644 index 00000000..af0eac30 --- /dev/null +++ b/packages/db/src/amendments-ocds-link.test.ts @@ -0,0 +1,235 @@ +// Issue #286 — OCDS amendments must link to their contract via the recovered УНП, without +// double-counting the EOP annexes or letting an OCDS "before" value understate current_value. +// +// Runs the REAL scripts (derive-amendments.sql → promote-amendments.sql) against SQLite via the +// sqlite3 CLI, exactly as the ETL does, so the bridge + prefer-EOP dedup + value semantics are +// exercised as shipped — not a hand-copied mirror. +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const initSchema = resolve(root, 'packages/db/migrations/0000_init.sql'); +const workStagingSchema = resolve(root, 'scripts/work-staging-schema.sql'); +const deriveAmendments = resolve(root, 'scripts/derive-amendments.sql'); +const promoteAmendments = resolve(root, 'scripts/promote-amendments.sql'); + +function sqlite(dbPath: string, sql: string): void { + execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8', stdio: 'pipe' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + // Enforce FK constraints (mirrors refresh-slice.test.ts) so the promotion is validated against the + // served schema, not run with FK checks silently off. + execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +// Captures sqlite3 stdout (the SELECT results a `.read` prints) so a test can assert the diagnostic +// numbers the ETL emits, not just the table state. +function readScriptCapture(dbPath: string, path: string): string { + return execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + encoding: 'utf8', + }); +} + +// The numeric rows derive-amendments.sql prints (its diagnostics + the final summary), split into integer +// arrays and identified by column count: 1 col = ocds_ambiguous_bridges, 2 = dropped/excess-over-eop, +// 4 = the run summary. sqlite3's default output separates columns by '|' and rows by newlines. +function diagRows(out: string): number[][] { + return out + .split('\n') + .map((line) => line.trim()) + .filter((line) => /^\d+(\|\d+)*$/.test(line)) + .map((line) => line.split('|').map(Number)); +} + +// EOP УНП for the twin contract (90029), the OCDS-only contract (55500), and the OCDS-only contract +// whose procedure is contract-only — a "synthetic tender" absent from raw_tenders (77700). +const UNP_TWIN = '00044-2022-0146'; +const UNP_ONLY = '00099-2022-0009'; +const UNP_SYNTH = '00077-2022-0007'; +// A fourth OCDS annex whose procedure bridges NOWHERE — its tender.id is in neither raw_tenders nor +// raw_contracts. On the live corpus ~3/4,800 OCDS amendments are like this (#286 links 4,797/4,800); +// they honestly keep the OCID in `unp` because they can't be keyed to a contract yet — NOT dropped. +const OCID_UNBRIDGED = 'ocds-e82gsb-666666'; + +let dir: string; +let db: string; + +beforeEach(() => { + dir = mkdtempSync(resolve(tmpdir(), 'amendments-ocds-')); + db = resolve(dir, 'work.sqlite'); + readScript(db, initSchema); // served `amendments` + `contracts` + readScript(db, workStagingSchema); // raw_* staging + + // Two EOP procedures: T1 (contract 90029) already has an EOP annex; T2 (contract 55500) has NONE. + sqlite( + db, + `INSERT INTO raw_tenders (source, fetched_at, tender_id, unp) VALUES + ('eop:tenders:2026-03-05', '2026-03-05', 'T1', '${UNP_TWIN}'), + ('eop:tenders:2026-03-05', '2026-03-05', 'T2', '${UNP_ONLY}'); + INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency, tender_ext_id) VALUES + ('eop:contracts:2026-03-05', '2026-03-05', '${UNP_TWIN}', '90029', 21602081.98, 'EUR', 'T1'), + ('eop:contracts:2026-03-05', '2026-03-05', '${UNP_ONLY}', '55500', 500000, 'EUR', 'T2'), + -- 77700's procedure is contract-only: it has NO raw_tenders row, so the bridge must fall back + -- to raw_contracts.tender_ext_id (T3) to recover the УНП. + ('eop:contracts:2026-03-05', '2026-03-05', '${UNP_SYNTH}', '77700', 300000, 'EUR', 'T3'); + -- EOP annex for 90029 carries the correct after-value (27.4M). + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05', '2026-03-05', '${UNP_TWIN}', '90029', '2026-03-05', 'E1', 21602081.98, 27435415.31, 'EUR'); + -- OCDS twin of the 90029 annex: unp is the OCID, value is the pre-amendment number stored as value_before. + INSERT INTO raw_amendments (source, fetched_at, unp, tender_ext_id, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('ocds:2026-03-05', '2026-03-05', 'ocds-e82gsb-245534', 'T1', '90029', '2026-03-05', 'O1', 21602081.98, NULL, 'EUR'); + -- OCDS-only annex for 55500: exists in NO EOP feed. This is the row #286 wants to make visible. + INSERT INTO raw_amendments (source, fetched_at, unp, tender_ext_id, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('ocds:2026-04-01', '2026-04-01', 'ocds-e82gsb-999999', 'T2', '55500', '2026-04-01', 'O2', 480000, NULL, 'EUR'); + -- OCDS-only annex for 77700 (contract-only procedure) — bridges via the raw_contracts fallback. + INSERT INTO raw_amendments (source, fetched_at, unp, tender_ext_id, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('ocds:2026-04-01', '2026-04-01', 'ocds-e82gsb-777777', 'T3', '77700', '2026-04-01', 'O3', 300000, NULL, 'EUR'); + -- OCDS annex whose tender.id (T_MISSING) is in NEITHER raw_tenders nor raw_contracts — unbridgeable. + INSERT INTO raw_amendments (source, fetched_at, unp, tender_ext_id, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('ocds:2026-04-01', '2026-04-01', '${OCID_UNBRIDGED}', 'T_MISSING', '66600', '2026-04-01', 'O4', 300000, NULL, 'EUR');`, + ); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe('OCDS amendment → contract linkage (issue #286)', () => { + it('bridges the УНП, drops EOP twins, and never understates current_value', () => { + readScript(db, deriveAmendments); + + // Bridge: every *bridgeable* OCDS amendment now carries the real УНП. The one unbridgeable procedure + // (tender.id in neither raw_tenders nor raw_contracts) honestly keeps its OCID — it is NOT dropped. + const stillOcid = sqliteJson<{ contract_number: string; unp: string }>( + db, + "SELECT contract_number, unp FROM raw_amendments WHERE source LIKE 'ocds:%' AND unp LIKE 'ocds-%'", + ); + expect(stillOcid).toEqual([{ contract_number: '66600', unp: OCID_UNBRIDGED }]); + + // Prefer-EOP dedup: the 90029 OCDS twin is gone; the OCDS-only rows survive with their УНП — + // 55500 bridged via raw_tenders, 77700 via the raw_contracts synthetic-tender fallback, and 66600 + // stays on its OCID (unbridgeable, but surfaced rather than silently deleted). + const ocds = sqliteJson<{ contract_number: string; unp: string }>( + db, + "SELECT contract_number, unp FROM raw_amendments WHERE source LIKE 'ocds:%' ORDER BY contract_number", + ); + expect(ocds).toEqual([ + { contract_number: '55500', unp: UNP_ONLY }, + { contract_number: '66600', unp: OCID_UNBRIDGED }, + { contract_number: '77700', unp: UNP_SYNTH }, + ]); + + // The value trap: 90029 rolls up the EOP after-value (27.4M), never the stale OCDS 21.6M. + const twin = sqliteJson<{ annex_count: number; current_value: number | null }>( + db, + "SELECT annex_count, current_value FROM raw_contracts WHERE contract_number = '90029'", + ); + expect(twin[0]?.annex_count).toBe(1); + expect(twin[0]?.current_value).toBe(27435415.31); + + // OCDS-only annex is now visible on its contract (annex_count = 1); current_value stays NULL because + // OCDS cannot know the after-value (honest — no fabricated figure). + const only = sqliteJson<{ annex_count: number; current_value: number | null }>( + db, + "SELECT annex_count, current_value FROM raw_contracts WHERE contract_number = '55500'", + ); + expect(only[0]?.annex_count).toBe(1); + expect(only[0]?.current_value).toBeNull(); + }); + + it('promotes served amendments, keeping only the unbridgeable OCID as the honest residual', () => { + readScript(db, deriveAmendments); + readScript(db, promoteAmendments); + + // The issue's mass symptom is gone (4,800 → the handful with no bridge). The one procedure that + // bridges nowhere is promoted with its OCID still in unp — surfaced honestly, not silently dropped + // and not fabricated onto a contract. promote-amendments.sql intentionally promotes every staged row. + const dead = sqliteJson<{ contract_number: string; unp: string }>( + db, + "SELECT contract_number, unp FROM amendments WHERE unp LIKE 'ocds-%'", + ); + expect(dead).toEqual([{ contract_number: '66600', unp: OCID_UNBRIDGED }]); + + // The served rows: one EOP annex (90029), two bridged OCDS-only annexes (55500, 77700) keyed by real + // УНП, and the unbridgeable OCDS residual (66600) still on its OCID. + const served = sqliteJson<{ contract_number: string; unp: string; source: string }>( + db, + "SELECT contract_number, unp, CASE WHEN source LIKE 'ocds:%' THEN 'ocds' ELSE 'eop' END AS source FROM amendments ORDER BY contract_number", + ); + expect(served).toEqual([ + { contract_number: '55500', unp: UNP_ONLY, source: 'ocds' }, + { contract_number: '66600', unp: OCID_UNBRIDGED, source: 'ocds' }, + { contract_number: '77700', unp: UNP_SYNTH, source: 'ocds' }, + { contract_number: '90029', unp: UNP_TWIN, source: 'eop' }, + ]); + }); + + it('refuses to bridge (keeps the OCID) when a tender.id resolves to more than one УНП', () => { + // The domain is 1-to-1 (one procedure = one УНП). Give T2 a SECOND distinct УНП in raw_tenders. Rather + // than pick one arbitrarily (which would mis-attribute every annex of the losing procedure), the bridge + // must REFUSE: 55500's annex keeps its OCID as an honest residual, and the ocds_ambiguous_bridges + // diagnostic reports exactly one refusal (review nikimilenkov LOW 1). + sqlite( + db, + `INSERT INTO raw_tenders (source, fetched_at, tender_id, unp) VALUES + ('eop:tenders:2026-03-05', '2026-03-05', 'T2', 'ZZ-9999-9999');`, + ); + const out = readScriptCapture(db, deriveAmendments); + + const row = sqliteJson<{ unp: string }>( + db, + "SELECT unp FROM raw_amendments WHERE source LIKE 'ocds:%' AND contract_number = '55500'", + ); + expect(row).toEqual([{ unp: 'ocds-e82gsb-999999' }]); + + // The single-column diagnostic row reports the one ambiguous refusal. + expect(diagRows(out).find((r) => r.length === 1)).toEqual([1]); + }); + + it('recovers the УНП from raw_tenders in preference to raw_contracts (COALESCE order)', () => { + // One tender_ext_id (TP) resolves to DIFFERENT УНП in the two tables. The bridge tries raw_tenders + // first, so it must recover UNP-FROM-TENDERS — swapping the two COALESCE arms would fail this (#286). + sqlite( + db, + `INSERT INTO raw_tenders (source, fetched_at, tender_id, unp) VALUES + ('eop:tenders:2026-03-05', '2026-03-05', 'TP', 'UNP-FROM-TENDERS'); + INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency, tender_ext_id) VALUES + ('eop:contracts:2026-03-05', '2026-03-05', 'UNP-FROM-CONTRACTS', 'PREC-1', 100, 'EUR', 'TP'); + INSERT INTO raw_amendments (source, fetched_at, unp, tender_ext_id, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('ocds:2026-04-01', '2026-04-01', 'ocds-e82gsb-tp', 'TP', 'PREC-1', '2026-04-01', 'OP', 100, NULL, 'EUR');`, + ); + readScript(db, deriveAmendments); + + expect( + sqliteJson<{ unp: string }>( + db, + "SELECT unp FROM raw_amendments WHERE contract_number = 'PREC-1'", + ), + ).toEqual([{ unp: 'UNP-FROM-TENDERS' }]); + }); + + it('emits the residual diagnostics (dropped / excess-over-eop / ambiguous) for monitoring', () => { + // The base fixture has exactly one twin (90029: one OCDS annex vs one EOP annex) → dropped = 1, + // excess-over-eop = 0; and no ambiguous procedures → 0. These are the numbers the PR promises are + // "surfaced, not hidden" — pin them so a wrong CASE branch, or moving a diagnostic SELECT below the + // DELETE (which would zero the dropped/excess counts), fails CI (review nikimilenkov MEDIUM 4). + const out = readScriptCapture(db, deriveAmendments); + const rows = diagRows(out); + expect(rows.find((r) => r.length === 2)).toEqual([1, 0]); // dropped, excess-over-eop + expect(rows.find((r) => r.length === 1)).toEqual([0]); // ocds_ambiguous_bridges + }); +}); diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 261a2874..2ab6831d 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -650,6 +650,189 @@ describe('refresh-slice EOP base derivation', () => { } }); + it('bridges an OCDS-only annex to its УНП on the slice path (issue #286)', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-ocds-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + readScript(dbPath, workStagingSchemaPath); + + // An EOP procedure (tender + base contract) with УНП UNP-SLICE / tender.id TENDER-SLICE. An + // OCDS-only annex arrives keyed by the OCID, carrying tender_ext_id = TENDER-SLICE and NO EOP + // twin. The slice path's bridge must recover UNP-SLICE so the annex links to the served contract + // instead of staying a dead OCID row — the #286 fix, exercised through refresh-slice.sql. + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject, + cpv_code, cpv_description, contract_kind, estimated_value, currency, authority_name, + authority_eik, authority_type, published_at) + VALUES + ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-SLICE', 'TENDER-SLICE', + 'open', 'Slice tender', '45000000', 'Construction', 'works', 5000, 'BGN', + 'Authority Slice', '733456781', 'public', '2026-06-01'); + + INSERT INTO raw_contracts + (source, dataset_year, dataset_variant, fetched_at, needs_enrichment, document_number, + published_at, unp, tender_ext_id, procedure_type, procurement_subject, cpv_code, + cpv_description, contract_kind, estimated_value, procurement_currency, legal_basis, + award_criteria, authority_name, authority_eik, authority_type, main_activity, notice_type, + lot_id, contract_number, contract_date, signing_value, currency, contract_subject, + awarded_to_group, contractor_eik, contractor_name, contractor_country, winner_size, + eu_funded, bids_received, bids_sme, bids_rejected, bids_non_eea, duration_days) + VALUES + ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-SLICE', + '2026-06-01', 'UNP-SLICE', 'TENDER-SLICE', 'open', 'Slice tender', '45000000', + 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Slice', '733456781', + 'public', 'activity', 'notice', NULL, 'CONTRACT-SLICE', '2026-06-02', 1000, 'BGN', + 'Slice contract', 0, '787777778', 'Bidder Slice', 'BG', 'small', 0, 1, 1, 0, 0, 30); + + INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, + contract_number, contract_date, published_at, unp, tender_ext_id, authority_eik, + authority_name, procurement_subject, contract_kind, value_before, value_after, value_delta, + currency, description) + VALUES + ('ocds:2026-06-02', 2026, 'ocds', '2026-06-08T00:00:00Z', '1', 'AMD-SLICE-O', + 'CONTRACT-SLICE', '2026-06-02', '2026-06-03', 'ocds-e82gsb-555', 'TENDER-SLICE', + '733456781', 'Authority Slice', 'Slice tender', 'works', 1000, NULL, NULL, 'BGN', + 'OCDS-only annex');`, + ); + + readScript(dbPath, refreshSlicePath); + + // No served amendment keeps an OCID — the OCDS-only annex bridged to the real УНП. + expect( + sqliteJson<{ n: number }>( + dbPath, + "SELECT COUNT(*) AS n FROM amendments WHERE unp LIKE 'ocds-%'", + )[0]?.n, + ).toBe(0); + + // The annex is served against CONTRACT-SLICE, keyed by the recovered УНП (not the OCID). + expect( + sqliteJson<{ unp: string; source: string }>( + dbPath, + `SELECT unp, CASE WHEN source LIKE 'ocds:%' THEN 'ocds' ELSE 'eop' END AS source + FROM amendments WHERE contract_number = 'CONTRACT-SLICE'`, + ), + ).toEqual([{ unp: 'UNP-SLICE', source: 'ocds' }]); + + // …and it shows on the contract (annex_count = 1); current_value stays NULL — OCDS never sets + // an after-value, so nothing is fabricated. + expect( + sqliteJson<{ annex_count: number; current_value: number | null }>( + dbPath, + "SELECT annex_count, current_value FROM contracts WHERE contract_number = 'CONTRACT-SLICE'", + )[0], + ).toEqual({ annex_count: 1, current_value: null }); + + expect(sqlite(dbPath, 'PRAGMA foreign_key_check;').trim()).toBe(''); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('reconciles OCDS twins against the cumulative served amendments across windows (#286, HIGH 1)', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-xwindow-')); + try { + // A base EOP procedure (tender + contract) with УНП UNP-XW / tender.id TXW. `source` varies per + // window so a later slice re-reads the same contract (same derived id → INSERT OR REPLACE, no dup). + const base = (source: string): string => + `INSERT INTO raw_tenders + (source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject, + cpv_code, cpv_description, contract_kind, estimated_value, currency, authority_name, + authority_eik, authority_type, published_at) + VALUES + ('eop:tenders:${source}', 2026, '2026-06-09T00:00:00Z', 'UNP-XW', 'TXW', 'open', + 'Cross-window tender', '45000000', 'Construction', 'works', 5000, 'BGN', + 'Authority XW', '833456781', 'public', '2026-06-01'); + INSERT INTO raw_contracts + (source, dataset_year, dataset_variant, fetched_at, needs_enrichment, document_number, + published_at, unp, tender_ext_id, procedure_type, procurement_subject, cpv_code, + cpv_description, contract_kind, estimated_value, procurement_currency, legal_basis, + award_criteria, authority_name, authority_eik, authority_type, main_activity, notice_type, + lot_id, contract_number, contract_date, signing_value, currency, contract_subject, + awarded_to_group, contractor_eik, contractor_name, contractor_country, winner_size, + eu_funded, bids_received, bids_sme, bids_rejected, bids_non_eea, duration_days) + VALUES + ('eop:contracts:${source}', 2026, 'eop', '2026-06-09T00:00:00Z', 0, 'DOC-XW', + '2026-06-01', 'UNP-XW', 'TXW', 'open', 'Cross-window tender', '45000000', + 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority XW', '833456781', + 'public', 'activity', 'notice', NULL, 'CONTRACT-XW', '2026-06-02', 1000, 'BGN', + 'Cross-window contract', 0, '887777778', 'Bidder XW', 'BG', 'small', 0, 1, 1, 0, 0, 30);`; + + const eopAnnex = `INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, + contract_number, contract_date, published_at, unp, authority_eik, authority_name, + procurement_subject, contract_kind, value_before, value_after, value_delta, currency, description) + VALUES + ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-09T00:00:00Z', '1', 'AMD-XW-E', + 'CONTRACT-XW', '2026-06-02', '2026-06-03', 'UNP-XW', '833456781', 'Authority XW', + 'Cross-window tender', 'works', 1000, 1500, 500, 'BGN', 'EOP annex');`; + + const ocdsTwin = (source: string): string => + `INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, + contract_number, contract_date, published_at, unp, tender_ext_id, authority_eik, + authority_name, procurement_subject, contract_kind, value_before, value_after, value_delta, + currency, description) + VALUES + ('ocds:${source}', 2026, 'ocds', '2026-06-09T00:00:00Z', '1', 'AMD-XW-O', + 'CONTRACT-XW', '2026-06-02', '2026-06-03', 'ocds-e82gsb-321', 'TXW', '833456781', + 'Authority XW', 'Cross-window tender', 'works', 1000, NULL, NULL, 'BGN', 'OCDS twin');`; + + const servedRows = (dbPath: string) => + sqliteJson<{ unp: string; source: string }>( + dbPath, + `SELECT unp, CASE WHEN source LIKE 'ocds:%' THEN 'ocds' ELSE 'eop' END AS source + FROM amendments WHERE contract_number = 'CONTRACT-XW'`, + ); + const rollup = (dbPath: string) => + sqliteJson<{ annex_count: number; current_value: number | null }>( + dbPath, + "SELECT annex_count, current_value FROM contracts WHERE contract_number = 'CONTRACT-XW'", + )[0]; + + // Direction A — EOP annex served first, OCDS twin arrives in a LATER window (the EOP annex is not + // in that window's raw_amendments). Pre-fix, the twin bridged, survived the raw-only DELETE, and + // promoted into the cumulative served table → annex_count = 2. The served-table check must drop it. + const dbA = resolve(dir, 'a.sqlite'); + initWorkDb(dbA); + sqlite(dbA, `${base('2026-06-01')}\n${eopAnnex}`); + readScript(dbA, refreshSlicePath); + expect(servedRows(dbA)).toEqual([{ unp: 'UNP-XW', source: 'eop' }]); + + resetRawStaging(dbA); + sqlite(dbA, `${base('2026-06-05')}\n${ocdsTwin('2026-06-05')}`); + readScript(dbA, refreshSlicePath); + expect(servedRows(dbA)).toEqual([{ unp: 'UNP-XW', source: 'eop' }]); // NOT doubled + expect(rollup(dbA)).toEqual({ annex_count: 1, current_value: 1500 }); + expect(sqlite(dbA, 'PRAGMA foreign_key_check;').trim()).toBe(''); + + // Direction B — OCDS-only annex served first (net-new, no EOP twin yet), then the EOP annex arrives + // in a later window. The convergence DELETE must drop the stale served OCDS row so promotion of the + // EOP annex leaves exactly one served row (not two). + const dbB = resolve(dir, 'b.sqlite'); + initWorkDb(dbB); + sqlite(dbB, `${base('2026-06-01')}\n${ocdsTwin('2026-06-01')}`); + readScript(dbB, refreshSlicePath); + expect(servedRows(dbB)).toEqual([{ unp: 'UNP-XW', source: 'ocds' }]); + + resetRawStaging(dbB); + sqlite(dbB, `${base('2026-06-05')}\n${eopAnnex}`); + readScript(dbB, refreshSlicePath); + expect(servedRows(dbB)).toEqual([{ unp: 'UNP-XW', source: 'eop' }]); // OCDS twin converged away + expect(rollup(dbB)).toEqual({ annex_count: 1, current_value: 1500 }); + expect(sqlite(dbB, 'PRAGMA foreign_key_check;').trim()).toBe(''); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it('does not insert an OCDS duplicate after an existing EOP contract', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const dbPath = resolve(dir, 'test.sqlite'); diff --git a/packages/ingest/src/ocds.test.ts b/packages/ingest/src/ocds.test.ts index fd42274a..fd8b15cd 100644 --- a/packages/ingest/src/ocds.test.ts +++ b/packages/ingest/src/ocds.test.ts @@ -289,6 +289,48 @@ describe('releaseToAmendments', () => { expect(rows).toHaveLength(1); expect(rows[0]?.contract_number).toBe('DOC-1'); }); + + // Issue #286: the УНП is absent from OCDS releases, so the amendment must carry tender.id (the EOP + // procedure id) for the ETL bridge to recover the УНП. And the release value is the pre-amendment + // value, so it is stored as value_before with a null value_after — an OCDS row must never masquerade + // as a fresh current_value. `unp` still holds the ocid here; rewriting it to the real УНП is the job + // of derive-amendments.sql / refresh-slice.sql, not the pure flattener's. + it('captures the tender.id bridge and records the release value as value_before (issue #286)', () => { + const rows = releaseToAmendments( + { + ...release, + tag: ['contractAmendment'], + tender: { ...release.tender, id: '425867' }, + contracts: [ + { + ...release.contracts![0]!, + id: '90029', + value: { amount: 21_602_081.98, currency: 'EUR' }, + amendments: [{ description: 'Анекс 1', rationale: 'ЗОП чл. 116' }], + }, + ], + }, + meta, + ); + + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + contract_number: '90029', + unp: 'ocds-bg-2026-000123', + tender_ext_id: '425867', + value_before: 21_602_081.98, + value_after: null, + value_delta: null, + currency: 'EUR', + description: 'Анекс 1', + }); + }); + + it('captures a null tender_ext_id when the release has no tender.id', () => { + const rows = releaseToAmendments({ ...release, tag: ['contractUpdate'] }, meta); + expect(rows).toHaveLength(1); + expect(rows[0]?.tender_ext_id).toBeNull(); + }); }); describe('OCDS enrichment mappers', () => { diff --git a/packages/ingest/src/ocds.ts b/packages/ingest/src/ocds.ts index 1133671e..131ba304 100644 --- a/packages/ingest/src/ocds.ts +++ b/packages/ingest/src/ocds.ts @@ -216,6 +216,11 @@ export interface AmendmentStagingRow { reason: string | null; circumstances: string | null; sme: string | null; + // The EOP procedure id (OCDS `tender.id`) — the bridge to the УНП. The OCID is a surrogate; the + // real УНП is not in the OCDS release, so the ETL recovers it via tender_ext_id → raw_tenders → unp + // in scripts/derive-amendments.sql (and scripts/refresh-slice.sql on the incremental path), mirroring + // the OCDS-lots bridge that lives in scripts/normalize-raw.sql. See issue #286. + tender_ext_id: string | null; } export interface PartyStagingRow { @@ -317,7 +322,7 @@ export function releaseToContracts(rel: OcdsRelease, meta: OcdsMeta): ContractSt dataset_variant: 'OCDS', seq_no: null, document_number: rel.id ?? null, - contract_number: c.id ?? null, + contract_number: clean(c.id), contract_date: dateOnly(c.dateSigned), published_at: ctx.published_at, unp: rel.ocid ?? null, @@ -363,7 +368,7 @@ export function releaseToAmendments(rel: OcdsRelease, meta: OcdsMeta): Amendment dataset_variant: 'OCDS', seq_no: null, document_number: rel.id ?? null, - contract_number: c.id ?? null, + contract_number: clean(c.id), contract_date: dateOnly(c.dateSigned), published_at: ctx.published_at, unp: rel.ocid ?? null, @@ -375,14 +380,23 @@ export function releaseToAmendments(rel: OcdsRelease, meta: OcdsMeta): Amendment contract_subject: c.title || sup.awardTitle || null, contractor_eik: sup.eik, contractor_name: sup.name, - value_before: null, - value_after: finiteNum(c.value?.amount), + // An OCDS contractAmendment/contractUpdate release carries the contract value as it stands in + // that release — measured against the EOP annex stream this is the value BEFORE the amendment, + // never a reliable "after" (issue #286). Record it as value_before with a null value_after so an + // OCDS row can never drive the derived current_value (derive-amendments.sql selects the latest + // non-null value_after); the authoritative after-value comes from the EOP annex. + value_before: finiteNum(c.value?.amount), + value_after: null, value_delta: null, currency: isoCurrency(c.value?.currency), description: amd?.description || null, reason: amd?.rationale || null, circumstances: null, sme: null, + // OCDS tender.id === the EOP procedure id; the bridge to the УНП (derive-amendments.sql, and + // refresh-slice.sql on the incremental path). The ocid stored in `unp` here is a surrogate that + // the ETL bridge rewrites to the real УНП. + tender_ext_id: clean(rel.tender?.id), }, ]; }); @@ -544,6 +558,7 @@ export const AMENDMENT_STAGING_COLS: (keyof AmendmentStagingRow)[] = [ 'reason', 'circumstances', 'sme', + 'tender_ext_id', ]; export const PARTY_STAGING_COLS: (keyof PartyStagingRow)[] = [ diff --git a/scripts/derive-amendments.sql b/scripts/derive-amendments.sql index 2ab13245..23aaff87 100644 --- a/scripts/derive-amendments.sql +++ b/scripts/derive-amendments.sql @@ -1,10 +1,115 @@ -- Sigma — roll raw_amendments up onto raw_contracts. -- Run AFTER scripts/load-eop.mjs (which stages the EOP base + in-bucket OCDS amendments). --- Re-runnable: resets the rollup, then matches amendments by (unp, contract_number). +-- Re-runnable. First recovers the УНП for OCDS amendments via the tender.id bridge (#286), then +-- prefers the EOP annex over its OCDS twin (dropping the twin), then matches amendments by +-- (unp, contract_number). -- current_value = the after-value of the LATEST amendment; annex_count = how many. -- Contracts without amendments keep annex_count = 0 and current_value = NULL (the -- convention downstream is COALESCE(current_value, signing_value)). +-- #286: OCDS amendments stage the OCID in `unp` (the УНП is absent from the OCDS release), so they +-- match no contract. Recover the real УНП through the same bridge the OCDS-lots enrichment uses +-- (normalize-raw.sql): OCDS tender.id (staged as raw_amendments.tender_ext_id) → EOP tenderId → УНП. +-- The EOP tenderId lives in raw_tenders.tender_id for procedures in the поръчки feed AND in +-- raw_contracts.tender_ext_id for procedures that appear only as contracts (the "synthetic tenders" +-- of normalize-raw §2b) — so try raw_tenders first, then fall back to raw_contracts, else leave the +-- ocid untouched. The ocid stays only as a surrogate. Idempotent: a full run re-stages raw_amendments +-- and recomputes the same УНП. Being the first amendment step in the full pipeline, it leaves +-- raw_amendments corrected for promote-amendments.sql. +-- KEEP THE BRIDGE UPDATE BELOW IN LOCKSTEP with scripts/refresh-slice.sql: the UPDATE between the +-- @bridge-lockstep markers must stay byte-for-byte equivalent across both scripts (enforced by +-- packages/db/src/amendments-bridge-lockstep.test.ts). The prefer-EOP dedup DELETE deliberately does NOT +-- match — the slice path additionally reconciles against the cumulative served `amendments`, which this +-- full path never needs (promote-amendments.sql rebuilds it from scratch). raw_tenders(tender_id) is +-- indexed in work-staging-schema.sql, but raw_contracts(tender_ext_id) is not, so the fallback lookups +-- below would full-scan raw_contracts once per OCDS row — index it defensively (mirrors the lots bridge, +-- which indexes raw_tenders.tender_id before its join in normalize-raw.sql). The WHERE bridges only when +-- the chosen source maps the tender_ext_id to exactly ONE distinct УНП — see the refuse-on-ambiguity note +-- inside the block; the ocds_ambiguous_bridges diagnostic below surfaces any refusal. +CREATE INDEX IF NOT EXISTS idx_raw_contracts_tender_ext_id ON raw_contracts(tender_ext_id); +-- @bridge-lockstep start +UPDATE raw_amendments +SET unp = COALESCE( + (SELECT rt.unp FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL ORDER BY rt.unp LIMIT 1), + (SELECT rc.unp FROM raw_contracts rc + WHERE rc.tender_ext_id = raw_amendments.tender_ext_id AND rc.unp IS NOT NULL ORDER BY rc.unp LIMIT 1) +) +WHERE source LIKE 'ocds:%' + AND tender_ext_id IS NOT NULL + AND ( + -- raw_tenders wins when it resolves the procedure to exactly ONE УНП; else fall back to raw_contracts + -- when IT is unambiguous. Refuse to bridge (leave the OCID as an honest residual) when the chosen + -- source maps one tender_ext_id to more than one distinct УНП — the domain is 1-to-1, so this guards a + -- feed anomaly rather than silently mis-attributing every annex of the losing procedure (issue #286). + (SELECT COUNT(DISTINCT rt.unp) FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL) = 1 + OR ( + NOT EXISTS (SELECT 1 FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL) + AND (SELECT COUNT(DISTINCT rc.unp) FROM raw_contracts rc + WHERE rc.tender_ext_id = raw_amendments.tender_ext_id AND rc.unp IS NOT NULL) = 1 + ) + ); +-- @bridge-lockstep end + +-- #286 diagnostic (printed by wrangler, review nikimilenkov LOW 1): count OCDS amendments the bridge +-- REFUSED because their tender_ext_id resolves to more than one distinct УНП in the chosen source. The +-- domain is 1-to-1, so this must be 0 on a healthy feed; a non-zero value is a feed anomaly to investigate, +-- not a silent mis-attribution. Runs after the bridge, so refused rows still carry their OCID. +SELECT COUNT(*) AS ocds_ambiguous_bridges +FROM raw_amendments o +WHERE o.source LIKE 'ocds:%' + AND o.tender_ext_id IS NOT NULL + AND o.unp LIKE 'ocds-%' + AND ( + (SELECT COUNT(DISTINCT rt.unp) FROM raw_tenders rt + WHERE rt.tender_id = o.tender_ext_id AND rt.unp IS NOT NULL) > 1 + OR ( + NOT EXISTS (SELECT 1 FROM raw_tenders rt + WHERE rt.tender_id = o.tender_ext_id AND rt.unp IS NOT NULL) + AND (SELECT COUNT(DISTINCT rc.unp) FROM raw_contracts rc + WHERE rc.tender_ext_id = o.tender_ext_id AND rc.unp IS NOT NULL) > 1 + ) + ); + +-- #286 diagnostic (printed by wrangler, BEFORE the drop below): keep the residual OCDS under-count +-- OBSERVABLE. The prefer-EOP dedup is contract-level — it drops EVERY OCDS annex on a contract that +-- already has an EOP annex, so a genuinely OCDS-only *extra* amendment there is lost. Per-annex twin +-- matching can't rescue it: OCDS document_number is the release id (ocds-…) while EOP's is the АОП +-- document number — different id spaces that never align, so we cannot prove per-annex which drops are +-- true twins. Report bounds instead (run after the bridge above, so o.unp is the recovered УНП): +-- dropped = every OCDS annex removed by the dedup (UPPER bound on annexes lost) +-- excess_over_eop = Σ max(0, ocds_on_contract − eop_on_contract) (LOWER bound: cannot all be twins) +SELECT + COALESCE(SUM(g.ocds_n), 0) AS ocds_annexes_dropped, + COALESCE(SUM(CASE WHEN g.ocds_n > g.eop_n THEN g.ocds_n - g.eop_n ELSE 0 END), 0) + AS ocds_annexes_excess_over_eop +FROM ( + SELECT o.unp, o.contract_number, + COUNT(*) AS ocds_n, + (SELECT COUNT(*) FROM raw_amendments e + WHERE e.source LIKE 'eop:%' AND e.unp = o.unp AND e.contract_number = o.contract_number) AS eop_n + FROM raw_amendments o + WHERE o.source LIKE 'ocds:%' + AND EXISTS (SELECT 1 FROM raw_amendments e + WHERE e.source LIKE 'eop:%' AND e.unp = o.unp AND e.contract_number = o.contract_number) + GROUP BY o.unp, o.contract_number +) g; + +-- #286: prefer the EOP annex. ~99% of OCDS amendments duplicate an EOP annex for the same contract; +-- keeping both would double annex_count and duplicate the served timeline. Drop the OCDS twin when an +-- EOP annex already exists for the same (unp, contract_number). Genuinely OCDS-only annexes survive +-- (they carry value_after = NULL from ingest, so they never drive current_value — issue #286). +DELETE FROM raw_amendments +WHERE source LIKE 'ocds:%' + AND EXISTS ( + SELECT 1 FROM raw_amendments e + WHERE e.source LIKE 'eop:%' + AND e.unp = raw_amendments.unp + AND e.contract_number = raw_amendments.contract_number + ); + UPDATE raw_contracts SET annex_count = 0, current_value = NULL; WITH keyed AS ( diff --git a/scripts/integrity-checks.mjs b/scripts/integrity-checks.mjs index b48c4912..e87a57de 100644 --- a/scripts/integrity-checks.mjs +++ b/scripts/integrity-checks.mjs @@ -372,6 +372,38 @@ export async function checkStagingReconciliation(runner) { }; } +// 7) Amendment twin dedup (#286). The OCDS→EOP bridge lets an OCDS amendment reach the same contract as +// its EOP twin; the prefer-EOP dedup keeps only one per (unp, contract_number) by DELETE-ing OCDS rows +// before promotion — the SOLE guard, since promotion is unconditional. On the incremental path a twin +// can straddle windows (EOP served earlier, OCDS arriving later), so the slice dedup reconciles against +// the served table; this gate is the post-condition of that (and of the full path): no (unp, +// contract_number) may carry BOTH an EOP and an OCDS served amendment, or annex_count double-counts. +export async function checkAmendmentTwins(runner) { + const name = 'amendment-twin-dedup'; + if (!(await tableExists(runner, 'amendments'))) + return { name, ok: true, skipped: true, detail: 'amendments table absent' }; + const n = num( + await scalar( + runner, + 'SELECT COUNT(*) AS n FROM (' + + 'SELECT unp, contract_number FROM amendments ' + + 'WHERE unp IS NOT NULL AND contract_number IS NOT NULL ' + + 'GROUP BY unp, contract_number ' + + "HAVING SUM(source LIKE 'eop:%') > 0 AND SUM(source LIKE 'ocds:%') > 0)", + 'n', + ), + ); + return { + name, + ok: n === 0, + skipped: false, + detail: + n === 0 + ? 'no (unp, contract_number) carries both an EOP and an OCDS amendment (prefer-EOP dedup intact)' + : `${n} (unp, contract_number) carry both an EOP and an OCDS amendment — prefer-EOP dedup regressed and annex_count double-counts (#286)`, + }; +} + export const CHECKS = [ checkNonEmptyCorpus, checkRollupReconciliation, @@ -380,6 +412,7 @@ export const CHECKS = [ checkEikValidity, checkDateSanity, checkStagingReconciliation, + checkAmendmentTwins, ]; export async function runIntegrityChecks(runner) { diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 3f4544be..247b321d 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -24,6 +24,69 @@ DROP TABLE IF EXISTS refresh_amendment_winners; CREATE TABLE refresh_touched_contracts (id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_bidders (bidder_id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_authorities (authority_id TEXT PRIMARY KEY); + +-- #286: recover the УНП for OCDS amendments via the tender.id bridge before any raw_amendments read +-- below, mirroring derive-amendments.sql (raw_tenders first, then the raw_contracts synthetic-tender +-- fallback). In the slice path the raw tables hold only the loaded window, so an OCDS amendment whose +-- procedure was staged outside the window stays unbridged until the next full derive — best-effort by +-- design; the full pipeline is authoritative. +-- KEEP THE BRIDGE UPDATE BELOW IN LOCKSTEP with scripts/derive-amendments.sql: the UPDATE between the +-- @bridge-lockstep markers must stay byte-for-byte equivalent to the full path (packages/db/src/ +-- amendments-bridge-lockstep.test.ts enforces it). The prefer-EOP dedup INTENTIONALLY DIVERGES from the +-- full path — the slice path also reconciles against the cumulative served `amendments` table (below and +-- before the promotion), which the full path never needs because promote-amendments.sql rebuilds it from +-- scratch. Index raw_contracts(tender_ext_id) so the fallback lookups don't full-scan raw_contracts per +-- OCDS row (raw_tenders(tender_id) is already indexed); ORDER BY unp keeps the recovered УНП deterministic. +CREATE INDEX IF NOT EXISTS idx_raw_contracts_tender_ext_id ON raw_contracts(tender_ext_id); +-- @bridge-lockstep start +UPDATE raw_amendments +SET unp = COALESCE( + (SELECT rt.unp FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL ORDER BY rt.unp LIMIT 1), + (SELECT rc.unp FROM raw_contracts rc + WHERE rc.tender_ext_id = raw_amendments.tender_ext_id AND rc.unp IS NOT NULL ORDER BY rc.unp LIMIT 1) +) +WHERE source LIKE 'ocds:%' + AND tender_ext_id IS NOT NULL + AND ( + -- raw_tenders wins when it resolves the procedure to exactly ONE УНП; else fall back to raw_contracts + -- when IT is unambiguous. Refuse to bridge (leave the OCID as an honest residual) when the chosen + -- source maps one tender_ext_id to more than one distinct УНП — the domain is 1-to-1, so this guards a + -- feed anomaly rather than silently mis-attributing every annex of the losing procedure (issue #286). + (SELECT COUNT(DISTINCT rt.unp) FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL) = 1 + OR ( + NOT EXISTS (SELECT 1 FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL) + AND (SELECT COUNT(DISTINCT rc.unp) FROM raw_contracts rc + WHERE rc.tender_ext_id = raw_amendments.tender_ext_id AND rc.unp IS NOT NULL) = 1 + ) + ); +-- @bridge-lockstep end + +-- #286: prefer the EOP annex — drop OCDS twins so annex_count and the served timeline aren't doubled. +-- The full path (derive-amendments.sql) only checks raw_amendments because it re-stages the whole corpus +-- every run. The slice path must ALSO consult the cumulative served `amendments`: an EOP annex promoted by +-- an EARLIER window is not in this window's raw_amendments, yet its OCDS twin must still be dropped, or the +-- INSERT OR REPLACE promotion below (keyed by a natural_key that never matches across sources) would leave +-- both rows and double annex_count (issue #286, review nikimilenkov HIGH 1). +DELETE FROM raw_amendments +WHERE source LIKE 'ocds:%' + AND ( + EXISTS ( + SELECT 1 FROM raw_amendments e + WHERE e.source LIKE 'eop:%' + AND e.unp = raw_amendments.unp + AND e.contract_number = raw_amendments.contract_number + ) + OR EXISTS ( + SELECT 1 FROM amendments s + WHERE s.source LIKE 'eop:%' + AND s.unp = raw_amendments.unp + AND s.contract_number = raw_amendments.contract_number + ) + ); + CREATE TABLE refresh_amendment_winners AS WITH keyed AS ( SELECT *, @@ -1503,6 +1566,19 @@ WHERE status <> 'awarded' -- 5) Promote window amendments into served domain history and roll touched contracts. -- @refresh-batch amendments +-- #286 convergence (review nikimilenkov HIGH 1), the other direction of the served-table reconciliation +-- above: an EOP annex arriving in THIS window supersedes an OCDS twin a PRIOR slice already served for the +-- same (unp, contract_number). Drop the stale served OCDS row before promotion so the rollup never counts +-- both. The full path needs no equivalent — promote-amendments.sql rebuilds `amendments` wholesale. +DELETE FROM amendments +WHERE source LIKE 'ocds:%' + AND EXISTS ( + SELECT 1 FROM raw_amendments e + WHERE e.source LIKE 'eop:%' + AND e.unp = amendments.unp + AND e.contract_number = amendments.contract_number + ); + INSERT OR REPLACE INTO amendments ( id, natural_key, contract_number, unp, value_before, value_after, value_delta, currency, published_at, document_number, description, source From 1ab3dcd514f80fad3de17f4c95fde869f0b66ddc Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Wed, 12 Aug 2026 20:31:49 +0000 Subject: [PATCH 55/69] =?UTF-8?q?test(db):=20=D0=B7=D0=B0=D0=BA=D0=BE?= =?UTF-8?q?=D0=B2=D0=B8=20gate-=D0=B0=20=D0=B7=D0=B0=20=D0=B1=D0=BB=D0=B8?= =?UTF-8?q?=D0=B7=D0=BD=D0=B0=D1=86=D0=B8=20=D0=BF=D1=80=D0=B8=20=D0=B0?= =?UTF-8?q?=D0=BD=D0=B5=D0=BA=D1=81=D0=B8=D1=82=D0=B5=20(#303)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Проверката `amendment-twin-dedup` влезе в CHECKS с #302, но нищо не доказваше, че може да падне: мутация `ok: n === 0` до `ok: true` оставяше целия суит зелен при напълно обезсилен гейт. Пет случая по идиома на файла: засято нарушение; формите, които работещият dedup наистина оставя и които не бива да палят гейта (два EOP анекса на договор, само-OCDS анекс на друг договор, отказан от моста ред под своя OCID, NULL номер на договор и NULL УНП); броене на двойки, не на редове, с три нарушения, които делят стойност по всяка от осите, тъй че се заковава и самият ключ на групиране; и самопропускането при липсваща таблица. Източниците са реалните дялове, а EOP страната стои в двата края на диапазона 2020-2026, тъй че закачане за една година вали. Плюс липсващата декларация на `checkAmendmentTwins` в `integrity-checks.d.mts`, която #302 беше пропуснал. Четири кръга независим преглед; всяка находка е поправена и проверена с мутация. --- packages/db/src/integrity-checks.test.ts | 96 ++++++++++++++++++++++++ scripts/integrity-checks.d.mts | 1 + 2 files changed, 97 insertions(+) diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index 52acac25..82d28de4 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -13,6 +13,7 @@ import { fileURLToPath } from 'node:url'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { assertIntegrity, + checkAmendmentTwins, checkCurrentAmountParity, checkDateSanity, checkEikValidity, @@ -117,6 +118,7 @@ describe('reconciliation gate — clean corpus', () => { 'eik-validity', 'date-sanity', 'staging-reconciliation', + 'amendment-twin-dedup', ]) expect(results.find((r) => r.name === nm)?.skipped, `${nm} must not skip`).toBe(false); }); @@ -229,6 +231,100 @@ describe('reconciliation gate — injected violations', () => { expect(result.detail).toMatch(/EMPTY corpus/); }); + // #286/#302: the OCDS→EOP bridge lets an OCDS amendment reach the same contract as its EOP twin, and + // the prefer-EOP dedup (a DELETE two scripts earlier) is the SOLE guard, since promotion into + // `amendments` is unconditional. This gate is that dedup's post-condition. The suite already pins the + // dedup behaviourally; these cases pin the GATE — without them an `ok: n === 0` → `ok: true` edit + // leaves every package green while the gate is inert. + // + // Sources are the real staged partition shapes: `eop:annexes:` spanning 2020-05-08..2026-08-11 on + // the live corpus, `ocds:` only 2026 (the OCDS feed is go-forward). The gate matches by source + // PREFIX, so the fixtures below use both ends of the real EOP range — a pattern pinned to one year + // drops an arm and blows the count. The OCDS side cannot be spread the same way without inventing a + // partition that does not exist, so an `ocds:2026%` narrowing stays out of reach of honest fixtures; + // it is latent-in-2027, not a present regression. + it('amendment-twin-dedup catches an EOP and an OCDS amendment on the same (unp, contract_number)', async () => { + const db = track(freshDb()); + sqlite( + db, + `INSERT INTO amendments (id, natural_key, contract_number, unp, published_at, document_number, source) + VALUES + ('am:UNP-1:C-1:E1','am:UNP-1:C-1:E1','C-1','UNP-1','2026-03-05','E1','eop:annexes:2026-03-05'), + ('am:UNP-1:C-1:ocds-1','am:UNP-1:C-1:ocds-1','C-1','UNP-1','2026-03-05','ocds-e82gsb-1','ocds:2026-03-05');`, + ); + const result = await checkAmendmentTwins(runner(db)); + expect(result.ok).toBe(false); + expect(result.skipped).toBe(false); + // Anchor the COUNT (so `11` cannot satisfy `1`) but keep the prose match to the stable half of the + // message, not the whole sentence. + expect(result.detail).toMatch(/^1\b/); + expect(result.detail).toMatch(/prefer-EOP dedup regressed/); + }); + + it('amendment-twin-dedup stays green on the shapes a working dedup actually leaves behind', async () => { + const db = track(freshDb()); + sqlite( + db, + `INSERT INTO amendments (id, natural_key, contract_number, unp, published_at, document_number, source) + VALUES + -- two EOP annexes on one contract: the normal case, not a twin + ('am:UNP-1:C-1:E1','am:UNP-1:C-1:E1','C-1','UNP-1','2022-03-01','E1','eop:annexes:2022-03-01'), + ('am:UNP-1:C-1:E2','am:UNP-1:C-1:E2','C-1','UNP-1','2022-06-01','E2','eop:annexes:2022-06-01'), + -- a genuinely OCDS-only annex on a DIFFERENT contract: what the dedup deliberately keeps + ('am:UNP-2:C-2:ocds-1','am:UNP-2:C-2:ocds-1','C-2','UNP-2','2026-02-03','ocds-e82gsb-1','ocds:2026-02-03'), + -- an OCDS row the bridge refused (still keyed by its OCID) next to an EOP annex on the same + -- contract number: an honest residual, NOT double counting, because every consumer keys on + -- (unp, contract_number) and this row's unp matches no contract + ('am:ocds-3:C-1:ocds-2','am:ocds-3:C-1:ocds-2','C-1','ocds-e82gsb-3','2026-04-07','ocds-e82gsb-2','ocds:2026-04-07'), + -- NULL contract_number on both sides: cannot roll onto a contract, so it cannot double count + ('am:UNP-3::E3','am:UNP-3::E3',NULL,'UNP-3','2026-05-06','E3','eop:annexes:2026-05-06'), + ('am:UNP-3::ocds-3','am:UNP-3::ocds-3',NULL,'UNP-3','2026-05-06','ocds-e82gsb-4','ocds:2026-05-06'), + -- NULL unp on both sides, same contract number: amendments.unp is nullable and both mappers + -- can leave it empty, but such a row joins no contract either, so it must stay green too + ('am:C-9:E4','am:C-9:E4','C-9',NULL,'2026-07-02','E4','eop:annexes:2026-07-02'), + ('am:C-9:ocds-5','am:C-9:ocds-5','C-9',NULL,'2026-07-02','ocds-e82gsb-5','ocds:2026-07-02');`, + ); + const result = await checkAmendmentTwins(runner(db)); + expect(result.ok).toBe(true); + expect(result.skipped).toBe(false); + expect(result.detail).toMatch(/prefer-EOP dedup intact/); + }); + + // THREE offending pairs that deliberately share a value along each axis — (U1,C1), (U1,C2), (U2,C1) — + // so the grouping key itself is pinned: collapsing it to `unp` alone counts 2, to `contract_number` + // alone counts 2, and only the real composite key counts 3. One pair also carries two EOP rows, so + // counting rows instead of pairs overshoots. The EOP sources sit at both ends of the real partition + // range (2020 and 2026), which is what kills a year-pinned pattern. + it('amendment-twin-dedup counts each offending pair once, keyed on BOTH columns', async () => { + const db = track(freshDb()); + sqlite( + db, + `INSERT INTO amendments (id, natural_key, contract_number, unp, published_at, document_number, source) + VALUES + ('am:UNP-1:C-1:E1','am:UNP-1:C-1:E1','C-1','UNP-1','2020-05-08','E1','eop:annexes:2020-05-08'), + ('am:UNP-1:C-1:E2','am:UNP-1:C-1:E2','C-1','UNP-1','2026-08-11','E2','eop:annexes:2026-08-11'), + ('am:UNP-1:C-1:ocds-1','am:UNP-1:C-1:ocds-1','C-1','UNP-1','2026-01-04','ocds-e82gsb-1','ocds:2026-01-04'), + -- ...and TWO OCDS rows, which the dedup would drop together: a gate that demanded exactly one + -- row per side would walk straight past this pair + ('am:UNP-1:C-1:ocds-4','am:UNP-1:C-1:ocds-4','C-1','UNP-1','2026-06-15','ocds-e82gsb-4','ocds:2026-06-15'), + ('am:UNP-1:C-2:E3','am:UNP-1:C-2:E3','C-2','UNP-1','2023-09-12','E3','eop:annexes:2023-09-12'), + ('am:UNP-1:C-2:ocds-2','am:UNP-1:C-2:ocds-2','C-2','UNP-1','2026-05-20','ocds-e82gsb-2','ocds:2026-05-20'), + ('am:UNP-2:C-1:E4','am:UNP-2:C-1:E4','C-1','UNP-2','2026-08-11','E4','eop:annexes:2026-08-11'), + ('am:UNP-2:C-1:ocds-3','am:UNP-2:C-1:ocds-3','C-1','UNP-2','2026-08-11','ocds-e82gsb-3','ocds:2026-08-11');`, + ); + const result = await checkAmendmentTwins(runner(db)); + expect(result.ok).toBe(false); + expect(result.detail).toMatch(/^3\b/); + }); + + it('amendment-twin-dedup self-skips when the amendments table is absent (staging-only DB)', async () => { + const db = track(freshDb()); + sqlite(db, 'DROP TABLE amendments;'); + const result = await checkAmendmentTwins(runner(db)); + expect(result.skipped).toBe(true); + expect(result.ok).toBe(true); + }); + it('eik-validity catches eik_valid=1 with a non-numeric eik_normalized', async () => { const db = track(freshDb()); sqlite(db, "UPDATE bidders SET eik_normalized = 'AB12' WHERE id = 'eik:131071587';"); diff --git a/scripts/integrity-checks.d.mts b/scripts/integrity-checks.d.mts index d358d0af..82a6207e 100644 --- a/scripts/integrity-checks.d.mts +++ b/scripts/integrity-checks.d.mts @@ -30,6 +30,7 @@ export function checkNoNegativeValues(runner: IntegrityRunner): Promise; export function checkDateSanity(runner: IntegrityRunner): Promise; export function checkStagingReconciliation(runner: IntegrityRunner): Promise; +export function checkAmendmentTwins(runner: IntegrityRunner): Promise; export const CHECKS: Array<(runner: IntegrityRunner) => Promise>; export function runIntegrityChecks(runner: IntegrityRunner): Promise; From 753dd4df3a7ebeb0e7050158afae44f6955b14bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 20:59:44 +0000 Subject: [PATCH 56/69] =?UTF-8?q?ci(deps):=20=D0=B2=D0=B4=D0=B8=D0=B3?= =?UTF-8?q?=D0=BD=D0=B8=20=D0=B3=D1=80=D1=83=D0=BF=D0=B0=D1=82=D0=B0=20git?= =?UTF-8?q?hub-actions=20(5=20=D0=BE=D0=B1=D0=BD=D0=BE=D0=B2=D1=8F=D0=B2?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D1=8F)=20(#292)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit checkout 7.0.0 → 7.0.1, pnpm/action-setup 6.0.9 → 6.0.10, setup-node 6.4.0 → 7.0.0, actions/cache restore и save 4 → 6.1.0. Всичките заковани по SHA с коментар за версията; само файлове в .github/workflows. Двете големи скачания са изпълнени от CI върху обновения клон, включително заданието cacbg, което иска Node 24. --- .github/workflows/ci.yml | 4 ++-- .github/workflows/deploy.yml | 2 +- .github/workflows/related-persons-data.yml | 10 +++++----- .github/workflows/scripts-test.yml | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cebb18a..b4a7f739 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: container: image: semgrep/semgrep:1.170.0@sha256:c98f8829eea377274ee4b10656458b078b88232469b2ff913f091c2317347c9d steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Static analysis (semgrep) run: semgrep scan --config p/security-audit --config p/secrets --config p/typescript --metrics=off --error @@ -55,7 +55,7 @@ jobs: tar -xzf "$RUNNER_TEMP/gitleaks.tar.gz" -C "$RUNNER_TEMP" gitleaks "$RUNNER_TEMP/gitleaks" dir . --redact --no-banner - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2c3f42ae..5d5ca680 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -83,7 +83,7 @@ jobs: SIGMA_VECTORIZE_NAME: ${{ vars.SIGMA_VECTORIZE_NAME }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index deb24884..a133ed54 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -61,9 +61,9 @@ jobs: SUPPRESSION_SALT: ${{ secrets.SUPPRESSION_SALT }} SUPPRESSION_KEY_VERSION: ${{ vars.SUPPRESSION_KEY_VERSION || '1' }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24 # node:sqlite (DatabaseSync) + native TS type-stripping, no flags cache: pnpm @@ -112,7 +112,7 @@ jobs: # (Cross-run only — a hard mid-crawl job timeout can't save; durable incremental persistence is the # sigma-etl Worker's R2 job per ADR-0006. GitHub's 10 GB repo cache is the ceiling here.) - name: Restore raw CACBG corpus cache - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: scratch/cacbg/raw key: cacbg-raw-${{ github.run_id }} @@ -129,7 +129,7 @@ jobs: # run resumes from here. Skipped on full_crawl=false runs (nothing new was fetched). - name: Save raw CACBG corpus cache if: ${{ always() && inputs.full_crawl }} - uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: scratch/cacbg/raw key: cacbg-raw-${{ github.run_id }} diff --git a/.github/workflows/scripts-test.yml b/.github/workflows/scripts-test.yml index 6ce4fa7d..b4dd794f 100644 --- a/.github/workflows/scripts-test.yml +++ b/.github/workflows/scripts-test.yml @@ -20,8 +20,8 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 # node:test only — the scripts under test import nothing outside node built-ins and each other, @@ -40,9 +40,9 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24 cache: pnpm From daeb6e999cd191736810dbf810785ea7064be51f Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Wed, 12 Aug 2026 21:07:57 +0000 Subject: [PATCH 57/69] =?UTF-8?q?fix(etl):=20=D1=85=D0=B2=D0=B0=D1=89?= =?UTF-8?q?=D0=B0=D0=B9=20=D1=81=D1=82=D0=BE=D0=B9=D0=BD=D0=BE=D1=81=D1=82?= =?UTF-8?q?,=20=D0=B2=D1=8A=D0=B2=D0=B5=D0=B4=D0=B5=D0=BD=D0=B0=20=D0=B2?= =?UTF-8?q?=20=D1=81=D1=82=D0=BE=D1=82=D0=B8=D0=BD=D0=BA=D0=B8=20(#298)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Когато възложителят пропусне десетичния знак, стойността излиза точно 100 пъти по-голяма. `value_suspect` искаше над 200x от прогнозата, тъй че тези случаи минаваха отдолу и се сервираха на лице. Прагът не е свален - добавена е тясна лента около 100x. Мярката върху корпуса показва изолирано струпване: 13 договора между 95x и 105x, само 2 между 85x и 95x, и НУЛА между 105x и 200x. Границата идва от данните, не от преценка. Всичките 13 са проверени поотделно и са безспорни. Най-ясният: 05492-2023-0002 е доставка на 25 хил. литра дизелово гориво - при показаните 3 049 971 EUR излиза 122 EUR на литър, при прогнозата 31 189 EUR излиза 1,25 EUR на литър. Сред тях е и 00105-2025-0026 от доклада: 102 258 376 EUR при прогноза 1 022 584. Ефект: 13 договора, 318,3 млн. EUR показвани, 3,2 млн. след поправката към прогнозата. Логиката на флага стои в пет копия в двата derive пътя; сменени са всичките. Тестът кара истинските скриптове срещу истинска sqlite веднъж за всеки път, тъй че пропуснато копие вали. Мутации: махането на лентата от който и да е от двата файла вали по един тест, а разширяването ѝ до 90x-110x вали два - границата е закована, не козметична. Затваря #247. --- .../db/src/value-flag-stotinki-sql.test.ts | 166 ++++++++++++++++++ scripts/normalize-raw.sql | 15 +- scripts/refresh-slice.sql | 18 +- 3 files changed, 193 insertions(+), 6 deletions(-) create mode 100644 packages/db/src/value-flag-stotinki-sql.test.ts diff --git a/packages/db/src/value-flag-stotinki-sql.test.ts b/packages/db/src/value-flag-stotinki-sql.test.ts new file mode 100644 index 00000000..2fc97624 --- /dev/null +++ b/packages/db/src/value-flag-stotinki-sql.test.ts @@ -0,0 +1,166 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +// The стотинки band (#247): a value entered in cents rather than leva lands at almost exactly 100x the +// procedure estimate, well under the 200x threshold that value_suspect used to need, so it was served +// at face value. Measured on the real corpus the ratios form an isolated cluster - 13 contracts between +// 95x and 105x, two between 85x and 95x, and NOTHING between 105x and 200x - which is why the band is +// narrow rather than a lowered multiplier. +// +// Both derive paths carry their own copy of the flag CASE (two in normalize-raw, three in +// refresh-slice), so the rule is exercised through the REAL scripts on a real SQLite database, once per +// path. A copy left behind fails here rather than in production. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const etlPaths = [ + ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], + ['refresh-slice', resolve(root, 'scripts/refresh-slice.sql')], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-stotinki-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** Estimate in BGN; the EUR estimate is this ÷ 1.95583. Contract values below are in BGN too. */ +const ESTIMATE_BGN = 195_583; // exactly 100_000 EUR, so the multiples below are easy to read +const AUTH_EIK = '000695114'; +const BIDDER_EIK = '831646048'; + +/** One tender + one contract per case, so each case gets its own procedure estimate. */ +function seed(dbPath: string, cases: { unp: string; valueBgn: number }[]): void { + const tenders = cases + .map( + (c) => + `('eop:tenders:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'public', ${ESTIMATE_BGN}, 'BGN')`, + ) + .join(',\n'); + const contracts = cases + .map( + (c) => + `('eop:contracts:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'C-${c.unp}', '2026-06-01', ${c.valueBgn}, 'BGN', '${BIDDER_EIK}', 'Тестов изпълнител')`, + ) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type, estimated_value, currency) + VALUES ${tenders}; + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, contractor_eik, contractor_name) + VALUES ${contracts};`, + ); +} + +interface Row { + id: string; + value_flag: string; + amount_eur: number | null; +} + +const flagsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT id, value_flag, ROUND(amount_eur) AS amount_eur FROM contracts`, + ); + const out = new Map(); + for (const r of rows) { + const unp = r.id.split(':')[2]!; + out.set(unp, r); + } + return out; +}; + +describe('стотинки band in the value_flag CASE', () => { + for (const [label, scriptPath] of etlPaths) { + it(`${label}: flags a value at ~100x the estimate and repairs it to the estimate`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // Exactly 100x: the signature of a value typed in стотинки. + { unp: 'UNP-X100', valueBgn: ESTIMATE_BGN * 100 }, + // Inside the band but not exactly 100x: the стотинки value of a contract whose own price came + // in slightly under the procedure estimate, so the ratio lands a little below the multiple. + { unp: 'UNP-X97', valueBgn: Math.round(ESTIMATE_BGN * 97) }, + ]); + readScript(dbPath, scriptPath); + + const flags = flagsByUnp(dbPath); + for (const unp of ['UNP-X100', 'UNP-X97']) { + expect(flags.get(unp)?.value_flag, `${unp} should be value_suspect`).toBe( + 'value_suspect', + ); + // Repaired to the procedure estimate (100 000 EUR), not served at 100x. + expect(flags.get(unp)?.amount_eur, `${unp} repaired amount`).toBe(100_000); + } + }); + }); + + it(`${label}: leaves ordinary overruns and the gap above the band alone`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // A real overrun: flagged 'review' since 2020, but the value is kept. + { unp: 'UNP-X20', valueBgn: ESTIMATE_BGN * 20 }, + // Just above the band. The corpus has nothing between 105x and 200x; if a case ever appears + // there it must NOT be silently repaired to the estimate. + { unp: 'UNP-X120', valueBgn: ESTIMATE_BGN * 120 }, + // Just below the band, where the two real 85x-95x contracts live. + { unp: 'UNP-X90', valueBgn: ESTIMATE_BGN * 90 }, + ]); + readScript(dbPath, scriptPath); + + const flags = flagsByUnp(dbPath); + for (const unp of ['UNP-X20', 'UNP-X120', 'UNP-X90']) { + expect(flags.get(unp)?.value_flag, `${unp} must not be value_suspect`).toBe('review'); + // Kept at face value, which is the whole point of 'review' as opposed to 'value_suspect'. + expect(flags.get(unp)?.amount_eur, `${unp} keeps its value`).toBeGreaterThan(1_000_000); + } + }); + }); + + it(`${label}: still catches the plain over-200x case it always did`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [{ unp: 'UNP-X500', valueBgn: ESTIMATE_BGN * 500 }]); + readScript(dbPath, scriptPath); + const flags = flagsByUnp(dbPath); + expect(flags.get('UNP-X500')?.value_flag).toBe('value_suspect'); + expect(flags.get('UNP-X500')?.amount_eur).toBe(100_000); + }); + }); + } +}); diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 68c3dbf5..1313b7e5 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -694,7 +694,8 @@ SET ownership_kind = ( -- 5) Contracts — awarded lines (1:1 with staging rows), linked to tender + winning bidder, -- with the data-quality verdict (see 0007_data_quality.sql): --- value_flag = 'value_suspect' effective value >2bn EUR, or >200× the procedure estimate +-- value_flag = 'value_suspect' effective value >2bn EUR, >200× the procedure estimate, or +-- inside the 95×–105× стотинки band (a dropped decimal point) -- when that estimate is at least 1000 EUR — repaired to the -- procedure estimate for sums/display -- | 'value_low' zero/negative, OR a tiny signed value (< 1000 EUR) that is also @@ -879,7 +880,11 @@ FROM ( CASE -- Over-valuation + absurd are checked FIRST and repaired to the procedure estimate. -- value_low is a labelled-but-counted flag (see the amount_eur CASE). - WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND c.eff_eur > 200 * c.proc_est_eur) THEN 'value_suspect' + WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND (c.eff_eur > 200 * c.proc_est_eur + -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly + -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with + -- nothing between 105x and 200x, so the band is narrow on purpose. + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) THEN 'value_suspect' -- value_low: zero/negative, OR a tiny signed value (< 1000 EUR) that is also < 5% of the -- estimate. Large legitimate framework call-offs (small share of a huge ceiling but big in -- absolute terms) are NOT caught — the < 1000 EUR floor keeps them OUT of value_low. @@ -1210,7 +1215,11 @@ SELECT 1, SELECT c.*, CASE -- Mirrors the main derive CASE above; value_low is checked AFTER over-valuation + absurd. - WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND c.eff_eur > 200 * c.proc_est_eur) THEN 'value_suspect' + WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND (c.eff_eur > 200 * c.proc_est_eur + -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly + -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with + -- nothing between 105x and 200x, so the band is narrow on purpose. + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) THEN 'value_suspect' WHEN COALESCE(c.current_value, c.signing_value) <= 0 THEN 'value_low' WHEN c.estimated_value > 0 AND c.signing_value IS NOT NULL AND ( CASE diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 247b321d..66f54b29 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -1122,7 +1122,11 @@ FROM ( -- Over-valuation + absurd FIRST and repaired to the procedure estimate. -- value_low is labelled-but-counted (see the amount_eur CASE). Keep in sync with -- normalize-raw.sql and the EOP block below. - WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND c.eff_eur > 200 * c.proc_est_eur) THEN 'value_suspect' + WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND (c.eff_eur > 200 * c.proc_est_eur + -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly + -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with + -- nothing between 105x and 200x, so the band is narrow on purpose. + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) THEN 'value_suspect' -- value_low: zero/negative, OR a tiny signed value (< 1000 EUR) that is also < 5% of the -- estimate. The < 1000 EUR floor keeps large legitimate framework call-offs OUT. WHEN COALESCE(c.current_value, c.signing_value) <= 0 THEN 'value_low' @@ -1402,7 +1406,11 @@ FROM ( -- Over-valuation + absurd FIRST and repaired to the procedure estimate. -- value_low is labelled-but-counted (see the amount_eur CASE). Keep in sync with -- normalize-raw.sql and the OCDS block above. - WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND c.eff_eur > 200 * c.proc_est_eur) THEN 'value_suspect' + WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND (c.eff_eur > 200 * c.proc_est_eur + -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly + -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with + -- nothing between 105x and 200x, so the band is narrow on purpose. + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) THEN 'value_suspect' -- value_low: zero/negative, OR a tiny signed value (< 1000 EUR) that is also < 5% of the -- estimate. The < 1000 EUR floor keeps large legitimate framework call-offs OUT. WHEN COALESCE(c.current_value, c.signing_value) <= 0 THEN 'value_low' @@ -1728,7 +1736,11 @@ WITH contract_base AS ( WHEN c.value_flag <> 'annex_suspect' AND NOT (c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100))) THEN c.value_flag - WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND c.eff_eur > 200 * c.proc_est_eur) THEN 'value_suspect' + WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND (c.eff_eur > 200 * c.proc_est_eur + -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly + -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with + -- nothing between 105x and 200x, so the band is narrow on purpose. + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) THEN 'value_suspect' WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100)) THEN 'annex_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' From 426e2ef3ebc302eae591a304fe9091780572f74a Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Wed, 12 Aug 2026 21:24:13 +0000 Subject: [PATCH 58/69] =?UTF-8?q?fix(etl):=20=D1=85=D0=B2=D0=B0=D1=89?= =?UTF-8?q?=D0=B0=D0=B9=20=D1=81=D0=B1=D1=8A=D1=80=D0=BA=D0=B0=D0=BD=D0=BE?= =?UTF-8?q?=20=D1=87=D0=B8=D1=81=D0=BB=D0=BE=20=D0=B2=20=D0=B0=D0=BD=D0=B5?= =?UTF-8?q?=D0=BA=D1=81=20(=D1=81=D1=82=D1=8A=D0=BF=D0=BA=D0=B0=20>=3D10x?= =?UTF-8?q?=20=D0=BF=D1=80=D0=B8=20=D1=81=D0=B1=D0=BE=D1=80=20>=3D5x)=20(#?= =?UTF-8?q?299)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit annex_suspect искаше сборът текуща/подписана да стигне 100x, тъй че анекс със сбъркано число, издул договора до 30-50x подписаната стойност, се сервираше на лице. Новото правило е съюз: единична анексна стъпка >=10x И сбор >=5x. Двете половини са премерени поотделно на staging: - само стъпката не стига: 3 договора имат стъпка до 36 058x, но по-късен анекс сваля грешката и сборът свършва ПОД подписаната стойност - флаг там би ВДИГНАЛ показваната сума; - само сборът не стига: 40 договора на 5-100x се катерят бавно, без нито една подозрителна стъпка - остават както са (16,8 млн. EUR). Ефект: 43 договора, 184,6 млн. EUR показвани, 2,9 млн. след връщането към подписаната стойност. Най-големият: 105,2 млн. EUR при 49x сбор и 48x стъпка. Правилото влиза в петте копия на флаговата CASE и в охраната на reconciliation паса на refresh-slice; без охраната договор, чийто анекс пристига в по-късен прозорец, оставаше 'ok' завинаги - има тест точно за този сценарий. По въпроса от прегледа за приоритета между флаговете: когато договор е едновременно в лентата 95-105x И носи сбъркан анекс, value_suspect печели и repair-ва до процедурната прогноза, вместо annex_suspect да падне към подписаната. Премерено - точно един договор на целия корпус пали и двете правила (00747-2024-0003): подписаната му стойност е 1 093 863 EUR, а прогнозата 1 096 593 EUR, тоест двете котви се разминават с 0,25%. По принцип подписаната е по-добрата котва, когато виновникът е анексът, защото е записано число за този договор, а не оценка на ниво процедура; при N=1 и такава разлика пренареждането на CASE-а не се оправдава. Затваря #248. --- docs/core-scope.md | 2 +- docs/etl.md | 6 +- .../db/src/value-flag-annex-step-sql.test.ts | 325 ++++++++++++++++++ scripts/normalize-raw.sql | 24 +- scripts/refresh-slice.sql | 39 ++- 5 files changed, 385 insertions(+), 11 deletions(-) create mode 100644 packages/db/src/value-flag-annex-step-sql.test.ts diff --git a/docs/core-scope.md b/docs/core-scope.md index ccf892fa..b1edd55d 100644 --- a/docs/core-scope.md +++ b/docs/core-scope.md @@ -58,7 +58,7 @@ | `review` | ≥10× процедурната оценка — задържан, но маркиран. | реалната EUR стойност | | `value_low` | Нула/отрицателна, или дребна (<1000 EUR **и** <5% от оценката) — задържан, но маркиран. | реалната EUR стойност | | `value_suspect` | >2 млрд. EUR, или >200× оценката (при оценка ≥1000 EUR) — **repair-ва се до процедурната оценка**. | процедурната оценка в EUR | -| `annex_suspect` | Анекс е вдигнал стойността ≥100× или до отрицателна — пада към `signing`/`current`. | `signing`/`current` в EUR | +| `annex_suspect` | Анекс е вдигнал стойността ≥100× или до отрицателна; или стъпка ≥10× при сбор ≥5× — пада към `signing`/`current`. | `signing`/`current` в EUR | > **Кога `amount_eur` е `NULL`** (и редът е изключен от сумите): когато няма надеждна EUR стойност — > чуждовалутен ред без покрит ECB курс (липсва latest-prior rate до 10 дни преди подписването), diff --git a/docs/etl.md b/docs/etl.md index 318ce288..b3fbf50b 100644 --- a/docs/etl.md +++ b/docs/etl.md @@ -266,8 +266,10 @@ CASE-ът се оценява по ред; печели първото съвп големите легитимни рамкови call-off-и (малък дял от огромен таван, но голям в абсолютна стойност) извън този флаг. 3. **`annex_suspect`** — анекс е тласнал `current_value` до отрицателна или до `>= 100x` подписаната - стойност, докато подписаната е разумна. Договорът **се връща към подписаната стойност** за - `amount_eur`; надутата текуща стойност се потиска. + стойност, докато подписаната е разумна; или единична анексна стъпка е скочила `>= 10x`, а сборът е + свършил `>= 5x` над подписаната (сбъркано число в анекс — само стъпката не стига, защото има + вериги, при които по-късен анекс сваля грешката обратно под подписаната стойност). Договорът + **се връща към подписаната стойност** за `amount_eur`; надутата текуща стойност се потиска. 4. **`review`** — надхвърляне в сива зона: `eff >= 10 * procEst`. Запазен, флагнат и **брои се по face value**. 5. **`ok`** — всичко останало, брои се по `eff`. diff --git a/packages/db/src/value-flag-annex-step-sql.test.ts b/packages/db/src/value-flag-annex-step-sql.test.ts new file mode 100644 index 00000000..ea9f91f5 --- /dev/null +++ b/packages/db/src/value-flag-annex-step-sql.test.ts @@ -0,0 +1,325 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +// The mis-keyed annex rule (#248): annex_suspect used to need the aggregate current/signing ratio to +// reach 100x, so a single annex with a mis-typed value that pushed a contract to, say, 30x its signed +// value was served at face value. The new rule is a CONJUNCTION: one annex step jumped >=10x AND the +// aggregate ended >=5x over signing. Both halves are load-bearing - the corpus has chains with a +// 36,058x single step whose aggregate ends BELOW signing (a later annex corrects the typo), where +// flagging would RAISE the shown value, and slow legitimate chains that double a few times without any +// single suspicious step. +// +// The flag CASE lives in five copies across the two derive paths, plus a re-flag guard in the +// refresh-slice reconciliation pass, so the rule is exercised through the REAL scripts (in their real +// pipeline order) on a real SQLite database. A copy left behind in normalize-raw, or in the +// reconciliation pass, fails here rather than in production. The two refresh-slice INSERT-time copies +// are the one thing these tests CANNOT pin in isolation: refresh-slice promotes the window's +// amendments itself and its reconciliation pass then re-flags every amended contract, so a stale +// INSERT copy is always superseded in-script and never observable in the end state. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const derivePath = resolve(root, 'scripts/derive-amendments.sql'); +const promotePath = resolve(root, 'scripts/promote-amendments.sql'); +// Real pipeline order (scripts/import.mjs): the full derive runs derive-amendments before +// normalize-raw; the slice derive runs derive-amendments, and promote-amendments has populated the +// served amendments table that refresh-slice's reconciliation pass re-rolls current_value from. +const etlRuns = [ + ['normalize-raw', [derivePath, resolve(root, 'scripts/normalize-raw.sql')]], + ['refresh-slice', [derivePath, promotePath, resolve(root, 'scripts/refresh-slice.sql')]], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-annexstep-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** Signing value in BGN; exactly 100_000 EUR so the multiples below read directly. The procedure + * estimate is the same, which keeps every case away from the estimate-driven flags (the стотинки + * band sits at 95x-105x the estimate; the largest aggregate used here is 30x). */ +const SIGNING_BGN = 195_583; +const AUTH_EIK = '000695114'; +const BIDDER_EIK = '831646048'; + +interface AmendmentStep { + before: number | null; + after: number; + publishedAt: string; +} + +interface Case { + unp: string; + steps: AmendmentStep[]; +} + +function seedContracts(dbPath: string, cases: { unp: string }[]): void { + const tenders = cases + .map( + (c) => + `('eop:tenders:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'public', ${SIGNING_BGN}, 'BGN')`, + ) + .join(',\n'); + const contracts = cases + .map( + (c) => + `('eop:contracts:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'C-${c.unp}', '2026-06-01', ${SIGNING_BGN}, 'BGN', '${BIDDER_EIK}', 'Тестов изпълнител')`, + ) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type, estimated_value, currency) + VALUES ${tenders}; + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, contractor_eik, contractor_name) + VALUES ${contracts};`, + ); +} + +function seedAmendments(dbPath: string, cases: Case[]): void { + const rows = cases + .flatMap((c) => + c.steps.map( + (s, i) => + `('eop:annexes:${c.unp}', '2026-06-01T00:00:00Z', 'A-${c.unp}-${i + 1}', '${c.unp}', 'C-${c.unp}', '${s.publishedAt}', ${s.before ?? 'NULL'}, ${s.after}, 'BGN')`, + ), + ) + .join(',\n'); + if (!rows) return; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, fetched_at, document_number, unp, contract_number, published_at, + value_before, value_after, currency) + VALUES ${rows};`, + ); +} + +interface Row { + id: string; + value_flag: string; + amount_eur: number | null; +} + +const flagsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT id, value_flag, ROUND(amount_eur) AS amount_eur FROM contracts`, + ); + const out = new Map(); + for (const r of rows) { + const unp = r.id.split(':')[2]!; + out.set(unp, r); + } + return out; +}; + +describe('mis-keyed annex step in the value_flag CASE', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags one big annex step whose aggregate stays inflated, and repairs to signing`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // A single mis-typed annex: 30x in one step, nothing corrects it. Far below the 100x the + // old rule needed. + { + unp: 'UNP-STEP30', + steps: [{ before: SIGNING_BGN, after: SIGNING_BGN * 30, publishedAt: '2026-06-10' }], + }, + // Both thresholds exactly at the boundary: step exactly 10x, aggregate exactly 5x. + { + unp: 'UNP-EDGE', + steps: [{ before: SIGNING_BGN / 2, after: SIGNING_BGN * 5, publishedAt: '2026-06-10' }], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const flags = flagsByUnp(dbPath); + for (const unp of ['UNP-STEP30', 'UNP-EDGE']) { + expect(flags.get(unp)?.value_flag, `${unp} should be annex_suspect`).toBe( + 'annex_suspect', + ); + // Fell back to the signed value (100 000 EUR), not served at the inflated current value. + expect(flags.get(unp)?.amount_eur, `${unp} repaired amount`).toBe(100_000); + } + }); + }); + + it(`${label}: never flags a chain whose later annex corrects the typo back down`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // The corpus counter-example class: a huge step UP, then a correction that lands the + // aggregate BELOW signing. Flagging would replace 50 000 EUR with 100 000 EUR - the + // repair itself would inflate the contract. + { + unp: 'UNP-CORRECTED', + steps: [ + { before: 5, after: SIGNING_BGN, publishedAt: '2026-06-10' }, + { before: SIGNING_BGN, after: SIGNING_BGN / 2, publishedAt: '2026-06-20' }, + ], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const flags = flagsByUnp(dbPath); + expect(flags.get('UNP-CORRECTED')?.value_flag).toBe('ok'); + // Served at the corrected current value - half the signing value. + expect(flags.get('UNP-CORRECTED')?.amount_eur).toBe(50_000); + }); + }); + + it(`${label}: needs BOTH halves - a big step under 5x aggregate, and a slow climb without one`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // Step 49x, but the aggregate stops at 4.9x signing - under the 5x floor. + { + unp: 'UNP-AGG49', + steps: [ + { before: SIGNING_BGN / 10, after: SIGNING_BGN * 4.9, publishedAt: '2026-06-10' }, + ], + }, + // Aggregate 8x through three doublings - no single step comes near 10x. + { + unp: 'UNP-SLOW8', + steps: [ + { before: SIGNING_BGN, after: SIGNING_BGN * 2, publishedAt: '2026-06-10' }, + { before: SIGNING_BGN * 2, after: SIGNING_BGN * 4, publishedAt: '2026-06-20' }, + { before: SIGNING_BGN * 4, after: SIGNING_BGN * 8, publishedAt: '2026-06-30' }, + ], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const flags = flagsByUnp(dbPath); + for (const unp of ['UNP-AGG49', 'UNP-SLOW8']) { + expect(flags.get(unp)?.value_flag, `${unp} must not be annex_suspect`).toBe('ok'); + } + // Both keep their as-recorded current value. + expect(flags.get('UNP-AGG49')?.amount_eur).toBe(490_000); + expect(flags.get('UNP-SLOW8')?.amount_eur).toBe(800_000); + }); + }); + + it(`${label}: the plain >=100x aggregate still flags even with no usable step`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // value_before missing on the only annex, so the step half can't fire - the original + // aggregate rule must still catch the 150x blow-up on its own. + { + unp: 'UNP-AGG150', + steps: [{ before: null, after: SIGNING_BGN * 150, publishedAt: '2026-06-10' }], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const flags = flagsByUnp(dbPath); + expect(flags.get('UNP-AGG150')?.value_flag).toBe('annex_suspect'); + expect(flags.get('UNP-AGG150')?.amount_eur).toBe(100_000); + }); + }); + } + + it('refresh-slice: the CLI slice path (no external promote) lands the same flags', () => { + // scripts/import.mjs runSliceDerive runs derive-amendments + refresh-slice WITHOUT + // promote-amendments. That still works because refresh-slice promotes the window's amendments + // itself (INSERT OR REPLACE INTO amendments) before its reconciliation pass, which is therefore + // authoritative for annex flags on this path too - this run pins exactly that pipeline shape. + withEtlDb('no-promote', (dbPath) => { + const cases: Case[] = [ + { + unp: 'UNP-STEP30', + steps: [{ before: SIGNING_BGN, after: SIGNING_BGN * 30, publishedAt: '2026-06-10' }], + }, + { + unp: 'UNP-CORRECTED', + steps: [ + { before: 5, after: SIGNING_BGN, publishedAt: '2026-06-10' }, + { before: SIGNING_BGN, after: SIGNING_BGN / 2, publishedAt: '2026-06-20' }, + ], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of [derivePath, resolve(root, 'scripts/refresh-slice.sql')]) + readScript(dbPath, p); + + const flags = flagsByUnp(dbPath); + expect(flags.get('UNP-STEP30')?.value_flag).toBe('annex_suspect'); + expect(flags.get('UNP-STEP30')?.amount_eur).toBe(100_000); + expect(flags.get('UNP-CORRECTED')?.value_flag).toBe('ok'); + expect(flags.get('UNP-CORRECTED')?.amount_eur).toBe(50_000); + }); + }); + + it('refresh-slice: re-flags a served contract when the annex arrives in a LATER window', () => { + // The production shape of this bug class: the contract clears one refresh window as 'ok', the + // mis-keyed annex lands in the next. The reconciliation pass matches the contract through + // raw_amendments alone, re-rolls current_value from the served amendments table, and its + // re-flag guard must let the contract through to be re-classified - a guard still keyed to the + // old >=100x rule would keep it 'ok' forever. + const [, refreshScripts] = etlRuns[1]!; + withEtlDb('two-window', (dbPath) => { + const cases: Case[] = [ + { + unp: 'UNP-LATE', + steps: [{ before: SIGNING_BGN, after: SIGNING_BGN * 30, publishedAt: '2026-07-10' }], + }, + ]; + // Window 1: the contract alone. + seedContracts(dbPath, cases); + for (const p of refreshScripts) readScript(dbPath, p); + expect(flagsByUnp(dbPath).get('UNP-LATE')?.value_flag).toBe('ok'); + + // Window 2: only the annex is in the transient staging. + sqlite(dbPath, 'DELETE FROM raw_contracts; DELETE FROM raw_tenders;'); + seedAmendments(dbPath, cases); + for (const p of refreshScripts) readScript(dbPath, p); + + const row = flagsByUnp(dbPath).get('UNP-LATE'); + expect(row?.value_flag).toBe('annex_suspect'); + expect(row?.amount_eur).toBe(100_000); + }); + }); +}); diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 1313b7e5..964132da 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -703,7 +703,9 @@ SET ownership_kind = ( -- LABELLED — large legitimate framework call-offs (a small share of -- a huge ceiling but big in absolute terms) are excluded by the -- < 1000 EUR floor, so they keep counting and stay unflagged --- | 'annex_suspect' amendment pushed current_value ≥100× signing, or negative → +-- | 'annex_suspect' amendment pushed current_value ≥100× signing, or negative, or +-- a single annex step jumped ≥10× while the aggregate ended ≥5× +-- over signing (a mis-keyed annex value) → -- fall back to signing_value, or current_value if signing is -- missing, so the contract still counts -- | 'review' ≥10× the procedure estimate (kept, but flagged) @@ -932,7 +934,15 @@ FROM ( ) END ), 0) < 0.05 THEN 'value_low' - WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100)) THEN 'annex_suspect' + WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 + -- Mis-keyed annex: a single step jumped ≥10× AND the aggregate ended ≥5× over signing. + -- The step alone is NOT enough — some chains have a huge step that a later annex pulls + -- back below signing, and flagging those would RAISE the shown value, not repair it. + OR (c.current_value / c.signing_value >= 5 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_before > 0 AND am.value_after >= 10 * am.value_before + ))))) THEN 'annex_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1264,7 +1274,15 @@ SELECT 1, ) END ), 0) < 0.05 THEN 'value_low' - WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100)) THEN 'annex_suspect' + WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 + -- Mis-keyed annex: a single step jumped ≥10× AND the aggregate ended ≥5× over signing. + -- The step alone is NOT enough — some chains have a huge step that a later annex pulls + -- back below signing, and flagging those would RAISE the shown value, not repair it. + OR (c.current_value / c.signing_value >= 5 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_before > 0 AND am.value_after >= 10 * am.value_before + ))))) THEN 'annex_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 66f54b29..d74ab266 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -1173,7 +1173,15 @@ FROM ( ) END ), 0) < 0.05 THEN 'value_low' - WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100)) THEN 'annex_suspect' + WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 + -- Mis-keyed annex: a single step jumped ≥10× AND the aggregate ended ≥5× over signing. + -- The step alone is NOT enough — some chains have a huge step that a later annex pulls + -- back below signing, and flagging those would RAISE the shown value, not repair it. + OR (c.current_value / c.signing_value >= 5 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_before > 0 AND am.value_after >= 10 * am.value_before + ))))) THEN 'annex_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1457,7 +1465,15 @@ FROM ( ) END ), 0) < 0.05 THEN 'value_low' - WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100)) THEN 'annex_suspect' + WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 + -- Mis-keyed annex: a single step jumped ≥10× AND the aggregate ended ≥5× over signing. + -- The step alone is NOT enough — some chains have a huge step that a later annex pulls + -- back below signing, and flagging those would RAISE the shown value, not repair it. + OR (c.current_value / c.signing_value >= 5 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_before > 0 AND am.value_after >= 10 * am.value_before + ))))) THEN 'annex_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1710,7 +1726,15 @@ WITH contract_base AS ( ) ORDER BY rc.source DESC, rc.id DESC LIMIT 1 - ), te.estimated_value) AS classifier_estimated_value + ), te.estimated_value) AS classifier_estimated_value, + -- One annex step jumped ≥10× — half of the mis-keyed-annex conjunction below. Checked against + -- the CUMULATIVE domain amendments, matching where this pass re-rolls current_value from. + EXISTS ( + SELECT 1 FROM amendments am + WHERE am.unp = substr(c.tender_id, 3) + AND am.contract_number = c.contract_number + AND am.value_before > 0 AND am.value_after >= 10 * am.value_before + ) AS has_step10 FROM contracts c JOIN tenders te ON te.id = c.tender_id WHERE ( @@ -1734,14 +1758,19 @@ WITH contract_base AS ( SELECT id, currency, signing_value, current_value, current_value_currency, fx_rate, proc_est_eur, proc_est_native, CASE WHEN c.value_flag <> 'annex_suspect' - AND NOT (c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100))) + AND NOT (c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 + OR (c.current_value / c.signing_value >= 5 AND c.has_step10))))) THEN c.value_flag WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND (c.eff_eur > 200 * c.proc_est_eur -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with -- nothing between 105x and 200x, so the band is narrow on purpose. OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) THEN 'value_suspect' - WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100)) THEN 'annex_suspect' + WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 + -- Mis-keyed annex: a single step jumped ≥10× AND the aggregate ended ≥5× over signing. + -- The step alone is NOT enough — some chains have a huge step that a later annex pulls + -- back below signing, and flagging those would RAISE the shown value, not repair it. + OR (c.current_value / c.signing_value >= 5 AND c.has_step10)))) THEN 'annex_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS new_value_flag From ac15498e73e59bb07a4f501b9eee51a9f5b35c0f Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 13 Aug 2026 19:54:35 +0300 Subject: [PATCH 59/69] test: apply migration 0006 wherever the merged suites build a schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge with upstream/main was textually clean and semantically broken — `git merge-tree` reported no conflict, and four suites then failed. Both sides had edited refresh-slice.sql and ship-related-persons.mjs, and upstream's new harnesses build their schemas from migrations that stop at 0003, while this branch made both files depend on interest_link_evidence (0006). - refresh-slice.test.ts: upstream's new OCDS-bridge site (#286) was a third setup site this branch had not seen; without 0006 sqlite3 aborts on "no such table" at refresh-slice.sql:1957, before reaching the logic under test. - value-flag-stotinki-sql.test.ts / value-flag-annex-step-sql.test.ts: new files, same cause — they died on the свързани-лица block while testing the value_flag CASE. - ship-e2e.test.ts: the evidence seal is now a shipped table, so a target built from 0003 alone makes the generated wipe abort before a row moves. Its corpus now seeds one seal per link, which is the only shape the ship should ever carry — a published link without one is what the audit fails on and what the read gate refuses to surface. One behaviour change worth naming. `a request that never landed fails the run` injected a skipped interest_links chunk; with seals shipping, that now trips the FOREIGN KEY as the orphaned rows land, before the read-back runs. Stronger, but it leaves the read-back gate — the thing the test exists for — unexercised. The injection moves to a seal chunk, which has no dependents, so the shortfall stays invisible until the counts are compared. All suites green on the merged tree: 421 db, 492 web, 61 ingest, 45 shared, 20 etl, 10 config, 91 root scripts, 248 pipeline; lint, typecheck, check:docs and the coverage ratchet clean. --- packages/db/src/refresh-slice.test.ts | 3 +++ .../db/src/value-flag-annex-step-sql.test.ts | 5 ++++ .../db/src/value-flag-stotinki-sql.test.ts | 5 ++++ scripts/ship-e2e.test.mjs | 26 +++++++++++++++---- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 58c434d2..9c76fb83 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -662,6 +662,9 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + // 0006 too: refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279), so the + // script cannot parse against a DB that stops at 0003 — every site here applies both. + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); // An EOP procedure (tender + base contract) with УНП UNP-SLICE / tender.id TENDER-SLICE. An diff --git a/packages/db/src/value-flag-annex-step-sql.test.ts b/packages/db/src/value-flag-annex-step-sql.test.ts index ea9f91f5..c2afb684 100644 --- a/packages/db/src/value-flag-annex-step-sql.test.ts +++ b/packages/db/src/value-flag-annex-step-sql.test.ts @@ -26,6 +26,10 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script +// only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it +// ever reaches the value_flag CASE these tests are about. +const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const promotePath = resolve(root, 'scripts/promote-amendments.sql'); @@ -60,6 +64,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/value-flag-stotinki-sql.test.ts b/packages/db/src/value-flag-stotinki-sql.test.ts index 2fc97624..eb41d860 100644 --- a/packages/db/src/value-flag-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-stotinki-sql.test.ts @@ -20,6 +20,10 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script +// only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it +// ever reaches the value_flag CASE these tests are about. +const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -49,6 +53,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/scripts/ship-e2e.test.mjs b/scripts/ship-e2e.test.mjs index cf971a2e..60d6d67e 100644 --- a/scripts/ship-e2e.test.mjs +++ b/scripts/ship-e2e.test.mjs @@ -28,6 +28,10 @@ const HERE = dirname(fileURLToPath(import.meta.url)); const ROOT = resolve(HERE, '..'); const SCRIPT = resolve(HERE, 'ship-related-persons.mjs'); const MIG = resolve(ROOT, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// 0006 too: the evidence seal is one of the shipped tables (#279, ADR-0033), so it appears in the ship's +// TABLES and WIPE_ORDER. A target built from 0003 alone makes the generated wipe abort on „no such table" +// before a single row moves — the schema under test has to be the schema the ship writes. +const MIG_EVIDENCE = resolve(ROOT, 'packages/db/migrations/0006_interest_link_evidence.sql'); const D1_NAME = 'sigma-test-local'; // Derived from BOTH production constants, and the run below does NOT override either: one row past @@ -51,11 +55,12 @@ if (spawnSync('sqlite3', ['-version'], { stdio: 'ignore' }).error) { throw new Error('scripts/ship-e2e.test.mjs requires the sqlite3 binary on PATH'); } -/** The five served tables plus the two FK parents they reference. */ +/** The served свързани-лица tables plus the two FK parents they reference. */ const SCHEMA = `PRAGMA foreign_keys=ON; CREATE TABLE bidders(id TEXT PRIMARY KEY); CREATE TABLE authorities(id TEXT PRIMARY KEY); .read ${MIG} +.read ${MIG_EVIDENCE} INSERT INTO bidders(id) VALUES('eik:1'); INSERT INTO authorities(id) VALUES('auth:1');`; @@ -66,7 +71,11 @@ INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VAL ${Array.from( { length: links }, (_, i) => - `INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('il${i}','p1|${i}','p1','eik:1','1','e','v1','B_distinctive','owns','published');`, + `INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('il${i}','p1|${i}','p1','eik:1','1','e','v1','B_distinctive','owns','published');\n` + + // One seal per link. Since #279 a published link without one is exactly the state the audit fails + // the run on (C_no_evidence) and the read gate refuses to surface, so a corpus without seals is not + // a shape the ship should ever be asked to carry. + `INSERT INTO interest_link_evidence(link_key,evidence_kind,lookup_date,rules_version,live_status) VALUES('p1|${i}','document','2026-08-13','tr-rules-1','live');`, ).join('\n')} `; // interest_link_authorities is deliberately left EMPTY above: its expected count is 0, which is the @@ -77,13 +86,15 @@ INSERT INTO persons(id,name) VALUES('stale','Стар запис'); INSERT INTO declarations(id,person_id,xml_file,folder_year,template,source_url) VALUES('sd','stale','s.xml','2019','assets','u'); INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VALUES('sdi','sd','S','s','shares'); INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('sil','stale|0','stale','eik:1','1','s','v0','B_distinctive','owns','published'); -INSERT INTO interest_link_authorities(link_key,authority_id,authority_name) VALUES('stale|0','auth:1','A');`; +INSERT INTO interest_link_authorities(link_key,authority_id,authority_name) VALUES('stale|0','auth:1','A'); +INSERT INTO interest_link_evidence(link_key,evidence_kind,lookup_date,rules_version,live_status) VALUES('stale|0','confirmed','2019-01-01','tr-rules-0','live');`; const EXPECTED_ROWS = { persons: 1, declarations: 1, declared_interests: 1, interest_links: LINKS, + interest_link_evidence: LINKS, // one seal per link (#279) interest_link_authorities: 0, }; @@ -255,10 +266,15 @@ test('a request that never landed fails the run', (t) => { const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-short-')); t.after(() => rmSync(dir, { recursive: true, force: true })); - const { res } = runShip(dir, { env: { SHIP_FAKE_SKIP: 'interest_links.2.sql' } }); + // The skip targets a LEAF table on purpose. Since #279 the ship also carries interest_link_evidence, + // which has an FK to interest_links — so dropping an interest_links chunk now fails on the FOREIGN KEY + // as the orphaned seals land, before the read-back ever runs. That is a stronger guard, but it would + // leave the read-back gate itself unexercised, which is what this test is for. A seal chunk has no + // dependents, so its loss is invisible until the counts are compared. + const { res } = runShip(dir, { env: { SHIP_FAKE_SKIP: 'interest_link_evidence.2.sql' } }); assert.notEqual(res.status, 0, 'a short target must fail the run'); assert.match(res.stderr, /ship verification FAILED/); - assert.match(res.stderr, /interest_links: shipped \d+, target has \d+/); + assert.match(res.stderr, /interest_link_evidence: shipped \d+, target has \d+/); }); test('a read-back that answers with a non-number fails closed', (t) => { From 24c80016550e90a8c25475c825e194dc1cd35c93 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 14 Aug 2026 09:49:41 +0300 Subject: [PATCH 60/69] fix(db): leave migration 0003 untouched and let 0007 enforce for every database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0003 is already applied everywhere, so adding CHECKs to it was the wrong move even though that is where the table is declared. `CREATE TABLE IF NOT EXISTS` is a no-op against an existing table and ship-related-persons wipes ROWS, never definitions — so the constraints would have existed only on databases built after the edit. Two schemas under one name, with the gate absent precisely on the served database, which is where a hand-run `UPDATE … SET status='published '` during an incident actually lands. The comment promised the opposite of what the deployed base would have had. 0003 is restored byte-for-byte and enforcement moves entirely into 0007, which both paths reach: a fresh chain runs it after 0003, a deployed database gets it as a retrofit. 0007 already carried the triggers, the natural-key index and the dedup, so nothing is lost — enforcement simply becomes uniform. A fresh build keeps 0003's table-level UNIQUE too; it treats NULLs as distinct, so it is strictly weaker than the index and rejects nothing the index accepts. 0006 keeps its own CHECKs: it is new here and has never been applied to a served environment, so declaring them there edits no applied history. The parity test is the point: a fresh 0000..0007 build and a legacy database retrofitted by 0007 must reject the SAME values and accept the SAME ones, including the UPDATE path no CHECK on a legacy table would ever have covered. Mutation-checked — dropping the UPDATE trigger fails it. Verified on a real local D1 built from the reverted 0003, not only in tests: the bad UPDATE returns SQLITE_CONSTRAINT_TRIGGER and the legitimate one succeeds. The dedup DELETE now announces itself before changing live data, so a deployment that drops declarations says how many instead of leaving a row count nobody recorded. Its subtraction is parenthesised because `||` binds tighter than `-` in SQLite — the first real apply printed "notice: -2" and proved it. No explicit transaction: `d1 execute --file` already runs the file as one, so a later failure rolls the DELETE back with it. --- .../0003_related_persons_foundation.sql | 33 +------ .../0007_publishing_gate_constraints.sql | 58 ++++++++---- packages/db/src/migrations.test.ts | 88 +++++++++++++++++++ 3 files changed, 133 insertions(+), 46 deletions(-) diff --git a/packages/db/migrations/0003_related_persons_foundation.sql b/packages/db/migrations/0003_related_persons_foundation.sql index 58a85878..6ad3be97 100644 --- a/packages/db/migrations/0003_related_persons_foundation.sql +++ b/packages/db/migrations/0003_related_persons_foundation.sql @@ -29,24 +29,9 @@ CREATE TABLE IF NOT EXISTS declarations ( category TEXT, -- CACBG category (e.g. Народни представители) institution TEXT, -- the body the official serves in position TEXT, -- declared position - source_url TEXT NOT NULL -- register.cacbg.bg// — provenance + source_url TEXT NOT NULL, -- register.cacbg.bg// — provenance + UNIQUE (xml_file, control_hash) ); --- Re-import idempotence (#279 §2). This REPLACES a table-level `UNIQUE (xml_file, control_hash)`, which --- did not constrain anything it was meant to: --- 1. SQLite counts NULLs as DISTINCT, and `control_hash` is genuinely optional at the source — the --- register omits on some declarations and parse.mjs carries that through as null. So --- every hashless declaration was mutually unique and re-imported as a fresh row each run, --- double-counting the stakes it carries. COALESCE folds them to one value. --- 2. `xml_file` is not unique ACROSS FOLDERS — the register splits years into suffixed folders and --- reuses basenames — which load.mjs already knows, since it namespaces the declaration id by folder --- for exactly this reason. Without folder_year here the two keys disagree, and the narrower one --- would reject a second official's genuinely different declaration. --- --- `control_hash` deliberately stays NULLABLE. NOT NULL was the other candidate, but it converts an --- optional source field into a run-stopping loader failure, and fabricating a hash to satisfy it would --- assert an integrity check we never performed. -CREATE UNIQUE INDEX IF NOT EXISTS idx_declarations_natural_key - ON declarations(xml_file, folder_year, COALESCE(control_hash, '')); -- Per-person lookup: audit.mjs scans a person's declarations, and LINK_SELECT's source_url subquery -- filters declarations by person_id — without this it's a full scan of the table. CREATE INDEX IF NOT EXISTS idx_declarations_person ON declarations(person_id); @@ -90,12 +75,7 @@ CREATE TABLE IF NOT EXISTS interest_links ( -- management_role — self, relation manages, a single declarant (ambiguous: private manager or small board) -- Materiality: only CLOSELY-HELD forms (ООД/ЕООД/ЕТ/…) reach ownership classes; listed АД/ЕАД securities -- and management-only roles never become private_ownership/family_ownership (ADR-0022). - -- fail-closed: a NON-surfaced class, so a writer that sets status but forgets the class can't leak to - -- the public surface (load.mjs always sets it explicitly). CHECKed for the same reason as `status`: - -- the surface reads this as an enum, so a typo'd class is a gate failure, not a cosmetic one. - interest_class TEXT NOT NULL DEFAULT 'management_role' - CHECK (interest_class IN ('private_ownership','family_ownership', - 'ex_officio_board','management_role')), + interest_class TEXT NOT NULL DEFAULT 'management_role', -- fail-closed: a NON-surfaced class, so a writer that sets status but forgets the class can't leak to the public surface (load.mjs always sets it explicitly) contemporaneous INTEGER NOT NULL DEFAULT 0, own_institution TEXT NOT NULL DEFAULT 'none', -- exact (deterministic) | locality (heuristic) | none evidence_count INTEGER NOT NULL DEFAULT 1, -- # declared_interests supporting this link @@ -107,12 +87,7 @@ CREATE TABLE IF NOT EXISTS interest_links ( contract_value_eur REAL, -- SUM(contracts.amount_eur); NULL if none summable first_contract_year TEXT, last_contract_year TEXT, - -- CHECKed, not merely documented (#279 §2). The public surface is `status = 'published'`, so a value - -- that only LOOKS like one — 'published ' with a trailing space is the case the ticket names — passes - -- every writer and then fails the gate silently, or slips a LIKE. A CHECK binds every writer at once, - -- including a hand-run UPDATE during an incident, which is exactly when this is most likely typed. - status TEXT NOT NULL DEFAULT 'held' - CHECK (status IN ('published','internal','held','withdrawn','suppressed')), + status TEXT NOT NULL DEFAULT 'held', -- published (public surface: private/family ownership) | internal (non-surfaced class) | held | withdrawn | suppressed verified_by TEXT, verified_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')) diff --git a/packages/db/migrations/0007_publishing_gate_constraints.sql b/packages/db/migrations/0007_publishing_gate_constraints.sql index 7720ac45..b3e9d4cf 100644 --- a/packages/db/migrations/0007_publishing_gate_constraints.sql +++ b/packages/db/migrations/0007_publishing_gate_constraints.sql @@ -1,14 +1,17 @@ --- Retrofit the publishing-gate constraints onto an ALREADY-DEPLOYED database (#279 §2). +-- The publishing-gate constraints, for EVERY database — freshly built and already-deployed alike (#279 §2). -- --- WHY THIS EXISTS AT ALL, given 0003 and 0006 now declare the same constraints: --- `CREATE TABLE IF NOT EXISTS` is a no-op against a table that already exists, and --- `scripts/ship-related-persons.mjs` wipes ROWS (`DELETE FROM`, WIPE_ORDER) and never table --- definitions. So every environment that applied 0003 before today keeps the unconstrained shape --- forever, no matter how many times the data is reloaded. Only a rebuild can change it. +-- WHY THIS MIGRATION OWNS THEM RATHER THAN 0003 (todorkolev, #309). The obvious move is to add the CHECKs +-- to `interest_links` in 0003, where the table is declared. That does not work and is worse than not +-- trying: 0003 is ALREADY APPLIED everywhere, `CREATE TABLE IF NOT EXISTS` is a no-op against an existing +-- table, and `ship-related-persons.mjs` wipes ROWS (`DELETE FROM`, WIPE_ORDER), never definitions. So an +-- in-place CHECK would exist only on databases built after the edit — putting two different schemas under +-- one name, with the constraint absent precisely on the served database. That is where a hand-run +-- `UPDATE … SET status='published '` during an incident actually lands. -- --- WHY IT IS SAFE TO RE-RUN: migrations here are applied by a bare `wrangler d1 execute --file` with no --- applied-migrations tracking (see related-persons-data.yml / deploy.yml), so re-application MUST be a --- no-op. Everything below is `IF NOT EXISTS` over statements that converge. +-- So 0003 stays byte-identical to what was applied, and enforcement lives here, reached by both paths: +-- a fresh chain runs 0007 after 0003, and a deployed database gets it as a retrofit. One shape, one +-- mechanism. `packages/db/src/migrations.test.ts` holds the two shapes to the same rejections and the same +-- acceptances, so this stays true rather than merely intended. -- -- WHY TRIGGERS AND NOT A TABLE REBUILD: SQLite cannot add a CHECK in place, so the textbook route is -- create-copy-drop-rename. That route is WRONG here, and a real `wrangler d1 migrations apply` proved it: @@ -18,10 +21,14 @@ -- child first would work mechanically but would strip every evidence seal, and since the read gate -- REQUIRES a seal that empties the public surface until the next monthly data run. -- --- A BEFORE INSERT/UPDATE trigger that RAISEs enforces the identical invariant for every writer — --- including a hand-run UPDATE during an incident, which is when a bad value is most likely typed — with --- no rebuild, no FK exposure and no seal loss. Freshly created databases still get true CHECK constraints --- from 0003/0006; this is the retrofit path for databases that already exist. +-- A BEFORE INSERT/UPDATE trigger that RAISEs enforces the identical invariant for every writer — including +-- the hand-run UPDATE above, which no CHECK on a legacy table would ever have covered — with no rebuild, +-- no FK exposure and no seal loss. 0006 keeps its own CHECKs: it is NEW in this change, has never been +-- applied to a served environment, so declaring them there edits no applied history. +-- +-- WHY IT IS SAFE TO RE-RUN: migrations here are applied by a bare `wrangler d1 execute --file` with no +-- applied-migrations tracking (see related-persons-data.yml / deploy.yml), so re-application MUST be a +-- no-op. Everything below is `IF NOT EXISTS` over statements that converge. -- ── declarations ──────────────────────────────────────────────────────────────────────────────────── -- The table-level `UNIQUE (xml_file, control_hash)` did not constrain what it was written for. SQLite @@ -36,10 +43,27 @@ -- optional source field into a run-stopping loader failure, and a fabricated placeholder hash would -- assert an integrity check nobody performed. -- --- No table rebuild is needed for this one: dropping the old constraint means dropping the table it lives --- on, but the same effect is had by creating the correct index — and the stale UNIQUE, being strictly --- weaker, rejects nothing the new index accepts. Duplicates ALREADY stored under the old non-constraint --- would block the new index, so they are collapsed first, keeping the earliest row per natural key. +-- No table rebuild is needed for this one: the table-level `UNIQUE (xml_file, control_hash)` declared in +-- 0003 STAYS (0003 is not edited — see the header), and the correct index is simply added alongside it. +-- The stale constraint is strictly WEAKER than the index — it treats NULLs as distinct, so it accepts a +-- superset — and therefore rejects nothing the index accepts. The index governs. +-- +-- Duplicates ALREADY stored under the old non-constraint would block the index, so they are collapsed +-- first, keeping the earliest row per natural key. +-- +-- The DELETE below changes live data, so it ANNOUNCES itself first (cefothe, #309): the count is emitted +-- before the rows go, and a run that removes nothing says so. Without it the only evidence a deployment +-- silently dropped declarations would be a row count nobody recorded beforehand. It needs no explicit +-- transaction — `wrangler d1 execute --file` runs the file as one implicit transaction, so a failure +-- anywhere below rolls the DELETE back with it rather than leaving a half-collapsed table. +-- The subtraction is parenthesised deliberately: `||` binds TIGHTER than `-` in SQLite, so without it +-- this reads as ('…' || countA) - (countB || '…') — two strings coerced to numbers — and reports a +-- meaningless figure instead of the row count. It did, before a real apply showed „notice: -2". +SELECT 'migration 0007: collapsing ' || + ((SELECT COUNT(*) FROM declarations) - + (SELECT COUNT(*) FROM (SELECT 1 FROM declarations + GROUP BY xml_file, folder_year, COALESCE(control_hash, '')))) || + ' duplicate declaration row(s) before the natural-key index' AS notice; DELETE FROM declarations WHERE id NOT IN ( SELECT MIN(id) FROM declarations GROUP BY xml_file, folder_year, COALESCE(control_hash, '') ); diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index bc5d9a22..25e5ef12 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -299,6 +299,94 @@ describe('served migrations', () => { } }); + // THE POINT OF 0007, stated as an executable claim rather than a comment (todorkolev, #309). + // + // 0003 is an ALREADY-APPLIED migration and is therefore never edited: `CREATE TABLE IF NOT EXISTS` is a + // no-op on a live database, so an in-place CHECK would exist only on freshly built ones. That would put + // two different schemas under one name — and the divergence would land precisely on the served database, + // which is where a hand-run `UPDATE status='published '` during an incident actually happens. + // + // So 0007 owns the enforcement for BOTH shapes, and this test is what keeps them one shape: a fresh + // 0000..0007 build and a legacy database retrofitted by 0007 must reject the SAME values and accept the + // SAME values. If either drifts, the parity assertions below fail rather than a reviewer noticing. + it('a fresh build and a retrofitted legacy database enforce the SAME gate', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-parity-')); + const fresh = resolve(dir, 'fresh.sqlite'); + const legacy = resolve(dir, 'legacy.sqlite'); + try { + // (1) fresh: the migration chain exactly as a new environment applies it. + for (const m of [migration0, migration3, migration6, migration7]) readScript(fresh, m); + + // (2) legacy: 0003 + 0006 as they were BEFORE this PR, then 0007 retrofits. `interest_links` is + // recreated without constraints because that is the shape every already-deployed database holds. + for (const m of [migration0, migration3, migration6]) readScript(legacy, m); + sqlite( + legacy, + `DROP TABLE interest_link_evidence; + DROP TABLE interest_links; + CREATE TABLE interest_links ( + id TEXT PRIMARY KEY, link_key TEXT NOT NULL UNIQUE, + person_id TEXT NOT NULL REFERENCES persons(id), bidder_id TEXT NOT NULL, eik TEXT NOT NULL, + entity_key TEXT NOT NULL, match_method TEXT, matcher_version TEXT NOT NULL, + publish_tier TEXT NOT NULL, relation TEXT NOT NULL, + interest_class TEXT NOT NULL DEFAULT 'management_role', + contemporaneous INTEGER NOT NULL DEFAULT 0, own_institution TEXT NOT NULL DEFAULT 'none', + evidence_count INTEGER NOT NULL DEFAULT 1, first_declared_year TEXT, last_declared_year TEXT, + contract_count INTEGER NOT NULL DEFAULT 0, contract_value_eur REAL, first_contract_year TEXT, + last_contract_year TEXT, status TEXT NOT NULL DEFAULT 'held', verified_by TEXT, + verified_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now'))); + CREATE TABLE interest_link_evidence ( + link_key TEXT PRIMARY KEY REFERENCES interest_links(link_key), evidence_kind TEXT NOT NULL, + registry_role TEXT, matched_fact TEXT, entry_number TEXT, entry_date TEXT, + lookup_date TEXT NOT NULL, rules_version TEXT NOT NULL, live_status TEXT NOT NULL, + sealed_at TEXT NOT NULL DEFAULT (datetime('now')));`, + ); + readScript(legacy, migration7); + + const link = (db: string, id: string, status: string, cls = 'private_ownership') => + sqlite( + db, + `INSERT INTO persons(id, name) VALUES ('person:a', 'А') ON CONFLICT DO NOTHING; + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:${id}', '${id}', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + '${cls}', '${status}');`, + ); + const seal = (db: string, key: string, kind: string) => + sqlite( + db, + `INSERT INTO interest_link_evidence + (link_key, evidence_kind, lookup_date, rules_version, live_status) + VALUES ('${key}', '${kind}', '2026-08-14', 'tr-rules-1', 'live');`, + ); + + for (const [label, db] of [ + ['fresh', fresh], + ['legacy', legacy], + ] as const) { + // REJECTED identically — 'published ' is #279 §2's named case, and it is the one a human types. + expect(() => link(db, `bad1-${label}`, 'published ')).toThrow(/CHECK/i); + expect(() => link(db, `bad2-${label}`, 'publushed')).toThrow(/CHECK/i); + expect(() => link(db, `bad3-${label}`, 'held', 'private_ownership ')).toThrow(/CHECK/i); + // ACCEPTED identically — the gate is a bound, not a blanket, on both shapes. + expect(() => link(db, `ok-${label}`, 'published')).not.toThrow(); + expect(() => seal(db, `ok-${label}`, 'document')).not.toThrow(); + expect(() => seal(db, `ok-${label}`, 'document_uncorroborated')).toThrow(/UNIQUE|CHECK/i); + // …and the evidence enum is enforced on both too. + expect(() => link(db, `k2-${label}`, 'held')).not.toThrow(); + expect(() => seal(db, `k2-${label}`, 'document_uncorroberated')).toThrow(/CHECK/i); + // The UPDATE path is the one an incident actually uses, and no CHECK would ever cover it on a + // legacy table — only the trigger does. + expect(() => + sqlite(db, `UPDATE interest_links SET status='published ' WHERE link_key='ok-${label}';`), + ).toThrow(/CHECK/i); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + // The retrofit has to be safe on a LIVE D1: ship-related-persons wipes rows, never table definitions, // so a deployed database keeps its unconstrained 0003 shape until this migration rebuilds it. it('0007 retrofits a pre-existing database, keeps its rows, and re-applies as a no-op', () => { From 23a1939a50bf330082b5f0e9813cf2a67d936727 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 14 Aug 2026 09:51:38 +0300 Subject: [PATCH 61/69] test: assert the seal vocabulary with the production predicate, not a looser copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both seal tests restated MATCHED_FACT_RE locally, and both got it wrong in the permissive direction: `seat:` followed by unlimited uppercase tokens, where production bounds a settlement to two. So `seat:ИВАН ПЕТРОВ ГЕОРГИЕВ` — a three-part Bulgarian name (ЗГР чл. 9) wearing an allowed prefix, which is exactly what a mis-split seat field produces — passed both tests and fails the real rail. They could not fail on the regression they exist to catch, which is the one shape this rail keeps off a served column. Both now import `isSealedFact` from the module that WRITES the vocabulary, so the assertion and the enforcement cannot drift apart again. Adds the direct unit test neither file had: MATCHED_FACT_RE accepts real one- and two-token settlements including a hyphenated one, accepts every role/field-code shape and `eik`, and rejects the three-token seat, a bare name, a name where a field code belongs, a role outside the vocabulary, an empty settlement and a seat carrying the ЕИК itself. Mutation-checked: widening the production bound to the old local pattern now fails with "wrongly accepted: seat:ИВАН ПЕТРОВ ГЕОРГИЕВ" — under the previous tests that same change was silent. --- scripts/cacbg/load.test.mjs | 7 ++++-- scripts/tr/evidence.test.mjs | 48 +++++++++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index 468f11fa..987cbdd8 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -10,6 +10,8 @@ import path from 'node:path'; import os from 'node:os'; import { fileURLToPath } from 'node:url'; import { fingerprint } from './suppressions.mjs'; +// The seal vocabulary is asserted with the PRODUCTION predicate, imported from the module that writes it. +import { isSealedFact } from '../tr/evidence.mjs'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); @@ -1241,9 +1243,10 @@ test('every link carries an evidence seal, and no seal carries a name', () => { const seals = db.prepare('SELECT * FROM interest_link_evidence').all(); assert.equal(seals.length, links.length, 'a seal for EVERY link, held and withdrawn included'); - const CLOSED = /^(?:seat:[\p{Lu} -]+|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + // The PRODUCTION predicate, imported from the module that WRITES the vocabulary — a restated regex + // here was looser than the real one and would have passed a seat token carrying a full name. for (const s of seals) { - if (s.matched_fact != null) assert.match(s.matched_fact, CLOSED, s.matched_fact); + assert.ok(isSealedFact(s.matched_fact), s.matched_fact); assert.ok(s.rules_version.length > 0); assert.match(s.lookup_date, /^\d{4}-\d{2}-\d{2}$/); } diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs index 32123070..d3f71085 100644 --- a/scripts/tr/evidence.test.mjs +++ b/scripts/tr/evidence.test.mjs @@ -6,7 +6,13 @@ // ownership claim, it attaches a real official to the wrong company's ЕИК, contracts and money. import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { RULES_VERSION, evidenceVerdict, reconcileTermination } from './evidence.mjs'; +import { + RULES_VERSION, + evidenceVerdict, + reconcileTermination, + MATCHED_FACT_RE, + isSealedFact, +} from './evidence.mjs'; const container = (t) => `

    ${t}

    `; @@ -452,8 +458,44 @@ test('a missing deed that is NOT marked outside-ТР is an error, not a silent h }); // ── the seal ────────────────────────────────────────────────────────────────── +test('MATCHED_FACT_RE bounds a settlement to two tokens — a NAME cannot wear the seat: prefix', () => { + // The rail tested DIRECTLY, not just through whatever verdicts the ladder happens to produce. Both + // seal tests previously restated this regex locally and got it WRONG in the permissive direction — + // `seat:` followed by unlimited uppercase tokens — so a three-part Bulgarian name (ЗГР чл. 9) wearing + // an allowed prefix passed them. That value is exactly what a mis-split of the seat field produces, + // and it is the one shape this rail exists to keep off a served column. + for (const ok of [ + 'seat:СОФИЯ', + 'seat:ВЕЛИКО ТЪРНОВО', // a real two-token settlement must still pass + 'seat:ГЕНЕРАЛ ТОШЕВО', + 'seat:ЦАР-КАЛОЯН', // hyphenated is one token + 'role:owner:CR_F_19_L', + 'role:manager:CR_F_7_L', + 'role:owner:CR_F_23_L', + 'eik', + ]) + assert.equal(MATCHED_FACT_RE.test(ok), true, `wrongly rejected: ${ok}`); + + for (const bad of [ + 'seat:ИВАН ПЕТРОВ ГЕОРГИЕВ', // THE case: three tokens is a name, not a settlement + 'seat:ИВАН ПЕТРОВ ГЕОРГИЕВ ДРУГ', + 'ИВАН ПЕТРОВ ГЕОРГИЕВ', // a bare name with no prefix at all + 'role:owner:ИВАН ПЕТРОВ', // a name where a field code belongs + 'role:cashier:CR_F_19_L', // a role outside the vocabulary + 'seat:', // an empty settlement asserts nothing + 'eik:201122335', // the ЕИК itself is never stored, only the fact that one matched + ]) + assert.equal(MATCHED_FACT_RE.test(bad), false, `wrongly accepted: ${bad}`); + + // null is legal — a rung may match no fact — and that is isSealedFact's job, not the regex's. + assert.equal(isSealedFact(null), true); + assert.equal(isSealedFact('seat:ИВАН ПЕТРОВ ГЕОРГИЕВ'), false); +}); + test('matched_fact stays inside the closed vocabulary — it can never carry a name', () => { - const CLOSED = /^(?:seat:[\p{Lu} -]+|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + // The PRODUCTION predicate, imported — never a local copy of it. A re-stated regex here was looser + // than `MATCHED_FACT_RE` (it allowed `seat:` + unlimited tokens), so this loop certified values the + // real rail rejects and could not fail on the regression it exists to catch (cefothe, #309). for (const v of [ evidenceVerdict(base), evidenceVerdict({ ...base, deed: deed([fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))]) }), @@ -468,7 +510,7 @@ test('matched_fact stays inside the closed vocabulary — it can never carry a n }), ]) { if (v.matchedFact == null) continue; - assert.match(v.matchedFact, CLOSED, `matched_fact escaped the vocabulary: ${v.matchedFact}`); + assert.ok(isSealedFact(v.matchedFact), `matched_fact escaped the vocabulary: ${v.matchedFact}`); assert.ok(!/ИВАН|ПЕТРОВ|ТЕСТОВ/.test(v.matchedFact), 'a NAME reached matched_fact'); } }); From 28137261004dcb0bcd8c75f3f2bd7ef8a5992d02 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 14 Aug 2026 09:58:47 +0300 Subject: [PATCH 62/69] fix(web,db,docs): render the entry number, bind the join invariant, correct the docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit registryEntryNumber was in the DTO, mapped in the query and present in every fixture, and rendered nowhere — carried to every client for nothing. It is also what makes the claim findable: a date does not identify a record in the register, and a reader checking whether we read the act correctly needs the number. Shown only when present, since a seat/ЕИК confirmation cites no act entry and an empty „№" would read as missing data rather than an inapplicable field. (cefothe cited ADR-0033 as requiring this. It does not — no such text exists there. The dead payload stands on its own.) The contract join is now pinned writer-to-reader. On the read side `tenders`/`authorities` are never projected, so both joins look dead and invite deletion — a reviewer proposed exactly that. They are not dead, they are the SHAPE: the writer at load.mjs stores contract_count/contract_value_eur through the same four tables, so deleting them on the read side alone would let the read exceed the stored aggregate and surface links the zero-contract gate excluded. Deleting them on BOTH sides is defensible but changes published figures and needs re-baselining against ADR-0033 §10, so it stays #226 §1.6. A comment cannot survive a refactor; the test compares both by relation and join condition. Mutation-checked in both directions. Docs: SUPPRESSION_KEY_VERSION is a repository VARIABLE, not a secret — looking for it under Secrets during an incident is a dead end. The suppression runbook gains the verification it lacked: a takedown that silently failed looks exactly like one that worked, so it now says to check the served D1 and the page, allow for the 1h public cache, and re-run the next load to prove the removal survives a rebuild. deploy.md no longer claims deploys do not migrate — deploy.yml applies 0003/0006/0007 explicitly, and why. review-testing.md records scripts/ as a reviewed carve-out from the ratchet, naming the three runners and the mutation discipline that stands in for a percentage. --- apps/web/app/components/ConflictCards.tsx | 5 +++ apps/web/app/routes/conflicts.render.test.tsx | 16 +++++++++ docs/deploy.md | 11 ++++-- docs/review-testing.md | 20 +++++++++++ docs/runbooks/related-persons-suppression.md | 30 ++++++++++++++-- packages/db/src/queries/related-persons.ts | 12 +++++++ packages/db/src/related-persons-sql.test.ts | 34 +++++++++++++++++++ scripts/cacbg/load.mjs | 6 ++++ 8 files changed, 129 insertions(+), 5 deletions(-) diff --git a/apps/web/app/components/ConflictCards.tsx b/apps/web/app/components/ConflictCards.tsx index 28783efd..a716c425 100644 --- a/apps/web/app/components/ConflictCards.tsx +++ b/apps/web/app/components/ConflictCards.tsx @@ -193,6 +193,11 @@ function ConflictCard({ {registryEvidenceLabel(l)} {l.registryEntryDate ? ` · вписване ${l.registryEntryDate}` : ''} + {/* The entry NUMBER, not just its date: a date does not identify a record, and this is + what a reader types to find the same act we read. Rendered only when present — + a seat/ЕИК confirmation cites no act entry, and an empty „№" would read as missing + data rather than as an inapplicable field. */} + {l.registryEntryNumber ? ` · № ${l.registryEntryNumber}` : ''} {l.registryLookupDate ? ` · справка ${l.registryLookupDate}` : ''} diff --git a/apps/web/app/routes/conflicts.render.test.tsx b/apps/web/app/routes/conflicts.render.test.tsx index 07339f05..814c3670 100644 --- a/apps/web/app/routes/conflicts.render.test.tsx +++ b/apps/web/app/routes/conflicts.render.test.tsx @@ -268,6 +268,22 @@ describe('Trade Register evidence on the card (#279, ADR-0033)', () => { expect(text).toContain('лицето е вписано като съдружник/собственик'); expect(text).toContain('вписване 2011-05-02'); // WHICH entry expect(text).toContain('справка 2026-08-05'); // and HOW FRESH it is + // The entry NUMBER is what makes the claim findable in the register — a date alone does not + // identify a record. It was carried to every client in the DTO and never rendered, which is the + // one payload that costs bytes and answers nothing (cefothe, #309). + expect(text).toContain('20110502101007'); + }); + + it('omits the entry number rather than printing an empty label when there is none', async () => { + // POSITIVE CONTROL for the row's shape: a confirmed link (seat/ЕИК) has no act entry to cite, so + // the label must be absent entirely — not „· №" with nothing after it, which reads as missing data + // rather than as an inapplicable field. Scoped to the evidence label: „№" alone is the card RANK. + await renderConflicts([ + link({ evidenceKind: 'confirmed', registryRole: null, registryEntryNumber: null }), + ]); + const text = container.textContent ?? ''; + expect(text).toContain('Регистър'); + expect(text).not.toContain('· №'); }); it('a seat/ЕИК confirmation never implies somebody was found in the act', async () => { diff --git a/docs/deploy.md b/docs/deploy.md index ac3e845c..eb25a430 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -170,8 +170,15 @@ domain таблиците и преизчислява rollup-ите + FTS. > [scripts/precompute.sql](../scripts/precompute.sql) — не изпращайте FTS съдържание през dump. > Промени в схемата след първото зареждане се прилагат out-of-band, за всяка среда: -> `SIGMA_D1_NAME=sigma-stage wrangler d1 migrations apply sigma-stage --remote`. Деплоите не мигрират -> и не презареждат данни. +> `SIGMA_D1_NAME=sigma-stage wrangler d1 migrations apply sigma-stage --remote`. Деплоите **не презареждат +> данни**. +> +> **Уточнение (беше неточно):** тук пишеше и „деплоите не мигрират". Вече не е вярно — +> [deploy.yml](../.github/workflows/deploy.yml) прилага изрично `0003`, `0006` и `0007` с +> `d1 execute --file` при всеки деплой. Причината е, че ledger-ът на wrangler на тези бази е празен +> (базовата схема е създадена out-of-band), тъй че `d1 migrations apply` би се сблъскал с `0000`. И трите +> са безопасни за повторно прилагане: `CREATE … IF NOT EXISTS` и, при `0007`, тригери и индекси, които +> се пресъздават до същото състояние. Останалите схемни промени наистина си остават out-of-band. ## 3. Конфигуриране на GitHub Environments diff --git a/docs/review-testing.md b/docs/review-testing.md index 3ab4b106..0cbd4b0a 100644 --- a/docs/review-testing.md +++ b/docs/review-testing.md @@ -43,6 +43,26 @@ - Нов workspace с тестове се добавя и в `coverage-baseline.json` (и получава `vitest.config.ts` с `sharedCoverage(...)`); `packages/api-contract` е освободен, докато няма тестове. +### `scripts/` е извън ratchet-а — прегледано изключение, не пропуск + +Ratchet-ът покрива само `apps/*` и `packages/*`. `scripts/` (~7 хил. реда, сред които клеветнически +критичната логика на свързаните лица — доказателственият ред, съвпадението по имена, гейтовете при +публикуване) не се измерва от него. Записано е тук, за да е решение, а не дупка, която някой открива +по-късно. + +**Защо:** тези файлове са `.mjs`, пускат се от три отделни runner-а извън `turbo run test` и голяма част +от тях са I/O обвивки около мрежа и SQLite, за които процент покритие би измервал колко от обвивката е +докосната, а не дали правилото е вярно. + +**Какво пази качеството вместо процент:** и трите набора са в CI и са задължителни — +`node --test scripts/*.test.mjs`, `node --import ./scripts/cacbg/register-ts.mjs --test +scripts/cacbg/*.test.mjs scripts/tr/*.test.mjs` — а гейтовете, при които грешка означава поименно +твърдение за реален човек, се проверяват **мутационно**: тестът трябва да пада при обърнато условие, +иначе не е гейт. Виж ADR-0027 за защо тест, който минава и преди реализацията, е фалшив гейт. + +**Кога това трябва да се преразгледа:** ако `scripts/` започне да носи логика, която не е нито +чиста функция с мутационно проверен тест, нито тънка I/O обвивка. + ## Integrity gate - Пуска се върху обслужвания D1 след `precompute` в `ship-domain.mjs` и след `runSliceDerive()` в diff --git a/docs/runbooks/related-persons-suppression.md b/docs/runbooks/related-persons-suppression.md index 1fffa8c1..7d407cd3 100644 --- a/docs/runbooks/related-persons-suppression.md +++ b/docs/runbooks/related-persons-suppression.md @@ -112,6 +112,30 @@ If a ship fails on any of these, fix the cause; do **not** empty the list or dro A rotated salt invalidates every existing fingerprint, so it is a coordinated change guarded by `key_version`: re-derive `fp` for each entry from its original `link_key` under the new salt (you need the plaintext keys, kept out-of-band, e.g. in the incident tickets), **bump every entry's `key_version`** to the -new value, replace the file in one commit, and update both CI secrets (`SUPPRESSION_SALT` and -`SUPPRESSION_KEY_VERSION`). Because the loader refuses any entry on a non-current `key_version`, a -half-finished rotation fails the build loudly instead of silently un-suppressing. +new value, replace the file in one commit, and update **both** CI settings — but note they are different +kinds: `SUPPRESSION_SALT` is a repository **secret**, while `SUPPRESSION_KEY_VERSION` is a repository +**variable** (`${{ vars.SUPPRESSION_KEY_VERSION || '1' }}` in `related-persons-data.yml`). The version is +not sensitive — it is a counter — and looking for it under Secrets is a dead end during an incident. +Because the loader refuses any entry on a non-current `key_version`, a half-finished rotation fails the +build loudly instead of silently un-suppressing. + +## Verifying a takedown actually worked + +A takedown that silently failed looks exactly like one that succeeded: the entry sits in the list, the +build is green, and the link is still public. The list is applied at LOAD time, so nothing changes on the +served surface until the next data run ships — check the surface, not the commit. + +1. **The loader saw it.** The run refuses an entry matching no built link (the B3 unused-entry rail), so a + green run already proves the fingerprint matched something. A run that fails with „suppression matched + NO link" means the `link_key` was wrong — a family link needs its `|family` suffix. +2. **The row is gone from the served D1**, which is the only copy a reader can reach: + ``` + wrangler d1 execute "$SIGMA_D1_NAME" --remote \ + --command "SELECT status FROM interest_links WHERE link_key = ''" + ``` + Expect `suppressed`, or no row at all. Anything else means the ship did not carry the decision. +3. **The page is gone**, allowing for cache: the link's page must 404 and the official's page must not + list it. `publicCache(3600)` means a reader can still see it for up to an hour after the write — if it + is still there beyond that, the takedown did not land. +4. **It stays gone.** Re-run the next scheduled load and repeat step 2: the list is what makes a takedown + survive a rebuild, and this is the only step that proves the survival rather than assuming it. diff --git a/packages/db/src/queries/related-persons.ts b/packages/db/src/queries/related-persons.ts index b149ebbd..25f2a752 100644 --- a/packages/db/src/queries/related-persons.ts +++ b/packages/db/src/queries/related-persons.ts @@ -92,6 +92,18 @@ interface LinkRow { // The winner's contracts, joined exactly as the ETL aggregate does (contracts→tenders→authorities→bidders, // matched by eik_normalized) so any read-time subset is a true subset of the stored contract_count/value. // Alias-distinct (cc/tt/aa/bb) so it composes as a correlated subquery under the LINK_SELECT `il`/`b` scope. +// +// `tt` and `aa` are NOT projected here, which makes both joins look dead and invites deleting them. They +// are not dead — they are the SHAPE, and its counterpart is the WRITER at scripts/cacbg/load.mjs (the +// per-winner contract query that fills contract_count / contract_value_eur). The two must stay identical. +// +// Removing them on the read side ALONE would widen the read to contracts the stored aggregate never +// counted: `contemporaneous_contract_count` could then exceed `contract_count`, and the EXISTS gate below +// (the one that decides whether a link surfaces at all) would publish links the I5 zero-contract gate had +// excluded. Removing them on BOTH sides is defensible — an unresolvable authority currently drops a +// contract from the money everywhere, consistently — but it changes published figures and so must be +// re-baselined against ADR-0033 §10's control totals, not slipped in as a cleanup. Tracked as #226 §1.6. +// `related-persons-sql.test.ts` pins this string to the writer's so neither can be "optimised" alone. const CONTRACT_JOIN = `FROM contracts cc JOIN tenders tt ON tt.id = cc.tender_id JOIN authorities aa ON aa.id = tt.authority_id diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index 1501f16d..f389d525 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -10,6 +10,7 @@ import { LEADERBOARD_SQL, LINK_CONTRACTS_LIMIT, LINK_CONTRACTS_SQL, + LINK_SELECT, OFFICIAL_SQL, SURFACED_OWNERSHIP, } from './queries/related-persons'; @@ -632,6 +633,39 @@ describe('свързани-лица SQL (real SQLite)', () => { }); }); +// The read-time contract join and the WRITER that fills contract_count/contract_value_eur must use the +// same join shape. On the read side `tenders`/`authorities` are never projected, so both joins read as +// dead and a reviewer will eventually propose deleting them (one did — cefothe on #309). Deleting them +// there alone widens the read past the stored aggregate: contemporaneous counts could exceed +// contract_count, and the EXISTS gate would surface links the zero-contract gate excluded. Nothing but a +// test can hold a TypeScript template string and a .mjs prepared statement to the same shape. +describe('the read-time contract join matches the writer that stored the aggregate', () => { + const writer = readFileSync(resolve(root, 'scripts/cacbg/load.mjs'), 'utf8'); + // The writer's per-winner contract query, located by its projection rather than by line number. + const writerJoin = /FROM contracts c JOIN tenders t ON[^"]*?WHERE b\.eik_normalized/.exec( + writer, + )?.[0]; + + it('the writer still joins contracts→tenders→authorities→bidders', () => { + expect(writerJoin, 'load.mjs per-winner contract query not found').toBeTruthy(); + for (const rel of ['tenders', 'authorities', 'bidders']) expect(writerJoin).toContain(rel); + }); + + it('CONTRACT_JOIN joins the same four relations, in the same direction', () => { + // Compared by RELATION and join condition, not by text: the two are written in different languages + // and alias differently (cc/tt/aa/bb vs c/t/a/b), so only the shape is comparable. + const flat = LINK_SELECT.replace(/\s+/g, ' '); + expect(flat).toContain('FROM contracts cc'); + expect(flat).toContain('JOIN tenders tt ON tt.id = cc.tender_id'); + expect(flat).toContain('JOIN authorities aa ON aa.id = tt.authority_id'); + expect(flat).toContain('JOIN bidders bb ON bb.id = cc.bidder_id'); + // …and the writer's equivalents, so a change to either side fails here. + expect(writerJoin).toContain('JOIN tenders t ON t.id=c.tender_id'); + expect(writerJoin).toContain('JOIN authorities a ON a.id=t.authority_id'); + expect(writerJoin).toContain('JOIN bidders b ON b.id=c.bidder_id'); + }); +}); + // The evidence-seal gate has FOUR copies, and they must say the same thing. Reviewers found the fourth // (the company-search badge) checking `status='published'` alone, so search advertised a свързани-лица // claim about a named official that the detail page correctly withheld. `refresh-slice.sql` matters for a diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 31e627c9..18a806f1 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -552,6 +552,12 @@ for (const r of readJsonl(path.join(STAGING, 'related.jsonl'))) { db.exec('COMMIT'); // --- enrich each (person,eik) → interest_links (+ per-authority breakdown) ----------------------- +// THE WRITER of contract_count / contract_value_eur. Its join shape (contracts→tenders→authorities→ +// bidders) is mirrored by CONTRACT_JOIN in packages/db/src/queries/related-persons.ts, so that the +// read-time subset can never exceed what was stored. `authorities` here IS projected (the per-authority +// breakdown needs the name), unlike on the read side where it looks dead — that asymmetry is why the two +// have to be pinned to each other rather than reasoned about separately. related-persons-sql.test.ts +// asserts they agree; change one only by changing both, and re-baseline ADR-0033 §10 when you do. const contractStmt = db.prepare( "SELECT strftime('%Y', c.signed_at) yr, a.id auth_id, a.name authority, c.amount_eur eur FROM contracts c JOIN tenders t ON t.id=c.tender_id JOIN authorities a ON a.id=t.authority_id JOIN bidders b ON b.id=c.bidder_id WHERE b.eik_normalized=?", ); From 9999ad53346cc63da81f344526378109aecedbd4 Mon Sep 17 00:00:00 2001 From: Stefan Angelov Date: Fri, 14 Aug 2026 13:29:07 +0300 Subject: [PATCH 63/69] =?UTF-8?q?fix(etl):=20=D0=BF=D0=BE=D0=BF=D1=80?= =?UTF-8?q?=D0=B0=D0=B2=D1=8F=D0=B9=20=D0=B8=20=D1=84=D0=BB=D0=B0=D0=B3?= =?UTF-8?q?=D0=B2=D0=B0=D0=B9=20=D0=B4=D0=B2=D0=BE=D0=B9=D0=BD=D0=BE=20?= =?UTF-8?q?=D0=B1=D1=80=D0=BE=D0=B5=D0=BD=D0=B5=20=D0=B2=20=D1=81=D1=82?= =?UTF-8?q?=D0=BE=D0=B9=D0=BD=D0=BE=D1=81=D1=82=D1=82=D0=B0=20=D0=BD=D0=B0?= =?UTF-8?q?=20=D0=B0=D0=BD=D0=B5=D0=BA=D1=81=20(#307)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Възложителят вписва новата ОБЩА стойност в полето за разликата, тъй че value_after излиза сбор от старата стойност и новия общ размер - договорът се сервира двойно по-скъп, отколкото е. Класификаторът чете основанието и решава кое от трите е: истинско увеличение (не пипай), обявен нов общ размер (пренапиши value_after на него) или непроменена стойност, удвоена от деноминация (върни към старата). Без текстов сигнал редът НЕ се пренаписва - пада към новия флаг annex_total_suspect и се изключва от сумите. Изборът е съзнателен: чл. 116, ал. 1, т. 1 от ЗОП допуска законно +100% през предварително обявена опция, а тихото разполовяване би подменило вярна стойност. Честна дупка бие подменено число. Пазачите около пренаписването са тесни, защото цената при грешка е несиметрична - подценен договор, невидим за читателя: - парична котва: число без валута до себе си не може да мине за стойност; - вето за непарични мерни единици, включително с определение („200 работни дни", „200 календарни дни", „200 (двеста) дни", кв.м/куб.м) - иначе брой дни, съвпаднал с разликата, подменя стойността на договора; - „в евро" пренаписва само в израза за деноминация („от лева в евро"), не и в клауза за валута на плащане. Премерено на целия корпус: 564 реда имат формата на двойното броене; ветата за мерни единици не местят нито един от тях, тоест стеснението не изяжда нищо живо. Спот-проверка срещу текста на анекса потвърди и обратната посока - семейството „се променя на 1,52 лв./км." се пренаписва вярно. Колоните value_restated / value_treatment / value_suspect стигат до разгърнатите бази през сонда в deploy.yml преди деплоя на Worker-а: ledger-ът на wrangler на тези бази е празен и `d1 migrations apply` би се сблъскал с 0000. Четири кръга преглед; всяка находка е проверена срещу истинския код, не приета по описание. Затваря #305. --- .github/workflows/deploy.yml | 59 ++ apps/web/app/lib/assistant/describe-schema.ts | 8 +- apps/web/app/routes/contract.tsx | 39 +- docs/README.md | 1 + .../305-amendment-value-double-count.md | 119 +++ packages/api-contract/src/index.ts | 5 + packages/db/migrations/0000_init.sql | 4 +- .../db/migrations/0006_amendment_restated.sql | 17 + .../0007_amendment_value_suspect.sql | 14 + packages/db/src/amendments-ocds-link.test.ts | 5 + packages/db/src/amendments-sql.test.ts | 6 + .../db/src/amendments-total-restated.test.ts | 301 ++++++++ .../db/src/amendments-total-suspect.test.ts | 708 ++++++++++++++++++ .../db/src/contractor-identity-sql.test.ts | 12 + .../etl-entity-canonicalization-sql.test.ts | 5 + packages/db/src/queries/details.test.ts | 58 ++ packages/db/src/queries/details.ts | 36 +- packages/db/src/refresh-slice.test.ts | 18 + .../db/src/value-flag-annex-step-sql.test.ts | 5 + .../db/src/value-flag-stotinki-sql.test.ts | 5 + packages/ingest/src/amendment-total.test.ts | 348 +++++++++ packages/ingest/src/amendment-total.ts | 227 ++++++ packages/ingest/src/base.test.ts | 48 ++ packages/ingest/src/base.ts | 31 + packages/ingest/tsconfig.json | 2 + scripts/derive-amendments.sql | 4 +- scripts/normalize-raw.sql | 168 +++++ scripts/precompute.sql | 7 +- scripts/promote-amendments.sql | 57 +- scripts/refresh-slice.sql | 283 ++++++- scripts/work-staging-schema.sql | 7 + tsconfig.base.json | 3 +- 32 files changed, 2573 insertions(+), 37 deletions(-) create mode 100644 docs/implementation-plans/305-amendment-value-double-count.md create mode 100644 packages/db/migrations/0006_amendment_restated.sql create mode 100644 packages/db/migrations/0007_amendment_value_suspect.sql create mode 100644 packages/db/src/amendments-total-restated.test.ts create mode 100644 packages/db/src/amendments-total-suspect.test.ts create mode 100644 packages/ingest/src/amendment-total.test.ts create mode 100644 packages/ingest/src/amendment-total.ts diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5d5ca680..835e208d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -214,6 +214,65 @@ jobs: ;; esac + # #305 additive columns (migrations 0006/0007) must exist BEFORE the Worker serves: the contract-page + # query reads amendments.value_restated/value_suspect, and promote-amendments/refresh-slice also INSERT + # value_treatment — so all three must be present or the read AND the ETL write fail. Same rationale as + # the currency step above: the migration ledger is empty, so `d1 migrations apply` would collide on + # 0000. Probe the actual table and ALTER only the missing columns (SQLite has no ADD COLUMN IF NOT + # EXISTS). These are pure additions with safe defaults (INTEGER NOT NULL DEFAULT 0 / nullable TEXT), so + # no backfill or completion marker is needed — an ALTER populates every existing row. Malformed + # responses are fatal. This also makes 0006/0007 replay-safe when they were applied out-of-ledger. + - name: Apply amendment restated/suspect columns + if: steps.guard.outputs.ok == 'true' + run: | + node scripts/wrangler-render.mjs apps/web/wrangler.jsonc + schema_json="$(pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes --json \ + --command "SELECT + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_restated') AS has_restated, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_treatment') AS has_treatment, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_suspect') AS has_suspect")" + + read_flag() { + printf '%s' "$schema_json" | node -e ' + const fs = require("fs"); + let payload; + try { + payload = JSON.parse(fs.readFileSync(0, "utf8")); + } catch { + process.exit(2); + } + const result = Array.isArray(payload) ? payload[0] : payload; + const row = result && Array.isArray(result.results) ? result.results[0] : null; + const key = process.argv[1]; + if (!row || !Object.hasOwn(row, key)) process.exit(2); + process.exit(Number(row[key]) === 1 ? 0 : 1); + ' "$1" + } + + add_column() { + echo "amendments.$1 missing; adding it." + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --command "ALTER TABLE amendments ADD COLUMN $2" + } + + ensure_column() { + set +e + read_flag "$1" + status="$?" + set -e + case "$status" in + 0) echo "amendments.$1 already exists." ;; + 1) add_column "$1" "$2" ;; + *) echo "::error::Could not determine whether amendments.$1 exists."; exit 1 ;; + esac + } + + ensure_column value_restated "value_restated INTEGER NOT NULL DEFAULT 0" + ensure_column value_treatment "value_treatment TEXT" + ensure_column value_suspect "value_suspect INTEGER NOT NULL DEFAULT 0" + # `run deploy`, not `deploy` — bare `pnpm deploy` is a pnpm built-in, not our package script. - name: Deploy explorer (sigma) if: steps.guard.outputs.ok == 'true' diff --git a/apps/web/app/lib/assistant/describe-schema.ts b/apps/web/app/lib/assistant/describe-schema.ts index 6157b382..8e887417 100644 --- a/apps/web/app/lib/assistant/describe-schema.ts +++ b/apps/web/app/lib/assistant/describe-schema.ts @@ -10,11 +10,13 @@ export const DATA_TRAPS: string[] = [ 'Парични агрегати: СУМИРАЙ САМО `contracts.amount_eur` (каноничен EUR, безопасен за сумиране). ' + 'НИКОГА не сумирай `contracts.amount` — то е „както е записано" в смесена валута (`currency`), само за показване.', 'Канонична база за всяка парична сума: `contracts.amount_eur IS NOT NULL`. НЕ филтрирай по ' + - '`value_flag`: включи `ok`, `review`, `annex_suspect`, `value_low` и поправените `value_suspect` редове.', + '`value_flag`: включи `ok`, `review`, `annex_suspect`, `annex_total_suspect`, `value_low` и ' + + 'поправените `value_suspect` редове.', '`amount_eur IS NULL` означава, че няма използваема EUR стойност (например `value_suspect` без ' + 'прогноза за поправка или чужда валута без FX курс); само тези редове се изключват от парични суми.', - '`value_flag` ∈ {ok, review, annex_suspect, value_suspect, value_low} мени значението на стойността ' + - 'на реда, но не и каноничната база; `date_flag` ∈ {ok, signed_after_publication} е вердикт за датата.', + '`value_flag` ∈ {ok, review, annex_suspect, annex_total_suspect, value_suspect, value_low} мени ' + + 'значението на стойността на реда, но не и каноничната база; `date_flag` ∈ {ok, ' + + 'signed_after_publication} е вердикт за датата.', "`tenders.procedure_type = 'неизвестна'` маркира СИНТЕТИЧНИ (само-договорни) преписки — " + 'изключи ги при анализ на разпределението по процедура, освен ако нарочно ги искаш.', '`lots` са на grain по обособена позиция — не ги брой едно към едно срещу `contracts`.', diff --git a/apps/web/app/routes/contract.tsx b/apps/web/app/routes/contract.tsx index 8eb1175b..e3347461 100644 --- a/apps/web/app/routes/contract.tsx +++ b/apps/web/app/routes/contract.tsx @@ -222,7 +222,13 @@ export default function Contract({ loaderData }: Route.ComponentProps) {
    Текуща стойност
    {v.currentEur != null ? money(v.currentEur) : '—'} - {v.suspect &&
    {UNVERIFIED_VALUE_LABEL}
    } + {v.currentValueDoubled ? ( +
    + стойността изглежда двойно отчетена и не се показва +
    + ) : ( + v.suspect &&
    {UNVERIFIED_VALUE_LABEL}
    + )} {v.deltaPct != null && (
    {signedPct(v.deltaPct)} спрямо сключване
    )} @@ -230,8 +236,10 @@ export default function Contract({ loaderData }: Route.ComponentProps) {
    {v.suspect && (

    - Показана е публикуваната стойност от източника, без СИГМА да я коригира. Виж{' '} - методология. + {v.currentValueDoubled + ? 'Текущата стойност изглежда двойно отчетена в източника и затова не се показва. ' + : 'Показана е публикуваната стойност от източника, без СИГМА да я коригира. '} + Виж методология.

    )} {c.frameworkAwards != null && ( @@ -267,11 +275,32 @@ export default function Contract({ loaderData }: Route.ComponentProps) { {c.amendments.map((a, i) => ( {a.date ? longDate(a.date) : '—'} + {/* #305 residual: an uncorrectable double-count — the source's value_after is the + untrusted doubled figure, so show „—" and mark the row rather than a number we + can't stand behind. A `restated` row is the opposite: СИГМА corrected the doubled + total from the основание text, so we show the corrected number and flag that we + rewrote it. */} - {a.valueAfterEur != null ? moneyBare(a.valueAfterEur) : '—'} + {a.suspect ? ( + <> + — непотвърден тотал + + ) : a.valueAfterEur != null ? ( + <> + {moneyBare(a.valueAfterEur)} + {a.restated && ( + <> + {' '} + коригиран тотал + + )} + + ) : ( + '—' + )} - {a.deltaEur != null ? signedMoney(a.deltaEur) : '—'} + {!a.suspect && a.deltaEur != null ? signedMoney(a.deltaEur) : '—'} diff --git a/docs/README.md b/docs/README.md index 408599db..01cda08a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,7 @@ - [`etl-architecture.md`](etl-architecture.md) — целевата ETL архитектура (RFC): предложение за състоянието и реда на изпълнение. - [`v1-implementation-plan.md`](v1-implementation-plan.md) — precompute слоят и пагинацията (защо rollup-и и keyset вместо per-request GROUP BY / OFFSET). - [`implementation-plans/286-ocds-amendment-unp.md`](implementation-plans/286-ocds-amendment-unp.md) — защо OCDS анексите не се свързват с договор (OCID вместо УНП) и планът за поправка през bridge-а `tender.id → УНП` + prefer-EOP dedup (#286). +- [`implementation-plans/305-amendment-value-double-count.md`](implementation-plans/305-amendment-value-double-count.md) — защо стойността на анекс се удвоява (ЦАИС ЕОП слага новия **тотал** в полето за промяна) и планът за откриване/поправка: `annex_total_suspect` флаг + текстова хеуристика за възстановяване на истинския тотал (#305). - [`integrity-gate.md`](integrity-gate.md) — reconciliation gate-ът: hard asserts върху тоталите при import/CI. - [`anomaly-report.md`](anomaly-report.md) — cross-row аномалии при опресняване: какво `value_flag` не хваща на ниво отделен договор. - [`deploy.md`](deploy.md) — деплой към Cloudflare: двата Worker-а (`sigma`, `sigma-etl`) и споделеният D1 per environment. diff --git a/docs/implementation-plans/305-amendment-value-double-count.md b/docs/implementation-plans/305-amendment-value-double-count.md new file mode 100644 index 00000000..7c67c492 --- /dev/null +++ b/docs/implementation-plans/305-amendment-value-double-count.md @@ -0,0 +1,119 @@ +# Implementation Plan: #305 — Amendment value double-count (a new *total* is added to the old value) + +## Executive Summary + +| Field | Value | +|---|---| +| Ticket | [midt-bg/sigma#305](https://github.com/midt-bg/sigma/issues/305) — labels `data-quality`, `etl`, `priority: high` | +| Problem | When an EOP annex announces a new **total** contract value, ЦАИС ЕОП puts that total in the *change* field, so `currentContractValue = lastContractValue + newTotal`. Sigma stores `value_after` verbatim, so the served value is **doubled** (the old value is counted twice). | +| Root cause | **Source data defect, faithfully stored.** `base.ts:313-315` maps `value_before ← lastContractValue`, `value_after ← currentContractValue`, `value_delta ← contractValueDifference` with **no arithmetic**. The bug is that, for a subset of annexes, `contractValueDifference` (→ `value_delta`) holds the **new total**, not the increment — and the feed's `currentContractValue` is `before + that total`. Verified: `value_after = value_before + value_delta` on **100%** of delta-carrying rows on `sigma-dev`. | +| Why existing flags miss it | A double-count is only ~2× signed value. `value_flag = 'annex_suspect'` needs ≥5× aggregate (+ a ≥10× per-step); #299 the same; the estimate-based flags need ≥10×/≥200×. 2× < 5× ⇒ classed **`ok`** ⇒ enters the `amount_eur` canonical value base and inflates every rollup. (`normalize-raw.sql:882-948`.) | +| Scale (real corpus) | On the issue's 2020→2026 local rebuild: 7,335 price-raising annexes, **686 at ≥100% growth**, ~666 unflagged, **€475.4M**. Independently reproduced on `sigma-dev`: **686 at ≥100% growth**, ~526 unflagged. The named records (145652, 189325, 84818) confirm. | +| Complexity | Medium. Detection is the hard part (a source-text heuristic with false-positive risk); the plumbing (flag + exclude, then optional correct) mirrors the existing `annex_suspect` machinery. | +| Risk | Medium. A naive correction that trusts a text heuristic can mis-restate genuine >100% increases; a flag-only tier is safe and ships first. The signature has a **blind spot** (a new-total *lower* than the old value hides below +100%), so the fix must not be sold as complete. | +| Status | **Draft — investigated with real DB calls + code trace.** | + +> **Two independently-true facts frame the fix.** (1) The source is internally *consistent* — `value_after = value_before + value_delta` always — so the correction can be expressed purely as *"when `value_delta` is a **total**, the true `value_after` is `value_delta`, not `value_before + value_delta`."* (2) The defect is `value_flag = 'ok'`, so it is inside the aggregated value base; the minimal safe fix is to move it *out* of that base (a new verdict), exactly as #299 did for its case. + +--- + +## 1. Problem, verified on real data + +### 1a. The mechanism (code, on `main`) +- `packages/ingest/src/base.ts:313-315` — EOP annexes map three source keys straight to columns, no math: + - `value_before ← lastContractValue` + - `value_after ← currentContractValue` + - `value_delta ← contractValueDifference` (the only signed field) +- `scripts/derive-amendments.sql:147-155` (mirrored in `scripts/normalize-raw.sql:752-759`): `contracts.current_value` = the latest amendment's non-null `value_after`. So the doubled `value_after` becomes the served contract value. +- `scripts/promote-amendments.sql:42-44`: `value_before/after/delta` copied verbatim into served `amendments`. + +**Conclusion:** the doubled figure is not computed by Sigma; it arrives in `currentContractValue`. For the affected annexes the authority entered the **new total** into `contractValueDifference`, and the feed's `currentContractValue = lastContractValue + newTotal`. Sigma stores it faithfully. + +### 1b. The named records (queried on `sigma-dev`, read-only) + +| contract | source | value_before | value_after | value_delta | doubled? | +|---|---|---|---|---|---| +| 145652 (УНП 00010-2023-0006) | `eop:annexes:2024-06-21` | 442,000 | **981,240** | 539,240 | yes — `value_delta` (539,240) is the announced new total; true `value_after` = 539,240 | +| 189325 (УНП 00210-2024-0024) | `eop:annexes:2025-10-07` | 77,000,000 | **154,000,000** | 77,000,000 | yes — exact 2× (currency-change annex) | +| 84818 (УНП 00080-2023-0001) | `eop:annexes:2026-07-17` (EUR) | 76,769,540.87 | **153,539,081.74** | 76,769,540.87 | yes — exact 2× | + +Caveats found in verification: 84818 has **6** amendment rows (only the 2026-07-17 EUR annex is the doubled one — earlier BGN annexes are consistent); the issue treated it as one. Absolute counts differ from the issue because `sigma-dev` (31,543 amendments) is a superset of the issue's local rebuild (26,921). + +### 1c. The math signature and its blind spot +- `value_delta = value_after − value_before` holds on **100%** of delta-carrying rows (`sigma-dev`), i.e. the source is self-consistent. The defect is semantic: `value_delta` is sometimes a *total*, not an *increment*. +- A double-count where `newTotal ≥ before` produces growth **≥ 100%** (686 rows). The exactly-+100% subset (`value_after = 2×before`) is the "same total re-stated" / currency-change case. +- **Blind spot:** if `newTotal < before`, the double-count yields growth **< 100%** and hides among clean rows. The +100% line is a *safety threshold*, not a proof of cleanliness — the fix must say so. + +--- + +## 2. Why the existing flags don't catch it + +`scripts/normalize-raw.sql:882-948` (mirrored in `refresh-slice.sql`), evaluated top-down against `eff_eur = EUR(COALESCE(current_value, signing_value))`: +- `value_suspect`: `eff_eur > 2e9`, or `> 200 × proc_est_eur`, or the стотинки band — estimate-relative, ignores a 2× overrun. +- `annex_suspect` (`:937-945`, the #299/#248 rule): `current_value/signing_value ≥ 100`, **or** (`≥ 5` **and** a per-step `value_after ≥ 10 × value_before`). A double-count is ~2× signed ⇒ below 5×. +- `review`: `eff_eur ≥ 10 × proc_est_eur`. +- else `ok`. + +A ~2× inflation clears none of these gates → `ok` → `amount_eur` takes `COALESCE(current_value, signing_value)` (`normalize-raw.sql:818-830`), so the doubled value is summed everywhere. Pinned by `packages/db/src/value-flag-annex-step-sql.test.ts` (the 5× floor is the smallest firing case; 4.9× stays `ok`). + +**Downstream consumers currently inflated** (all via the shared `amount_eur` base — `precompute.sql:16-19`): contract-list totals/sort/buckets and CSV export (`queries/contracts.ts:318,61-62,432,475`); `company_totals.won_eur`, `authority_totals.spent_eur`, `home_totals.value_eur` (`precompute.sql:42-60`); the contract detail value strip and the **amendment timeline**, which reads `amendments.value_after` unrepaired (`queries/details.ts:451-460,670-684`). *(The `/anomalies` #239 and `/overruns` #171 signals named in the issue are not on `main` yet — they will inherit the fix once they land.)* + +--- + +## 3. Detection + +The correction hinges on classifying each price-raising annex as **increment** vs **total**. Layer the signals; never rely on free text alone. + +1. **Arithmetic gate (necessary, cheap, high-recall / low-precision):** `value_before > 0 AND value_after ≥ 2 × value_before` (equivalently `value_delta ≥ value_before`). A single annex whose *increment* is ≥ the entire prior value is implausible; a double-count always lands here. Catches the 686. Does **not** catch the sub-100% blind spot (out of scope for v1, documented). +2. **Text confirmation (raises precision):** the основание free-text carries a number equal to `value_delta` in a *total* context — keywords `обща|общата|крайна|краен|възлиза|става` near the figure (the issue found 355 such records). Available fields at the raw/derive stage: `raw_amendments.description` (`changeDescription`), `reason` (`changeReason`), `circumstances` (`changeReasonDescription`) — see §5 note. Parse Bulgarian number formats (`1 234 567,89` / `1234567.89`), compare to `value_delta` within a small relative tolerance. +3. **Currency-change tell (special-case, very high precision):** exactly-+100% rows whose text mentions `евро|валута|EUR|лева в евро` are currency re-denominations with the total doubled (e.g. 189325). Treat as confirmed total. + +Classification: +- **Confirmed total** = gate (1) AND (text (2) or (3)). → correct (Tier 2) and/or flag. +- **Suspected total** = gate (1) only (no text confirmation). → flag-only (Tier 1); do not silently rewrite the value. + +--- + +## 4. The fix + +Two tiers. Ship Tier 1 first (safe, immediate); Tier 2 is the higher-value correction and needs the text heuristic hardened by tests. + +### Tier 1 — Flag and exclude from aggregates (minimal, safe, ships first) +Mirror the `annex_suspect` machinery so the ~475M/€ inflation leaves every rollup immediately, without trusting any text parse. +- Add a new `value_flag` verdict, e.g. **`annex_total_suspect`**, in `scripts/normalize-raw.sql` (and the `refresh-slice.sql` mirror + its reconciliation re-flag), placed **before** the `ELSE 'ok'`: fires when the contract's current-value-driving annex satisfies the arithmetic gate (§3.1) and the source-consistency check (`value_after ≈ value_before + value_delta`). +- Route it through the existing suspect fallback: `amount_eur`/`trusted_native` fall back to `signing_value` (`normalize-raw.sql:818-830`) and `precompute.sql:36` NULLs `current_value_eur` — so these contracts drop out of totals/CSV/pages exactly like `annex_suspect`. +- Emit a diagnostic count (like #286's diagnostics) so the flagged volume is observable in the ETL log. +- **Per-amendment flag (the issue's specific gap):** the contract flag does not fix the timeline row. Add a per-amendment marker so `queries/details.ts` can render the row as "suspected re-stated total" and suppress its `+%`. Options: a `value_flag`/`total_restated` column on served `amendments` (schema migration + carry through `promote-amendments.sql`), or recompute the same predicate in the details query. Prefer the column (single source of truth, avoids duplicating the heuristic in TS). + +### Tier 2 — Correct the value (higher value, needs text confirmation) +For **confirmed totals** (§3), restate the amendment at the **raw/derive stage** (where `value_before`, `value_delta`, and all three text fields coexist — see §5): +- Because the source is self-consistent, the correction is simply **`value_after := value_delta`** (the announced new total) and **`value_delta := value_after_old − value_before`** *no* — restate as: `corrected_after = value_delta_source` (the total); `corrected_delta = corrected_after − value_before` (the true increase). Keep the raw source values immutable in `raw_amendments`; write corrected values on the way to served `amendments` (a `promote`/derive transform), plus a `total_restated = 1` marker. +- `contracts.current_value` then derives from the corrected `value_after`, so headline value, deltas, and the timeline are all right — not merely excluded. +- Keep Tier-1 flagging for the **suspected-but-unconfirmed** set (gate only, no text) so nothing inflates while remaining un-restated. + +### Fix location (decided) +Raw/derive stage, **not** ingest and **not** the served query layer: +- Ingest (`base.ts`) must stay a faithful mirror of the source (per `docs/etl.md` non-destructive-staging stance) — do not mutate `raw_amendments`. +- The correction/flag needs `value_before`, `value_delta`, and the основание text on one row, which is true in `raw_amendments` and consumed by `derive-amendments.sql` / `normalize-raw.sql` / `promote-amendments.sql`. Implement there; keep `derive-amendments.sql` and `refresh-slice.sql` in lockstep (a drift guard already exists for the #286 bridge block — extend the pattern). + +--- + +## 5. Schema / data note (blocking for Tier 2) +Only `description` survives to served `amendments`; `reason` and `circumstances` are dropped at `promote-amendments.sql:9-11` (served DDL `0000_init.sql:169-182`). The text heuristic therefore must run at the **raw/derive** stage where all three exist (`work-staging-schema.sql:178-180`). If any per-row flag or corrected value must be *visible* to the app, add the column(s) to the served `amendments` table (migration) and carry them through `promote-amendments.sql` + the `refresh-slice.sql` amendments promotion. + +--- + +## 6. Testing — proving the fix +1. **Unit — number/keyword parser** (`packages/ingest` or a new `packages/db` SQL-driven test): Bulgarian number formats; total-context keywords vs increment phrasing; the currency-change tell. Fixtures from the real examples (145652 "възлезе на 539 240.00 лв."; 189325 currency change; a genuine >100% *increment* that must NOT be corrected). +2. **End-to-end SQL** (extend `packages/db/src/refresh-slice.test.ts` / a new `amendments-total-suspect.test.ts`): run the real `derive-amendments.sql → normalize-raw.sql → promote-amendments.sql` and assert: (a) a confirmed-total annex is restated (`value_after = value_delta`, `current_value` correct, `total_restated = 1`); (b) a suspected-only annex is flagged `annex_total_suspect` and excluded from `amount_eur`; (c) a genuine >100% increment stays `ok` and untouched (guard against false positives); (d) the exactly-+100% currency case restates to no real growth. +3. **Flag-coverage regression**: assert the new verdict count on a seeded corpus, and that `value-flag-annex-step-sql.test.ts`'s existing cases are unchanged. +4. **Real-corpus before/after**: rebuild the 2020→2026 corpus (local work-DB, then optionally ship), and report flagged/corrected counts + the EUR removed from `company_totals`/`authority_totals`/`home_totals`. Target: the 686 (≥100%) restated or flagged, headline totals drop by the double-counted amount, zero genuine-increment regressions. + +--- + +## 7. Scope boundaries & risks +- **In scope:** EOP annexes (the sole driver — OCDS rows carry `value_after = NULL` and never drive `current_value`, per #286). +- **Blind spot (out of scope for v1, must be documented):** double-counts where the new total is *lower* than the old value (growth < 100%) — not detectable by the arithmetic gate. Text-only detection could reach them but at higher false-positive cost; defer. +- **False-positive risk (the main hazard):** a genuine annex that legitimately more-than-doubles a contract. Tier 1 only *flags* (reversible, excludes from aggregates); Tier 2 *rewrites* and must require text confirmation + be unit-tested against a real >100%-increment fixture. When uncertain, prefer flag over rewrite. +- **Adjacent / not this:** #299 (`c44a7ee`) and #248 handle the ≥10×/≥5× mis-keying case (kept); #245 (EUR double-conversion) and #304/#247 (стотинки) are separate. This defect is the sub-5× band those cannot reach. +- **Consumer follow-through:** once `/anomalies` (#239) and `/overruns` (#171) land, confirm they read a value base that already excludes/corrects these (they will, if they use `amount_eur` + `value_flag`). diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index d1d9f709..84009da8 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -252,6 +252,9 @@ export interface ContractValueTimeline { currentEur: number | null; deltaPct: number | null; // (current − signing) / signing, when both present suspect: boolean; // value_/annex_suspect/review → render with an unverified-value label + // annex_total_suspect → the current value is a KNOWN exact 2× double-count. currentEur is blanked (—) + // rather than shown as a labelled doubled figure — a known-wrong number is worse than an honest gap (#307). + currentValueDoubled: boolean; } export interface ContractLotRow { @@ -282,6 +285,8 @@ export interface AmendmentEntry { description: string | null; // recorded reason/notes, when the source carries them valueAfterEur: number | null; // the contract value after this annex deltaEur: number | null; // value_after − value_before + restated: boolean; // #305 Tier-2: value_after was text-corrected from a double-counted total + suspect: boolean; // #305 residual: an uncorrectable double-count — value_after/delta suppressed, row marked } export interface ContractDetail { diff --git a/packages/db/migrations/0000_init.sql b/packages/db/migrations/0000_init.sql index 90f98dac..57002fd3 100644 --- a/packages/db/migrations/0000_init.sql +++ b/packages/db/migrations/0000_init.sql @@ -125,13 +125,13 @@ CREATE TABLE contracts ( bids_received INTEGER, contract_kind TEXT, -- Доставки / Услуги / Строителство awarded_to_group INTEGER, -- this AWARD went to an обединение (per-contract, distinct from bidders.is_consortium) - value_flag TEXT NOT NULL DEFAULT 'ok', -- ok | review | value_low | value_suspect | annex_suspect (data-quality verdict; assigned in scripts/normalize-raw.sql) + value_flag TEXT NOT NULL DEFAULT 'ok', -- ok | review | value_low | value_suspect | annex_suspect | annex_total_suspect (data-quality verdict; assigned in scripts/normalize-raw.sql) date_flag TEXT NOT NULL DEFAULT 'ok', -- ok | signed_after_publication (non-destructive date-quality verdict) amount_eur REAL, -- canonical EUR, SAFE TO SUM; populated for all flags (value_suspect repaired to the procedure estimate); NULL only when no trustworthy EUR figure (FX-rateless foreign / value_suspect w/o estimate / no signing+current) fx_converted INTEGER NOT NULL DEFAULT 0, -- 1 = amount_eur came from a foreign-currency market rate fx_rate REAL, -- EUR per 1 unit of `currency` for foreign rows (amount × fx_rate = amount_eur) signing_value_eur REAL, -- signing_value in EUR (peg/fx); NULL for value_suspect — for the contract value timeline - current_value_eur REAL, -- current_value in EUR; NULL for value_suspect/annex_suspect (suspect annex suppressed) + current_value_eur REAL, -- current_value in EUR; NULL for value_suspect/annex_suspect/annex_total_suspect (suspect annex suppressed) lot_id TEXT, -- domain lot id ('lot:'||УНП||':'||raw) when the award is lot-scoped; soft-links lots(id) document_number TEXT, -- Номер на документ published_at TEXT, -- Публикуван на diff --git a/packages/db/migrations/0006_amendment_restated.sql b/packages/db/migrations/0006_amendment_restated.sql new file mode 100644 index 00000000..5f158067 --- /dev/null +++ b/packages/db/migrations/0006_amendment_restated.sql @@ -0,0 +1,17 @@ +-- #305 Tier-2 text-based value correction (packages/ingest/src/amendment-total.ts). Some ЦАИС ЕОП +-- annexes put the announced NEW TOTAL into the change field, doubling value_after. The основание text +-- resolves each: a restated total drives the corrected value_after (and current_value), a genuine +-- increment is confirmed correct. The served amendments row records the outcome so the UI can mark a +-- corrected row and the refresh-slice reconciliation can skip text-treated annexes when arithmetic-flagging. +-- +-- value_restated = 1 when value_after was rewritten to the text-confirmed true total, else 0. +-- value_treatment = the raw treatment label ('total_restated' / 'unchanged_restated' / +-- 'genuine_increment', NULL when the text carried no signal). Kept alongside +-- value_restated because the slice reconciliation re-classifies from the served +-- amendments and must skip confirmed-genuine increments (value_restated stays 0 there). +-- Additive columns. On the live stage DB (whose migration ledger is empty — base schema imported +-- out-of-band) these are applied by the column probe in .github/workflows/deploy.yml, which ALTERs only +-- when the column is missing; on a fresh ledger `d1 migrations apply` runs this file exactly once. SQLite +-- has no `ADD COLUMN IF NOT EXISTS`, so do not replay this file against a DB that already has the columns. +ALTER TABLE amendments ADD COLUMN value_restated INTEGER NOT NULL DEFAULT 0; +ALTER TABLE amendments ADD COLUMN value_treatment TEXT; diff --git a/packages/db/migrations/0007_amendment_value_suspect.sql b/packages/db/migrations/0007_amendment_value_suspect.sql new file mode 100644 index 00000000..faa1b195 --- /dev/null +++ b/packages/db/migrations/0007_amendment_value_suspect.sql @@ -0,0 +1,14 @@ +-- #305 residual: a contract flagged value_flag = 'annex_total_suspect' has its current_value excluded +-- from every aggregate, but the served amendments row still carried the DOUBLED value_after — so the +-- amendment timeline kept showing the untrusted figure. Only text-corrected rows (value_restated) had +-- their served value rewritten; the ~183 flag-only doubles (non-exact >2×, no основание signal) did not. +-- We do NOT know their true total, so we MARK the row and let the UI SUPPRESS the untrusted figure — +-- never invent a number. +-- +-- value_suspect = 1 when the served amendment is a suspected double-count NOT already text-treated +-- (value_treatment IS NULL), else 0. A restated/genuine row (value_treatment set) is +-- never also suspect. The UI blanks value_after/delta for a suspect row. +-- Additive column. Applied on the live stage DB by the column probe in .github/workflows/deploy.yml (ALTER +-- only when missing); on a fresh ledger `d1 migrations apply` runs it once. SQLite has no `ADD COLUMN IF +-- NOT EXISTS`, so do not replay this file against a DB that already has the column. +ALTER TABLE amendments ADD COLUMN value_suspect INTEGER NOT NULL DEFAULT 0; diff --git a/packages/db/src/amendments-ocds-link.test.ts b/packages/db/src/amendments-ocds-link.test.ts index af0eac30..73dd62a5 100644 --- a/packages/db/src/amendments-ocds-link.test.ts +++ b/packages/db/src/amendments-ocds-link.test.ts @@ -13,6 +13,9 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const initSchema = resolve(root, 'packages/db/migrations/0000_init.sql'); +// #305 Tier-2: promote-amendments.sql writes value_restated/value_treatment to served amendments. +const migration6 = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7 = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); const workStagingSchema = resolve(root, 'scripts/work-staging-schema.sql'); const deriveAmendments = resolve(root, 'scripts/derive-amendments.sql'); const promoteAmendments = resolve(root, 'scripts/promote-amendments.sql'); @@ -72,6 +75,8 @@ beforeEach(() => { dir = mkdtempSync(resolve(tmpdir(), 'amendments-ocds-')); db = resolve(dir, 'work.sqlite'); readScript(db, initSchema); // served `amendments` + `contracts` + readScript(db, migration6); // #305 Tier-2 value_restated/value_treatment on served amendments + readScript(db, migration7); // #305 residual value_suspect on served amendments readScript(db, workStagingSchema); // raw_* staging // Two EOP procedures: T1 (contract 90029) already has an EOP annex; T2 (contract 55500) has NONE. diff --git a/packages/db/src/amendments-sql.test.ts b/packages/db/src/amendments-sql.test.ts index 2d161410..75ceea4b 100644 --- a/packages/db/src/amendments-sql.test.ts +++ b/packages/db/src/amendments-sql.test.ts @@ -16,6 +16,10 @@ import { AMENDMENTS_SQL } from './queries/details'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); +// #305 Tier-2: AMENDMENTS_SQL selects am.value_restated, added to served amendments by this migration. +const migration6 = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +// #305 residual: AMENDMENTS_SQL also selects am.value_suspect, added by this migration. +const migration7 = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); @@ -65,6 +69,8 @@ function withDb(fn: (dbPath: string) => T): T { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, migration0); + readScript(dbPath, migration6); + readScript(dbPath, migration7); return fn(dbPath); } finally { rmSync(dir, { recursive: true, force: true }); diff --git a/packages/db/src/amendments-total-restated.test.ts b/packages/db/src/amendments-total-restated.test.ts new file mode 100644 index 00000000..5909f710 --- /dev/null +++ b/packages/db/src/amendments-total-restated.test.ts @@ -0,0 +1,301 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +// #305 Tier-2 text-based value correction. Some ЦАИС ЕОП annexes put the announced NEW TOTAL into the +// change field, so the feed's value_after is doubled. The основание-text heuristic (computed in TS +// ingest, packages/ingest/src/amendment-total.ts) classifies each annex and — because the correction is +// computed BEFORE the SQL runs — the raw row lands with value_treatment + value_after_restated already +// set. This suite simulates that ingest output and drives the REAL derive → normalize/refresh-slice → +// promote → precompute scripts in pipeline order on a real SQLite DB, asserting: +// (a) a total_restated annex drives current_value + served value_after with the corrected total and is +// NOT annex_total_suspect; +// (b) a genuine_increment annex is not flagged and keeps its (larger, correct) value_after; +// (c) an untreated doubled annex still gets the Tier-1 annex_total_suspect flag; +// (d) full-vs-slice parity for the restated contract. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const derivePath = resolve(root, 'scripts/derive-amendments.sql'); +const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); +const promotePath = resolve(root, 'scripts/promote-amendments.sql'); +const precomputePath = resolve(root, 'scripts/precompute.sql'); +const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); + +const etlRuns = [ + ['normalize-raw', [derivePath, normalizePath, promotePath, precomputePath]], + ['refresh-slice', [derivePath, refreshSlicePath, precomputePath]], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-totalrestated-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +const AUTH_EIK = '000695114'; +const BIDDER_EIK = '831646048'; + +interface AmendmentStep { + before: number | null; + after: number; + publishedAt: string; + treatment: string | null; + restatedAfter: number | null; +} + +interface Case { + unp: string; + signing: number; + steps: AmendmentStep[]; +} + +function seedContracts(dbPath: string, cases: Case[]): void { + const tenders = cases + .map( + (c) => + `('eop:tenders:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'public', ${c.signing}, 'BGN')`, + ) + .join(',\n'); + const contracts = cases + .map( + (c) => + `('eop:contracts:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'C-${c.unp}', '2026-06-01', ${c.signing}, 'BGN', '${BIDDER_EIK}', 'Тестов изпълнител')`, + ) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type, estimated_value, currency) + VALUES ${tenders}; + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, contractor_eik, contractor_name) + VALUES ${contracts};`, + ); +} + +// Seed raw_amendments with value_treatment + value_after_restated already populated — simulating the TS +// ingest output (base.ts runs the amendment-total.ts heuristic before staging). +function seedAmendments(dbPath: string, cases: Case[]): void { + const rows = cases + .flatMap((c) => + c.steps.map( + (s, i) => + `('eop:annexes:${c.unp}', '2026-06-01T00:00:00Z', 'A-${c.unp}-${i + 1}', '${c.unp}', 'C-${c.unp}', '${s.publishedAt}', ${s.before ?? 'NULL'}, ${s.after}, 'BGN', ${s.treatment === null ? 'NULL' : `'${s.treatment}'`}, ${s.restatedAfter ?? 'NULL'})`, + ), + ) + .join(',\n'); + if (!rows) return; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, fetched_at, document_number, unp, contract_number, published_at, + value_before, value_after, currency, value_treatment, value_after_restated) + VALUES ${rows};`, + ); +} + +interface ContractRow { + id: string; + value_flag: string; + current_value: number | null; +} + +const contractsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT id, value_flag, current_value FROM contracts`, + ); + const out = new Map(); + for (const r of rows) out.set(r.id.split(':')[2]!, r); + return out; +}; + +interface AmendmentRow { + unp: string; + value_after: number | null; + value_delta: number | null; + value_restated: number | null; +} + +const amendmentsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT unp, value_after, value_delta, value_restated FROM amendments`, + ); + const out = new Map(); + for (const r of rows) out.set(r.unp, r); + return out; +}; + +// (a) total_restated: doubled value_after (981240) but the основание text announced the true total +// (539240). Ingest set value_after_restated=539240, value_treatment='total_restated'. +const RESTATED: Case = { + unp: 'UNP-RESTATED', + signing: 442_000, + steps: [ + { + before: 442_000, + after: 981_240, + publishedAt: '2026-06-10', + treatment: 'total_restated', + restatedAfter: 539_240, + }, + ], +}; + +// (b) genuine_increment: value_after (60226.85) is a real ≥2× increase already applied; the text confirmed +// it, so ingest set value_treatment='genuine_increment' with restatedAfter NULL. Must NOT be flagged. +const GENUINE: Case = { + unp: 'UNP-GENUINE', + signing: 10_226.85, + steps: [ + { + before: 10_226.85, + after: 60_226.85, + publishedAt: '2026-06-10', + treatment: 'genuine_increment', + restatedAfter: null, + }, + ], +}; + +// (c) untreated doubled annex: no text signal, before ≈ signing, same currency → Tier-1 unchanged. +const UNTREATED: Case = { + unp: 'UNP-DOUBLE', + signing: 77_000_000, + steps: [ + { + before: 77_000_000, + after: 154_000_000, + publishedAt: '2026-06-10', + treatment: null, + restatedAfter: null, + }, + ], +}; + +describe('#305 Tier-2 text-based amendment value correction', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: a total_restated annex drives the corrected total and is not flagged`, () => { + withEtlDb(label, (dbPath) => { + seedContracts(dbPath, [RESTATED]); + seedAmendments(dbPath, [RESTATED]); + for (const p of scriptPaths) readScript(dbPath, p); + + const contract = contractsByUnp(dbPath).get('UNP-RESTATED'); + expect(contract?.value_flag, 'restated annex is not arithmetic-flagged').toBe('ok'); + expect( + contract?.current_value, + 'current_value is the corrected total, NOT the doubled value', + ).toBe(539_240); + + const amendment = amendmentsByUnp(dbPath).get('UNP-RESTATED'); + expect(amendment?.value_after, 'served value_after is the corrected total').toBe(539_240); + expect(amendment?.value_delta, 'served delta is self-consistent (after − before)').toBe( + 539_240 - 442_000, + ); + expect(amendment?.value_restated, 'served row is marked restated').toBe(1); + }); + }); + + it(`${label}: a genuine_increment annex is not flagged and keeps its value_after`, () => { + withEtlDb(label, (dbPath) => { + seedContracts(dbPath, [GENUINE]); + seedAmendments(dbPath, [GENUINE]); + for (const p of scriptPaths) readScript(dbPath, p); + + const contract = contractsByUnp(dbPath).get('UNP-GENUINE'); + expect(contract?.value_flag, 'confirmed-genuine increment is not flagged').not.toBe( + 'annex_total_suspect', + ); + expect(contract?.current_value, 'current_value keeps the genuine increase').toBe(60_226.85); + + const amendment = amendmentsByUnp(dbPath).get('UNP-GENUINE'); + expect(amendment?.value_after, 'served value_after unchanged').toBe(60_226.85); + expect(amendment?.value_restated, 'genuine increment is not marked restated').toBe(0); + }); + }); + + it(`${label}: an untreated doubled annex still gets the Tier-1 annex_total_suspect flag`, () => { + withEtlDb(label, (dbPath) => { + seedContracts(dbPath, [UNTREATED]); + seedAmendments(dbPath, [UNTREATED]); + for (const p of scriptPaths) readScript(dbPath, p); + + const contract = contractsByUnp(dbPath).get('UNP-DOUBLE'); + expect(contract?.value_flag, 'untreated double is still flagged').toBe( + 'annex_total_suspect', + ); + + const amendment = amendmentsByUnp(dbPath).get('UNP-DOUBLE'); + expect(amendment?.value_restated, 'untreated double is not marked restated').toBe(0); + }); + }); + } + + it('full-vs-slice parity: the total_restated contract resolves identically on both paths', () => { + let full: ContractRow | undefined; + withEtlDb('parity-full', (dbPath) => { + seedContracts(dbPath, [RESTATED]); + seedAmendments(dbPath, [RESTATED]); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + full = contractsByUnp(dbPath).get('UNP-RESTATED'); + }); + + let slice: ContractRow | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedContracts(dbPath, [RESTATED]); + seedAmendments(dbPath, [RESTATED]); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + slice = contractsByUnp(dbPath).get('UNP-RESTATED'); + }); + + // Pin both paths to the concrete expected values — a cross-equality (full === slice) would also + // pass on dual-undefined, so assert against the literal on each path instead. + expect(full?.value_flag).toBe('ok'); + expect(slice?.value_flag).toBe('ok'); + expect(full?.current_value).toBe(539_240); + expect(slice?.current_value).toBe(539_240); + }); +}); diff --git a/packages/db/src/amendments-total-suspect.test.ts b/packages/db/src/amendments-total-suspect.test.ts new file mode 100644 index 00000000..2fa3c913 --- /dev/null +++ b/packages/db/src/amendments-total-suspect.test.ts @@ -0,0 +1,708 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +// #305 single-annex value double-count: a driving annex whose value_after is >=2x its value_before is +// a data defect — ЗОП чл.116 caps a single amendment at +50%, so one step cannot legally more than +// double a contract. Such contracts get value_flag = 'annex_total_suspect' and fall back to +// signing_value, exactly like annex_suspect, so the doubled figure is excluded from every EUR +// aggregate. The ABS(value_after - current_value) tie binds the flag to the annex that DRIVES +// current_value: a doubled annex later superseded by a correct one is NOT flagged. +// +// The flag CASE and its value-fallback siblings live in copies across both derive paths (normalize-raw +// and refresh-slice) plus refresh-slice's reconciliation re-flag, so the rule is exercised through the +// REAL scripts in pipeline order on a real SQLite database. A copy left behind fails here. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +// #305 residual: served amendments gained value_suspect (promote + refresh-slice write it). +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const derivePath = resolve(root, 'scripts/derive-amendments.sql'); +const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); +const promotePath = resolve(root, 'scripts/promote-amendments.sql'); +const precomputePath = resolve(root, 'scripts/precompute.sql'); +const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); + +// Real pipeline order (scripts/import.mjs): full derive runs derive-amendments → normalize-raw → +// promote-amendments → precompute; the slice derive runs derive-amendments → refresh-slice (which +// promotes the window's amendments itself) → precompute. precompute populates current_value_eur, which +// these assertions read. +const etlRuns = [ + ['normalize-raw', [derivePath, normalizePath, promotePath, precomputePath]], + ['refresh-slice', [derivePath, refreshSlicePath, precomputePath]], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-totalsuspect-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +// Signing value in BGN; exactly 100_000 EUR at the fixed peg (÷1.95583) so the fallback reads directly. +const SIGNING_BGN = 195_583; +const AUTH_EIK = '000695114'; +const BIDDER_EIK = '831646048'; + +interface AmendmentStep { + before: number | null; + after: number; + publishedAt: string; +} + +interface Case { + unp: string; + signing: number; + steps: AmendmentStep[]; +} + +function seedContracts(dbPath: string, cases: Case[]): void { + const tenders = cases + .map( + (c) => + `('eop:tenders:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'public', ${c.signing}, 'BGN')`, + ) + .join(',\n'); + const contracts = cases + .map( + (c) => + `('eop:contracts:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'C-${c.unp}', '2026-06-01', ${c.signing}, 'BGN', '${BIDDER_EIK}', 'Тестов изпълнител')`, + ) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type, estimated_value, currency) + VALUES ${tenders}; + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, contractor_eik, contractor_name) + VALUES ${contracts};`, + ); +} + +function seedAmendments(dbPath: string, cases: Case[]): void { + const rows = cases + .flatMap((c) => + c.steps.map( + (s, i) => + `('eop:annexes:${c.unp}', '2026-06-01T00:00:00Z', 'A-${c.unp}-${i + 1}', '${c.unp}', 'C-${c.unp}', '${s.publishedAt}', ${s.before ?? 'NULL'}, ${s.after}, 'BGN')`, + ), + ) + .join(',\n'); + if (!rows) return; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, fetched_at, document_number, unp, contract_number, published_at, + value_before, value_after, currency) + VALUES ${rows};`, + ); +} + +interface Row { + id: string; + value_flag: string; + amount_eur: number | null; + current_value_eur: number | null; +} + +const rowsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT id, value_flag, ROUND(amount_eur) AS amount_eur, + ROUND(current_value_eur) AS current_value_eur + FROM contracts`, + ); + const out = new Map(); + for (const r of rows) out.set(r.id.split(':')[2]!, r); + return out; +}; + +// #305 residual: seed raw_amendments including value_treatment + value_after_restated (the TS ingest +// output) so the served value_suspect / value_restated marks can be asserted end-to-end. +interface TreatedStep { + before: number | null; + after: number; + publishedAt: string; + treatment: string | null; + restatedAfter: number | null; +} +interface TreatedCase { + unp: string; + signing: number; + steps: TreatedStep[]; +} + +function seedTreatedContracts(dbPath: string, cases: TreatedCase[]): void { + seedContracts( + dbPath, + cases.map((c) => ({ unp: c.unp, signing: c.signing, steps: [] })), + ); +} + +function seedTreatedAmendments(dbPath: string, cases: TreatedCase[]): void { + const rows = cases + .flatMap((c) => + c.steps.map( + (s, i) => + `('eop:annexes:${c.unp}', '2026-06-01T00:00:00Z', 'A-${c.unp}-${i + 1}', '${c.unp}', 'C-${c.unp}', '${s.publishedAt}', ${s.before ?? 'NULL'}, ${s.after}, 'BGN', ${s.treatment === null ? 'NULL' : `'${s.treatment}'`}, ${s.restatedAfter ?? 'NULL'})`, + ), + ) + .join(',\n'); + if (!rows) return; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, fetched_at, document_number, unp, contract_number, published_at, + value_before, value_after, currency, value_treatment, value_after_restated) + VALUES ${rows};`, + ); +} + +interface ServedAmendment { + unp: string; + value_after: number | null; + value_suspect: number | null; + value_restated: number | null; +} +const servedByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT unp, value_after, value_suspect, value_restated FROM amendments`, + ); + const out = new Map(); + for (const r of rows) out.set(r.unp, r); + return out; +}; + +// (a) flag-only double: no основание signal (value_treatment NULL), before ≈ signing, value_after = 3× +// before. We can't bridge the true total → mark value_suspect = 1, keep value_restated = 0, and the +// served value_after stays the untrusted figure (the UI blanks it; the number is never rewritten). +const FLAG_ONLY: TreatedCase = { + unp: 'UNP-FLAGONLY', + signing: 100, + steps: [ + { before: 100, after: 300, publishedAt: '2026-06-10', treatment: null, restatedAfter: null }, + ], +}; +// (b) total_restated: text-treated → value_restated = 1, value_suspect = 0. +const RESTATED: TreatedCase = { + unp: 'UNP-RESTATED', + signing: 442_000, + steps: [ + { + before: 442_000, + after: 981_240, + publishedAt: '2026-06-10', + treatment: 'total_restated', + restatedAfter: 539_240, + }, + ], +}; +// (c) genuine_increment: text-confirmed real increase → both marks 0. +const GENUINE: TreatedCase = { + unp: 'UNP-GENUINE', + signing: 10_226.85, + steps: [ + { + before: 10_226.85, + after: 60_226.85, + publishedAt: '2026-06-10', + treatment: 'genuine_increment', + restatedAfter: null, + }, + ], +}; + +describe('#305 residual: per-amendment value_suspect marker on the served row', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: a flag-only double is marked value_suspect=1, value_restated=0, value_after untouched`, () => { + withEtlDb(label, (dbPath) => { + seedTreatedContracts(dbPath, [FLAG_ONLY, RESTATED, GENUINE]); + seedTreatedAmendments(dbPath, [FLAG_ONLY, RESTATED, GENUINE]); + for (const p of scriptPaths) readScript(dbPath, p); + + const served = servedByUnp(dbPath); + + const flagOnly = served.get('UNP-FLAGONLY'); + expect(flagOnly?.value_suspect, 'flag-only double is marked suspect').toBe(1); + expect(flagOnly?.value_restated, 'flag-only double is NOT restated').toBe(0); + expect(flagOnly?.value_after, 'served value_after is never rewritten').toBe(300); + + const restated = served.get('UNP-RESTATED'); + expect(restated?.value_restated, 'total_restated row is marked restated').toBe(1); + expect(restated?.value_suspect, 'a restated row is never also suspect').toBe(0); + + const genuine = served.get('UNP-GENUINE'); + expect(genuine?.value_suspect, 'genuine increment is not suspect').toBe(0); + expect(genuine?.value_restated, 'genuine increment is not restated').toBe(0); + }); + }); + } + + it('full-vs-slice parity: the flag-only double gets value_suspect=1 on both paths', () => { + let full: ServedAmendment | undefined; + withEtlDb('parity-full', (dbPath) => { + seedTreatedContracts(dbPath, [FLAG_ONLY]); + seedTreatedAmendments(dbPath, [FLAG_ONLY]); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + full = servedByUnp(dbPath).get('UNP-FLAGONLY'); + }); + + let slice: ServedAmendment | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedTreatedContracts(dbPath, [FLAG_ONLY]); + seedTreatedAmendments(dbPath, [FLAG_ONLY]); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + slice = servedByUnp(dbPath).get('UNP-FLAGONLY'); + }); + + // Pin both paths to the concrete expected value — a cross-equality (full === slice) would also pass + // on dual-undefined, so assert against the literal on each path instead. + expect(full?.value_suspect).toBe(1); + expect(slice?.value_suspect).toBe(1); + }); +}); + +describe('#305 annex_total_suspect single-annex value double-count', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags a doubled driving annex and falls back to the signing value`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // (a) The bug: 77M signed, one annex reports value_after = 2x value_before (154M). ЗОП caps a + // single amendment at +50%, so this is a double-count defect: flag it and drop back to signing. + { + unp: 'UNP-DOUBLE', + signing: 77_000_000, + steps: [{ before: 77_000_000, after: 154_000_000, publishedAt: '2026-06-10' }], + }, + // (b) A genuine small increase (+30%): before 100 → after 130. Must stay 'ok' and keep its + // current value, guarding against false positives. + { + unp: 'UNP-SMALL', + signing: 100, + steps: [{ before: 100, after: 130, publishedAt: '2026-06-10' }], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const rows = rowsByUnp(dbPath); + + // (a) flagged, value base falls back to signing (77M BGN ÷1.95583), NOT the doubled 154M. + const doubled = rows.get('UNP-DOUBLE'); + expect(doubled?.value_flag, 'doubled annex flagged').toBe('annex_total_suspect'); + const signingEur = Math.round(77_000_000 / 1.95583); + expect(doubled?.amount_eur, 'amount_eur falls back to signing').toBe(signingEur); + // Excluded from the current_value aggregate entirely. + expect(doubled?.current_value_eur, 'current_value_eur suppressed').toBeNull(); + + // (b) genuine +30% increase untouched: stays ok, value reflects current_value (130 BGN). + const small = rows.get('UNP-SMALL'); + expect(small?.value_flag, 'small increase stays ok').toBe('ok'); + expect(small?.amount_eur, 'small increase keeps current value').toBe( + Math.round(130 / 1.95583), + ); + expect(small?.current_value_eur).toBe(Math.round(130 / 1.95583)); + }); + }); + + it(`${label}: an old doubled annex superseded by a correct later annex is NOT flagged`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // (d) An early annex doubled (before 100 → after 200), but a LATER annex sets a correct, + // sub-2x current_value (before 200 → after 130). The doubled step no longer DRIVES + // current_value, so the ABS(...-current_value) tie must leave the contract 'ok'. + { + unp: 'UNP-SUPERSEDED', + signing: 100, + steps: [ + { before: 100, after: 200, publishedAt: '2026-06-10' }, + { before: 200, after: 130, publishedAt: '2026-06-20' }, + ], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const row = rowsByUnp(dbPath).get('UNP-SUPERSEDED'); + expect(row?.value_flag, 'superseded double is not flagged').toBe('ok'); + // Served at the corrected current value (130 BGN), not the transient doubled 200. + expect(row?.amount_eur).toBe(Math.round(130 / 1.95583)); + expect(row?.current_value_eur).toBe(Math.round(130 / 1.95583)); + }); + }); + } + + it('full-vs-slice parity: the doubled contract gets the same value_flag on both paths', () => { + const cases: Case[] = [ + { + unp: 'UNP-DOUBLE', + signing: 77_000_000, + steps: [{ before: 77_000_000, after: 154_000_000, publishedAt: '2026-06-10' }], + }, + ]; + + let fullFlag: string | undefined; + withEtlDb('parity-full', (dbPath) => { + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + fullFlag = rowsByUnp(dbPath).get('UNP-DOUBLE')?.value_flag; + }); + + let sliceFlag: string | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + sliceFlag = rowsByUnp(dbPath).get('UNP-DOUBLE')?.value_flag; + }); + + expect(fullFlag).toBe('annex_total_suspect'); + expect(sliceFlag).toBe('annex_total_suspect'); + expect(fullFlag).toBe(sliceFlag); + }); +}); + +// #305 multi-annex residual: the doubled step is NOT always the first annex. When a later annex reports +// a new TOTAL added to an already-grown value, its value_before is the prior CUMULATIVE total (a preceding +// annex's value_after), not signing. The relaxed anchor flags these too; the ≥2× single-step gate (ЗОП +// чл.116) keeps slow legitimate climbs — whose later steps never reach 2× — untouched. +describe('#305 annex_total_suspect multi-annex value double-count', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags a later-in-chain double whose value_before ties to a prior annex total, not signing`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // Chain: 1M signed → annex1 +40% (1.4M, legal, not a double) → annex2 doubles the 1.4M total to + // 2.8M. The driving annex's value_before (1.4M) equals the PRIOR annex's value_after, not signing + // (1M), so the old signing-only anchor missed it. Must now flag and fall back to signing. + { + unp: 'UNP-MULTI-DOUBLE', + signing: 1_000_000, + steps: [ + { before: 1_000_000, after: 1_400_000, publishedAt: '2026-06-10' }, + { before: 1_400_000, after: 2_800_000, publishedAt: '2026-06-20' }, + ], + }, + // Control: same shape but the later step is a legal +36% (1.4M → 1.9M), below 2×. The relaxed + // anchor matches value_before to the prior total, but the ≥2× gate must keep this 'ok'. + { + unp: 'UNP-MULTI-OK', + signing: 1_000_000, + steps: [ + { before: 1_000_000, after: 1_400_000, publishedAt: '2026-06-10' }, + { before: 1_400_000, after: 1_900_000, publishedAt: '2026-06-20' }, + ], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const rows = rowsByUnp(dbPath); + + const multiDouble = rows.get('UNP-MULTI-DOUBLE'); + expect(multiDouble?.value_flag, 'later-in-chain double flagged').toBe( + 'annex_total_suspect', + ); + expect(multiDouble?.amount_eur, 'amount_eur falls back to signing').toBe( + Math.round(1_000_000 / 1.95583), + ); + expect(multiDouble?.current_value_eur, 'current_value_eur suppressed').toBeNull(); + + const multiOk = rows.get('UNP-MULTI-OK'); + expect(multiOk?.value_flag, 'legal <2x later step stays ok').toBe('ok'); + expect(multiOk?.amount_eur, 'ok row keeps current value').toBe( + Math.round(1_900_000 / 1.95583), + ); + expect(multiOk?.current_value_eur).toBe(Math.round(1_900_000 / 1.95583)); + }); + }); + } + + // The per-row value_suspect marker (served amendments) must also catch the later-in-chain double, so the + // UI blanks that specific annex row — not just the contract-level flag. + const MULTI_TREATED: TreatedCase = { + unp: 'UNP-MULTI-SUSPECT', + signing: 1_000_000, + steps: [ + { + before: 1_000_000, + after: 1_400_000, + publishedAt: '2026-06-10', + treatment: null, + restatedAfter: null, + }, + { + before: 1_400_000, + after: 2_800_000, + publishedAt: '2026-06-20', + treatment: null, + restatedAfter: null, + }, + ], + }; + + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: marks value_suspect=1 on the later-in-chain doubled annex row, not the legal earlier one`, () => { + withEtlDb(label, (dbPath) => { + seedTreatedContracts(dbPath, [MULTI_TREATED]); + seedTreatedAmendments(dbPath, [MULTI_TREATED]); + for (const p of scriptPaths) readScript(dbPath, p); + + const served = sqliteJson<{ value_after: number; value_suspect: number }>( + dbPath, + `SELECT value_after, value_suspect FROM amendments + WHERE unp = 'UNP-MULTI-SUSPECT' ORDER BY value_after`, + ); + expect(served.length, 'both annex rows served').toBe(2); + // The legal +40% step (1.4M) is not suspect; the doubled step (2.8M) is. + const legal = served.find((r) => r.value_after === 1_400_000); + const doubled = served.find((r) => r.value_after === 2_800_000); + expect(legal?.value_suspect, 'legal earlier step not suspect').toBe(0); + expect(doubled?.value_suspect, 'later-in-chain double marked suspect').toBe(1); + }); + }); + } +}); + +// #305 NEW-HIGH-1 (multi-annex chain contamination): the double-count correction is per-row and does NOT +// propagate down a chain. A restated prior annex (doubled → corrected down) leaves a LATER annex still +// computed by the feed on the contaminated (raw, doubled) base. The later annex's own step ratio is +// legitimate (<2×) so the arithmetic gate misses it and the prior is text-treated (excluded) — yet +// current_value inherited the doubled total. The new branch flags it → signing fallback, on both paths. +describe('#305 NEW-HIGH-1 multi-annex chain contamination', () => { + // annex1 doubled 1M→2.4M, text-restated to 1.4M; annex2 is a real +15% the feed computed on the RAW 2.4M + // base (2.4M→2.76M). annex2's own ratio is 1.15× so the gate misses it; annex1 is treated so it is + // excluded. Without the fix, current_value serves the contaminated 2.76M. + const CONTAM: TreatedCase = { + unp: 'UNP-CHAIN-CONTAM', + signing: 1_000_000, + steps: [ + { + before: 1_000_000, + after: 2_400_000, + publishedAt: '2026-06-10', + treatment: 'total_restated', + restatedAfter: 1_400_000, + }, + { + before: 2_400_000, + after: 2_760_000, + publishedAt: '2026-06-20', + treatment: null, + restatedAfter: null, + }, + ], + }; + // Control: the same shape on an HONEST base — annex1 is a genuine +40% (no restatement), annex2 +15% on + // the clean 1.4M base. No treated prior, so the contamination branch must NOT fire; stays 'ok'. + const CLEAN: TreatedCase = { + unp: 'UNP-CHAIN-CLEAN', + signing: 1_000_000, + steps: [ + { + before: 1_000_000, + after: 1_400_000, + publishedAt: '2026-06-10', + treatment: null, + restatedAfter: null, + }, + { + before: 1_400_000, + after: 1_610_000, + publishedAt: '2026-06-20', + treatment: null, + restatedAfter: null, + }, + ], + }; + + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags a later annex riding a restated prior's doubled base, and keeps a clean chain ok`, () => { + withEtlDb(label, (dbPath) => { + seedTreatedContracts(dbPath, [CONTAM, CLEAN]); + seedTreatedAmendments(dbPath, [CONTAM, CLEAN]); + for (const p of scriptPaths) readScript(dbPath, p); + + const rows = rowsByUnp(dbPath); + + const contam = rows.get('UNP-CHAIN-CONTAM'); + expect(contam?.value_flag, 'contaminated later annex flagged').toBe('annex_total_suspect'); + expect(contam?.amount_eur, 'falls back to signing, not the contaminated 2.76M').toBe( + Math.round(1_000_000 / 1.95583), + ); + expect(contam?.current_value_eur, 'contaminated current_value suppressed').toBeNull(); + + const clean = rows.get('UNP-CHAIN-CLEAN'); + expect(clean?.value_flag, 'honest two-step growth stays ok').toBe('ok'); + expect(clean?.current_value_eur).toBe(Math.round(1_610_000 / 1.95583)); + }); + }); + } + + it('full-vs-slice parity: the contaminated chain gets the same flag on both paths', () => { + let fullFlag: string | undefined; + withEtlDb('parity-full', (dbPath) => { + seedTreatedContracts(dbPath, [CONTAM]); + seedTreatedAmendments(dbPath, [CONTAM]); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + fullFlag = rowsByUnp(dbPath).get('UNP-CHAIN-CONTAM')?.value_flag; + }); + + let sliceFlag: string | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedTreatedContracts(dbPath, [CONTAM]); + seedTreatedAmendments(dbPath, [CONTAM]); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + sliceFlag = rowsByUnp(dbPath).get('UNP-CHAIN-CONTAM')?.value_flag; + }); + + expect(fullFlag).toBe('annex_total_suspect'); + expect(sliceFlag).toBe('annex_total_suspect'); + }); +}); + +// #305 84818-class: an EXACT single-step 2× (value_after ≈ 2× value_before) is the ЗОП чл.116 defect +// signature even when value_before anchors to NEITHER signing NOR a prior annex total (an orphan base) — +// as in real contract 84818, whose annex reports 76.77M → 153.54M on a base unrelated to the contract's +// signing. The gate flags it → signing fallback (EXCLUDE), without ever rewriting the value. +describe('#305 84818-class orphan exact-double', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags an exact 2× on an orphan base, but leaves a non-exact orphan jump ok`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // Orphan exact 2×: value_before 90 000 ties neither signing (195 583) nor any prior annex, and + // value_after is exactly 2×. Flag → signing fallback (100 000 EUR), never the doubled 180 000. + { + unp: 'UNP-ORPHAN-EXACT', + signing: 195_583, + steps: [{ before: 90_000, after: 180_000, publishedAt: '2026-06-10' }], + }, + // Control: an orphan jump that is NOT an exact 2× (1.5×) is ambiguous — with no anchor and no + // exact-double signature it must stay ok (the relaxed rule is scoped to EXACT 2× only). + { + unp: 'UNP-ORPHAN-SMALL', + signing: 195_583, + steps: [{ before: 90_000, after: 135_000, publishedAt: '2026-06-10' }], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const rows = rowsByUnp(dbPath); + + const orphan = rows.get('UNP-ORPHAN-EXACT'); + expect(orphan?.value_flag, 'orphan exact 2× flagged').toBe('annex_total_suspect'); + expect(orphan?.amount_eur, 'falls back to signing, not the doubled 180k').toBe( + Math.round(195_583 / 1.95583), + ); + expect(orphan?.current_value_eur, 'doubled current suppressed').toBeNull(); + + const small = rows.get('UNP-ORPHAN-SMALL'); + expect(small?.value_flag, 'non-exact orphan jump stays ok').toBe('ok'); + expect(small?.current_value_eur).toBe(Math.round(135_000 / 1.95583)); + }); + }); + } +}); + +// #305 NEW-HIGH-2 (reconciliation parity): the slice reconciliation reads the CUMULATIVE served +// `amendments`, whose value_after is RESTATED, while the full path anchors on RAW values. A restated prior +// annex used to flip the anchor's "prev not itself a double" test (restated 1.4M < 2×1.2M passes; the raw +// 2.4M would fail), flagging on the slice but not on the full rebuild. The `prev.value_restated = 0` guard +// restores parity: both paths reach the same verdict for a restated-prior + doubled-later chain. +describe('#305 NEW-HIGH-2 slice reconciliation parity', () => { + // annex1 doubled 1.2M→2.4M, restated to 1.4M; annex2 grows the RESTATED 1.4M to 2.9M (a ≥2× step, but + // deliberately NOT an exact 2× so the 84818-class rule doesn't fire and mask the guard under test). On the + // full (raw) path annex2's value_before (1.4M) anchors to neither signing (1M) nor the raw prior total + // (2.4M) → 'ok'. Pre-guard the slice reconciliation matched the restated 1.4M and flagged — the flip. + // Post-guard (prev.value_restated = 0): 'ok', matching the full rebuild. + const FLIP: TreatedCase = { + unp: 'UNP-RECON-FLIP', + signing: 1_000_000, + steps: [ + { + before: 1_200_000, + after: 2_400_000, + publishedAt: '2026-06-10', + treatment: 'total_restated', + restatedAfter: 1_400_000, + }, + { + before: 1_400_000, + after: 2_900_000, + publishedAt: '2026-06-20', + treatment: null, + restatedAfter: null, + }, + ], + }; + + it('full and slice agree on a restated-prior + doubled-later chain (no flag flip)', () => { + let fullFlag: string | undefined; + withEtlDb('parity-full', (dbPath) => { + seedTreatedContracts(dbPath, [FLIP]); + seedTreatedAmendments(dbPath, [FLIP]); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + fullFlag = rowsByUnp(dbPath).get('UNP-RECON-FLIP')?.value_flag; + }); + + let sliceFlag: string | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedTreatedContracts(dbPath, [FLIP]); + seedTreatedAmendments(dbPath, [FLIP]); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + sliceFlag = rowsByUnp(dbPath).get('UNP-RECON-FLIP')?.value_flag; + }); + + // The canonical full-rebuild verdict, matched by the slice (pre-guard the slice flipped to suspect). + expect(sliceFlag).toBe(fullFlag); + }); +}); diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index efd769fe..e47c775a 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -16,6 +16,16 @@ const migration3 = readFileSync( resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'), 'utf8', ); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6 = readFileSync( + resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'), + 'utf8', +); +// #305 residual: served amendments gained value_suspect (promote + refresh-slice write it). +const migration7 = readFileSync( + resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'), + 'utf8', +); const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); const normalize = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); const precompute = readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8'); @@ -67,6 +77,8 @@ function build(path: 'normalize' | 'refresh'): DatabaseSync { db.exec(schema); db.exec(migration2); db.exec(migration3); + db.exec(migration6); + db.exec(migration7); db.exec(staging); db.exec(seed); if (path === 'normalize') { diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index df67b27e..7a682ffa 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -11,6 +11,9 @@ const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // refresh-slice.sql's officials block reads interest_links (0003); build it so the script doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -40,6 +43,8 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/queries/details.test.ts b/packages/db/src/queries/details.test.ts index 9371762f..014ffa8e 100644 --- a/packages/db/src/queries/details.test.ts +++ b/packages/db/src/queries/details.test.ts @@ -195,9 +195,35 @@ describe('getContract', () => { expect(detail?.value.suspect).toBe(true); expect(detail?.value.signingEur).toBe(256.49); expect(detail?.value.currentEur).toBe(flag === 'annex_suspect' ? 1025.96 : 256.49); + expect(detail?.value.currentValueDoubled).toBe(false); } }); + it('#307 blanks the current value for a KNOWN 2× double-count (annex_total_suspect)', async () => { + const detail = await getContract( + fakeDb( + { + ...baseContractRow, + signing_value: 256.49, + current_value: 512.98, // the doubled native figure — must NOT resurface + signing_value_eur: 256.49, + current_value_eur: null, // excluded from aggregates upstream + value_flag: 'annex_total_suspect', + }, + [], + ), + 'c:1', + ); + + expect(detail?.value.suspect).toBe(true); + expect(detail?.value.currentValueDoubled).toBe(true); + // Blanked (—), never the doubled 512.98 nor a fabricated fallback. + expect(detail?.value.currentEur).toBeNull(); + expect(detail?.value.deltaPct).toBeNull(); + // The trustworthy signing value is still shown. + expect(detail?.value.signingEur).toBe(256.49); + }); + // Exercises the real cohort path end-to-end (baseContractRow is clean-value, CPV '72', amount 5000). // Guards the argument order into contractCohort: swapping value_flag ↔ division would make it return // null and this would fail. @@ -351,6 +377,38 @@ describe('getContract', () => { }); }); + it('#305 residual: suppresses value_after and delta for a suspect (uncorrectable double-count) annex', async () => { + const detail = await getContract( + fakeDb( + { ...baseContractRow, contract_number: 'C-6' }, + [], + [ + { + value_before: 1000, + value_after: 3000, // the source's untrusted doubled/tripled total + value_delta: 2000, + currency: 'EUR', + published_at: '2024-03-01', + document_number: 'A1', + description: 'Изменение на стойността', + value_restated: 0, + value_suspect: 1, + fx_rate: null, + }, + ], + ), + 'c:1', + ); + + expect(detail?.amendments[0]).toMatchObject({ + valueAfterEur: null, // suppressed — we don't stand behind the doubled figure + deltaEur: null, + suspect: true, + restated: false, + description: 'Изменение на стойността', // description still shown + }); + }); + it('converts foreign-currency amendments to EUR via the annex fx rate', async () => { const detail = await getContract( fakeDb( diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index 7a211d8d..9786347e 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -431,6 +431,8 @@ interface AmendmentRow { published_at: string | null; document_number: string | null; description: string | null; + value_restated: number | null; + value_suspect: number | null; fx_rate: number | null; } @@ -454,7 +456,8 @@ export const AMENDMENTS_SQL = `SELECT am.value_before, am.value_after, am.value_ WHERE f.base_currency = am.currency AND f.rate_date <= am.published_at AND f.rate_date >= date(am.published_at, '-10 days') - ORDER BY f.rate_date DESC LIMIT 1) AS fx_rate + ORDER BY f.rate_date DESC LIMIT 1) AS fx_rate, + am.value_restated, am.value_suspect FROM amendments am WHERE am.unp = ? AND am.contract_number = ? ORDER BY am.published_at, am.id`; @@ -542,14 +545,20 @@ export async function getContract( const suspect = r.value_flag === 'value_suspect' || r.value_flag === 'annex_suspect' || + r.value_flag === 'annex_total_suspect' || r.value_flag === 'review' || r.value_flag === 'value_low'; const dateSuspect = r.date_flag === 'signed_after_publication'; + // #307 — annex_total_suspect is a KNOWN exact 2× double-count in current_value. Its current_value_eur is + // already NULL (excluded from aggregates), so the native fallback below would resurface the doubled figure + // under an "unverified" label. Blank it instead: a known-wrong number is worse than an honest gap. + const currentValueDoubled = r.value_flag === 'annex_total_suspect'; const signingEur = r.signing_value_eur ?? eurFromNative(r.signing_value, r.contract_currency, r.fx_rate); - const currentRaw = - r.current_value_eur ?? - eurFromNative(r.current_value, r.current_value_currency || r.contract_currency, r.fx_rate); + const currentRaw = currentValueDoubled + ? null + : (r.current_value_eur ?? + eurFromNative(r.current_value, r.current_value_currency || r.contract_currency, r.fx_rate)); const procedureEstimatedEur = eurFromNative( r.estimated_value, r.tender_currency, @@ -606,12 +615,13 @@ export async function getContract( estimatedEur: currentLotEstimatedEur ?? procedureEstimatedEur, procedureEstimatedEur, signingEur, - currentEur: currentRaw ?? signingEur, + currentEur: currentValueDoubled ? null : (currentRaw ?? signingEur), deltaPct: !suspect && currentRaw != null && signingEur != null && signingEur !== 0 ? (currentRaw - signingEur) / signingEur : null, suspect, + currentValueDoubled, }; const authority: ContractParty = { @@ -670,16 +680,28 @@ export async function getContract( const amendments: ContractDetail['amendments'] = amendmentRows.results.map((am) => { const beforeEur = eurFromNative(am.value_before, am.currency, am.fx_rate); const afterEur = eurFromNative(am.value_after, am.currency, am.fx_rate); + // #305 residual: a suspected double-count we could NOT correct from the основание text. Its + // value_after is the untrusted doubled figure, so suppress it (and the derived delta) rather than + // show a number we can't stand behind — the UI marks the row „непотвърден тотал". + const suspect = am.value_suspect === 1; return { date: am.published_at, documentNumber: am.document_number, description: am.description?.trim() || null, - valueAfterEur: afterEur, + valueAfterEur: suspect ? null : afterEur, // Compute delta from the SAME before/after we display, so the row is self-consistent (after − // before == delta) even when the source's recorded value_delta disagrees with them. When only // one of before/after is known, the recorded delta can't be reconciled against valueAfterEur — // show „—" rather than a figure that might not add up. - deltaEur: beforeEur != null && afterEur != null ? afterEur - beforeEur : null, + deltaEur: suspect + ? null + : beforeEur != null && afterEur != null + ? afterEur - beforeEur + : null, + // #305 Tier-2: the served value_after was rewritten from the основание text (a double-count total + // restated to the true value) — let the UI mark the corrected row. + restated: am.value_restated === 1, + suspect, }; }); diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 2ab6831d..3685e1e4 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -13,6 +13,10 @@ const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bid const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // refresh-slice.sql / precompute.sql officials block reads interest_links (0003) — build it in every chain. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (refresh-slice promotes them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +// #305 residual: served amendments gained value_suspect (refresh-slice promotes it). +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const deriveAmendmentsPath = resolve(root, 'scripts/derive-amendments.sql'); @@ -185,6 +189,8 @@ function initWorkDb(dbPath: string): void { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); } @@ -576,6 +582,8 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -658,6 +666,8 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); // An EOP procedure (tender + base contract) with УНП UNP-SLICE / tender.id TENDER-SLICE. An @@ -841,6 +851,8 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -892,6 +904,8 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -943,6 +957,8 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -1080,6 +1096,8 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, diff --git a/packages/db/src/value-flag-annex-step-sql.test.ts b/packages/db/src/value-flag-annex-step-sql.test.ts index ea9f91f5..bfa8af32 100644 --- a/packages/db/src/value-flag-annex-step-sql.test.ts +++ b/packages/db/src/value-flag-annex-step-sql.test.ts @@ -26,6 +26,9 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const promotePath = resolve(root, 'scripts/promote-amendments.sql'); @@ -60,6 +63,8 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/value-flag-stotinki-sql.test.ts b/packages/db/src/value-flag-stotinki-sql.test.ts index 2fc97624..d2fe07e5 100644 --- a/packages/db/src/value-flag-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-stotinki-sql.test.ts @@ -20,6 +20,9 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -49,6 +52,8 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/ingest/src/amendment-total.test.ts b/packages/ingest/src/amendment-total.test.ts new file mode 100644 index 00000000..ff1f9cef --- /dev/null +++ b/packages/ingest/src/amendment-total.test.ts @@ -0,0 +1,348 @@ +// #305 — the value-double-count text heuristic, tested against REAL основание text pulled from the live +// corpus (contracts 145652, 189325, 84818, 108677, 79382, 113291, 103903). The hazard is false positives, +// so the controls (genuine increment, uncorrectable, normal increase) matter as much as the hits. +import { describe, expect, it } from 'vitest'; +import { + classifyAmendmentValue, + restatedValueAfter, + isGenuineIncrement, + type AmendmentValueInput, +} from './amendment-total'; + +const mk = ( + valueBefore: number, + valueAfter: number, + valueDelta: number, + currency: string, + ...texts: string[] +): AmendmentValueInput => ({ valueBefore, valueAfter, valueDelta, currency, texts }); + +describe('#305 amendment value double-count heuristic', () => { + it('restates a total announced as "…на " (145652)', () => { + const t = classifyAmendmentValue( + mk( + 442000, + 981240, + 539240, + 'BGN', + 'относно актуализиране стойността на договора … общата стойност на договора ще възлезе на 539 240.00 лв. без ДДС', + ), + ); + expect(t).toEqual({ kind: 'total_restated', correctedAfter: 539240 }); + }); + + it('restates a total announced as "…от X на " (79382, 113291 family)', () => { + expect( + restatedValueAfter( + mk( + 197720, + 484414, + 286694, + 'BGN', + 'Общата стойност на договор № 447 се променя от 197 720.00 лева без ДДС на 286 694,00 (двеста осемдесет и шест хиляди) лева без ДДС', + ), + ), + ).toBe(286694); + expect( + restatedValueAfter( + mk( + 13662405.12, + 28356190.64, + 14693785.52, + 'BGN', + 'в чл. 7 (1) от Договора общата цена за изпълнение предмета на договора се променя от 13 662 405,12 лв. без ДДС на 14 693 785,52 лв. без ДДС', + ), + ), + ).toBe(14693785.52); + }); + + it('does NOT restate "в размер на / ресурс / increment" phrasings — they name the change, not the total', () => { + // Real corpus 271148→650754: "…максималния ресурс за изменението в размер на 379 606.50" names the + // INCREMENT; restating value_after to it would understate a genuine >100% increase. Must be `none`. + expect( + classifyAmendmentValue( + mk( + 271147.5, + 650754, + 379606.5, + 'BGN', + 'Срокът се удължава до изчерпване на максималния ресурс за изменението в размер на 379 606.50 лв. без ДДС', + ), + ).kind, + ).toBe('none'); + // "…или сума в размер на 86 363.08" — the added-work amount, not the new contract total. + expect( + restatedValueAfter( + mk( + 71969.23, + 151662.82, + 79693.59, + 'BGN', + 'Общата стойност на договорените СМР се променя от 71 969.23 без ДДС или сума в размер на 79 693.59 лв.', + ), + ), + ).toBeNull(); + }); + + it('restates a currency re-denomination that doubled an unchanged total (189325)', () => { + const t = classifyAmendmentValue( + mk( + 77000000, + 154000000, + 77000000, + 'BGN', + 'Считано от 01.10.2025 г., отпечатваната върху акцизните бандероли продажна цена се променя от лева в евро.', + ), + ); + expect(t).toEqual({ kind: 'unchanged_restated', correctedAfter: 77000000 }); + }); + + it('does NOT touch a genuine increment announced as "…с " (108677)', () => { + const input = mk( + 10226.85, + 60226.85, + 50000, + 'EUR', + 'Увеличава се финансовият ресурс на договор № АО – 05 – 168 с 50 000 /петдесет хиляди/ евро без ДДС.', + ); + expect(isGenuineIncrement(input)).toBe(true); + expect(restatedValueAfter(input)).toBeNull(); + }); + + it('does NOT restate an exact-2× when the text announces a DIFFERENT total ("до 18 900") — flag, not rewrite (103903)', () => { + // #307: exact 2× (delta 15 120 = value_before) BUT the text says the value rose "до 18 900" — it is NOT + // unchanged. Neither the doubled 30 240 nor the halved 15 120 is the true total, and 18 900 is not + // recoverable here, so return `none` and let the arithmetic annex_total_suspect flag exclude the row. + const t = classifyAmendmentValue( + mk( + 15120, + 30240, + 15120, + 'BGN', + 'Прогнозната стойност по договора се увеличава от 15 120 лв. без ДДС до 18 900 лв. без ДДС', + ), + ); + expect(t).toEqual({ kind: 'none' }); + }); + + it('does NOT text-freely restate an exact-2× when the основание carries no restatement signal (84818)', () => { + // #307: restructuring note with no value/unchanged signal. A text-free halving could erase a legitimate + // ЗОП чл.116 ал.1 т.1 in-scope +100% (pre-announced option clause), so it must fall to the arithmetic + // annex_total_suspect flag (exclude), not be rewritten to the before-value. + const t = classifyAmendmentValue( + mk( + 76769540.87, + 153539081.74, + 76769540.87, + 'EUR', + 'Следните курсове за 22 пилота се преструктурират и се изпълняват в рамките на гаранционния период', + ), + ); + expect(t).toEqual({ kind: 'none' }); + }); + + it('restates an exact-2× administrative annex (non-value change) to the before-value', () => { + const t = classifyAmendmentValue( + mk( + 2685, + 5370, + 2685, + 'BGN', + 'Променя се упълномощеното лице по договора. Несъществени промени.', + ), + ); + expect(t).toEqual({ kind: 'unchanged_restated', correctedAfter: 2685 }); + }); + + it('does NOT text-freely restate an exact-2× on an outside-ЗОП exception contract', () => { + // ЗОП чл.116's +50% cap does not bind exception contracts, so an exact +100% there can be a genuine + // increase — the text-free rule 3 must stand down and let the arithmetic flag exclude (not rewrite) it. + const base = mk( + 2685, + 5370, + 2685, + 'BGN', + 'Променя се упълномощеното лице по договора. Несъществени промени.', + ); + expect(classifyAmendmentValue({ ...base, outsideZop: true }).kind).toBe('none'); + // …but the same row in-scope of ЗОП is still restated (guard is scoped to rule 3 only). + expect(classifyAmendmentValue({ ...base, outsideZop: false })).toEqual({ + kind: 'unchanged_restated', + correctedAfter: 2685, + }); + }); + + it('still applies the text-confirmed rules on an outside-ЗОП contract', () => { + // The double-count is a feed defect independent of ЗОП scope, so a text-confirmed total is corrected + // even for an exception contract — only the text-free exact-2× fallback is gated by outsideZop. + const total = { + ...mk( + 442000, + 981240, + 539240, + 'BGN', + 'общата стойност на договора ще възлезе на 539 240.00 лв.', + ), + outsideZop: true, + }; + expect(classifyAmendmentValue(total)).toEqual({ + kind: 'total_restated', + correctedAfter: 539240, + }); + const incr = { + ...mk(10226.85, 60226.85, 50000, 'EUR', 'Увеличава се ресурсът с 50 000 евро без ДДС.'), + outsideZop: true, + }; + expect(isGenuineIncrement(incr)).toBe(true); + }); + + it('ignores normal increases (< 2×) and non-self-consistent rows', () => { + expect(classifyAmendmentValue(mk(100, 130, 30, 'BGN', 'обща стойност на 130 лв.')).kind).toBe( + 'none', + ); + // a ≠ b + d ⇒ the double-count model does not apply + expect(classifyAmendmentValue(mk(100, 250, 100, 'BGN', 'обща стойност на 100')).kind).toBe( + 'none', + ); + }); + + it('requires the text figure to actually equal the delta (no coincidental match)', () => { + // delta 500000 appears nowhere as a total; a different figure 12345 does — must not restate. + expect( + restatedValueAfter(mk(400000, 900000, 500000, 'BGN', 'обща стойност на 12 345 лв.')), + ).toBeNull(); + }); + + it('does NOT restate a bare "…на " over a NON-monetary number (#307 HIGH-1 — days / article nos.)', () => { + // "…удължава на 200 дни": 200 is a day count that coincidentally == value_delta. Without a currency + // anchor around the figure it must stay `none`, never overwrite the published 300 with 200. + expect( + classifyAmendmentValue(mk(100, 300, 200, 'BGN', 'Срокът на договора се удължава на 200 дни.')) + .kind, + ).toBe('none'); + // An article number after "на" — non-monetary, must not restate. + expect( + classifyAmendmentValue( + mk(100, 300, 200, 'BGN', 'Договорът се изменя на 200 съгласно чл. 116 на ЗОП.'), + ).kind, + ).toBe('none'); + }); + + it('does NOT anchor a day-count on a currency token elsewhere in the sentence (#307 MONEY_AFTER window)', () => { + // "…удължава на 200 дни, стойността остава 100 лв.": 200 is a DAY count; the "лв." belongs to a + // different figure downstream. A non-monetary unit right after 200 must veto it, not restate 300→200. + expect( + classifyAmendmentValue( + mk(100, 300, 200, 'BGN', 'Срокът се удължава на 200 дни, стойността остава 100 лв.'), + ).kind, + ).toBe('none'); + expect( + classifyAmendmentValue( + mk( + 100, + 300, + 200, + 'BGN', + 'Срокът за изпълнение на договора се удължава на 200 дни, без промяна в договорената сума в лв.', + ), + ).kind, + ).toBe('none'); + }); + + it('vetoes the QUALIFIED day/quantity unit, not just the bare word (#307 review — работни/календарни дни class)', () => { + // The unit almost never comes bare in real annexes ("работни дни", "календарни дни", "200 (двеста) + // дни", "кв.м"). Each of these is a duration/quantity that coincidentally == value_delta; none may + // overwrite the published 300 with 200. Tests the error CLASS, not one literal sentence. + const dayCounts = [ + 'Срокът се удължава на 200 работни дни, стойността остава 100 лв.', + 'Срокът се удължава на 200 календарни дни, стойността остава 100 лв.', + 'Срокът се удължава на 200 работни дни, без промяна в договорената сума в лв.', + 'Срокът се удължава на 200 к.д., стойността остава 100 лв.', + 'Срокът се удължава на 200 (двеста) дни, стойността остава 100 лв.', + 'Срокът се удължава на 200 р.д., стойността остава 100 лв.', + 'Площта се увеличава на 200 кв.м, стойността остава 100 лв.', + 'Обемът се увеличава на 200 куб.м, стойността остава 100 лв.', + ]; + for (const text of dayCounts) { + expect(classifyAmendmentValue(mk(100, 300, 200, 'BGN', text)).kind).toBe('none'); + } + }); + + it('the wider unit veto does NOT swallow a real monetary total (#307 review — reverse direction)', () => { + // A qualified/adjacent-word unit veto must not fire on genuine money phrasings: the figure still + // restates to the announced total. Guards against the veto over-reaching. + const realTotals = [ + 'Общата стойност на договора възлиза на 200 лв. без ДДС.', + 'Общата стойност на договора възлиза на 200 лева.', + 'Новата обща стойност възлиза на 200 лв. за срок от 12 месеца.', + 'Общата стойност се увеличава на 200 лева месечно.', + 'Общата стойност възлиза на 200 лв. и срокът се удължава с 30 работни дни.', + ]; + for (const text of realTotals) { + expect(restatedValueAfter(mk(100, 300, 200, 'BGN', text))).toBe(200); + } + }); + + it('does NOT restate an exact-2× on a bare payment-in-euro clause (#307 в-евро narrowing)', () => { + // "Плащанията…се извършват в евро…" is a payment-currency clause, NOT an unchanged-value signal — it + // must not halve a real +100%. Only "X в евро" re-denomination phrasing may restate (see 189325). + const t = classifyAmendmentValue( + mk( + 250000, + 500000, + 250000, + 'BGN', + 'Плащанията по договора се извършват в евро по сметка на изпълнителя.', + ), + ); + expect(t).toEqual({ kind: 'none' }); + }); + + it('restates a bare "…на " only WHEN a currency unit follows the figure (#307 HIGH-1 anchor)', () => { + // Same "…на " shape as the days case, but a currency unit anchors it as money ⇒ genuine total. + expect( + restatedValueAfter( + mk(100, 300, 200, 'BGN', 'Общата стойност на договора се променя на 200 лв. без ДДС.'), + ), + ).toBe(200); + }); + + it('does NOT rewrite an exact-2× with empty / whitespace-only texts (#307 HIGH-2 repro)', () => { + const t = classifyAmendmentValue({ + valueBefore: 539240, + valueAfter: 1078480, + valueDelta: 539240, + currency: 'BGN', + texts: [null, '', ' '], + outsideZop: null, + }); + expect(t).toEqual({ kind: 'none' }); + }); + + it('does NOT rewrite an exact-2× when the text is unrelated to value (#307 HIGH-2 repro)', () => { + const t = classifyAmendmentValue({ + valueBefore: 250000, + valueAfter: 500000, + valueDelta: 250000, + currency: 'BGN', + outsideZop: false, + texts: ['Смяна на адреса за кореспонденция на изпълнителя.'], + }); + expect(t).toEqual({ kind: 'none' }); + }); + + it('parses a dot-thousands + comma-decimal figure "1.234,56" (#305 number-format recall)', () => { + // Mixed-separator total announced as "…на 1.234,56 лв." — the resolver must read 1234.56, not 1.23. + expect( + restatedValueAfter( + mk(700, 1934.56, 1234.56, 'BGN', 'Общата стойност на договора се променя на 1.234,56 лв.'), + ), + ).toBe(1234.56); + // …and the US ordering "1,234.56" resolves to the same value. + expect( + restatedValueAfter( + mk(700, 1934.56, 1234.56, 'BGN', 'Общата стойност на договора се променя на 1,234.56 лв.'), + ), + ).toBe(1234.56); + }); +}); diff --git a/packages/ingest/src/amendment-total.ts b/packages/ingest/src/amendment-total.ts new file mode 100644 index 00000000..fb5e4495 --- /dev/null +++ b/packages/ingest/src/amendment-total.ts @@ -0,0 +1,227 @@ +// #305 — amendment value double-count. ЦАИС ЕОП sometimes puts the announced NEW TOTAL contract value +// into the "change" field (`contractValueDifference` → `value_delta`), so the feed's +// `currentContractValue` (→ `value_after`) = `lastContractValue` + newTotal — the value is doubled. The +// source is internally consistent (`value_after = value_before + value_delta` at ~100% of rows), so the +// signal is semantic, not arithmetic: is `value_delta` an increment or a total? +// +// This module answers that from the основание free text, which the raw feed carries in three fields +// (changeDescription/changeReason/changeReasonDescription). The discriminator is the Bulgarian preposition +// in front of the figure: "на " (to N) / "възлиза/става/обща стойност" ⇒ N is a TOTAL; "с " (by N) / +// "увеличава се … с" ⇒ N is an INCREMENT. See docs/implementation-plans/305-amendment-value-double-count.md. +// +// Conservative by design: it only classifies when the text unambiguously confirms; otherwise it returns +// `none` and leaves the row to the arithmetic `annex_total_suspect` flag (Tier 1). It NEVER rewrites a +// value it cannot corroborate from text. Note: JS `\b`/`\w` are ASCII-only, so all boundaries/letters use +// Unicode (`\p{L}`, explicit non-letter boundary) with the `u` flag. + +export type AmendmentValueTreatment = + // The delta is an announced total; the true value_after is value_delta (double-count corrected). + | { kind: 'total_restated'; correctedAfter: number } + // An exact 2× (value_delta ≈ value_before): the "difference" field echoed the OLD value, so the value + // is unchanged and value_after was doubled onto itself; the true value_after is value_before. Covers + // currency re-denominations and non-value administrative annexes alike. + | { kind: 'unchanged_restated'; correctedAfter: number } + // The delta is a genuine increment already correctly applied — value_after is right; do NOT flag it. + | { kind: 'genuine_increment' } + // No text signal — leave to the arithmetic flag. + | { kind: 'none' }; + +export interface AmendmentValueInput { + valueBefore: number | null; + valueAfter: number | null; + valueDelta: number | null; + currency: string | null; + texts: Array; + // #305 — the text-free exact-2× rule leans on ЗОП чл.116 (a single amendment caps at +50%, so +100% is a + // defect not a real increase). чл.116 does NOT bind contracts procured outside ЗОП (exception contracts), + // where a genuine +100% is legal — so for those, only the text-confirmed rules may restate. NULL/false = + // in-scope of ЗОП (the safe default: apply the rule). + outsideZop?: boolean | null; +} + +const REL_TOL = 0.005; // 0.5% — the text figure must be the SAME number as value_delta, allowing rounding. +// #305 — capture a full number token that may group thousands with space/nbsp/narrow-nbsp OR with '.'/',' +// (BG "1.234,56", US "1,234.56"); normalizeBgNumber disambiguates the decimal mark below. The token must +// END on a digit so a trailing sentence period ("…100. Нов срок") is not swallowed into the number. +const NUMBER_RE = /\d[\d\u0020\u00a0\u202f.,]*\d|\d/g; +const WS = /[\s  ]/g; + +// A left boundary: start-of-window or a non-letter, non-digit character (Unicode-aware — Cyrillic is a +// letter). Keywords that end the "before the figure" window signal how the figure should be read. +const B = '(?:^|[^\\p{L}\\d])'; +const TOTAL_CTX = new RegExp( + `${B}(?:възлиз\\p{L}*|възлез\\p{L}*|става|обща\\p{L}*\\s+(?:стойност|цена)|крайн\\p{L}*\\s+(?:стойност|цена)|нов\\p{L}*\\s+(?:обща\\s+)?(?:стойност|цена)|на)\\s*$`, + 'iu', +); +// The figure sits right after "от " (the OLD value) or "с/със " (an increment) — not a total. +const NOT_TOTAL_CTX = new RegExp(`${B}(?:от|с|със)\\s*$`, 'iu'); +// "…с " / "…със " — N is an increment already applied. +const INCREMENT_CTX = new RegExp(`${B}(?:с|със)\\s*$`, 'iu'); +// A wider veto on the "…на " total match: Bulgarian "в размер на " ("in the amount of N"), +// "ресурс … в размер на N", "допълнителни … на обща стойност N" name the CHANGE/added-work amount, not +// the new contract total — restating value_after := N there would be wrong (verified on the real corpus). +// Checked over a wider window than NOT_TOTAL_CTX because these markers sit a few words before the figure. +const TOTAL_VETO = /(?:в\s+размер|ресурс\p{L}*|допълнителн\p{L}*|увеличени\p{L}*|намалени\p{L}*)/iu; + +// #307 — a total restatement needs a MONETARY anchor bracketing the figure. Bare "на " is not a money +// signal ("на" also precedes days, article numbers, quantities), so "…удължава на 200 дни" would otherwise +// rewrite the value with a day count. Accept the figure only when a value keyword sits immediately before +// it (MONEY_BEFORE) OR a currency unit follows it (MONEY_AFTER). On the real corpus the value keyword is +// usually far from the figure ("…ще възлезе на 539 240.00 лв."), so the currency unit after the number is +// the load-bearing anchor. No ASCII \b (Cyrillic). +const MONEY_BEFORE = /(?:стойност|цена)\p{L}*\s*$/iu; +const MONEY_AFTER = /(?:^|[^\p{L}])(?:лв\.?|лева|лев|bgn|eur|евро|euro|€|usd|\$)(?![\p{L}])/iu; +// #307 — MONEY_AFTER scans the whole ~60-char window, so a sentence that names both a term and a value +// ("…удължава на 200 дни, стойността остава 100 лв.") lets a downstream currency token anchor a figure +// that is actually a day count. A non-monetary unit sitting IMMEDIATELY after the figure (days, months, +// years, count, percent) overrides any currency further along: the figure is a duration/quantity, never +// the contract value. Anchored at ^ against the post-figure slice so only the immediate suffix counts. +// Real BG annexes almost never write the unit bare — the term is qualified ("работни дни", "календарни +// дни") — so allow one optional adjective word (and an optional spelled-out number in brackets, "200 +// (двеста) дни") between the figure and the unit, and cover area/volume/weight units too. Errs safe: a +// false veto only downgrades a row to `none`, dropping it to the arithmetic annex_total_suspect flag +// rather than publishing a substituted value. +const NON_MONEY_UNIT_AFTER = + /^\s*(?:\([^)]*\)\s*)?(?:\p{L}+\s+)?(?:дни|дн\.|к\.\s?д\.|р\.\s?д\.|месец\p{L}*|години|год\.|броя|бр\.|кв\.?\s?м|куб\.?\s?м|тона|литра|%|процент\p{L}*)/iu; + +// #307 — the exact-2× "unchanged" restatement (rule 3) may only fire WITH a positive textual signal that +// the value did not really change: a currency re-denomination that mechanically doubled the figure, or an +// explicit "unchanged / non-material" phrasing. Absent any signal the row returns `none` and falls to the +// arithmetic annex_total_suspect flag (exclude), rather than silently halving a possibly-legitimate +// ЗОП чл.116 ал.1 т.1 in-scope +100% (a pre-announced option clause `outsideZop` cannot model). +// #307 — the anchor is the "X в евро" re-denomination phrasing (`лев… в евро`), NOT a bare "в евро": a +// payment-currency clause ("Плащанията…се извършват в евро…") says nothing about an unchanged total and +// would silently halve a real doubling. The bare form was also redundant — the 189325 fixture +// ("…се променя от лева в евро") is already caught by the `лев… в евро` alternative. +const RESTATE_UNCHANGED_CTX = + /(?:лев\p{L}*\s+в\s+евро|деноминаци\p{L}*|не\s*се\s+промен\p{L}*|остава\p{L}*\s+непромен\p{L}*|без\s+промяна|несъществен\p{L}*)/iu; + +function normalizeBgNumber(raw: string): number | null { + let t = raw.replace(WS, ''); + // #305 — when BOTH '.' and ',' appear the number uses one as a thousands separator and the other as the + // decimal mark (BG "1.234,56" or US "1,234.56"). The LAST-occurring separator is the decimal; strip the + // other (thousands) and normalise the decimal to '.'. Single-separator numbers keep the existing + // ≤2-fraction-digit convention (the space-thousands corpus: "539 240.00", "286 694,00"). + if (t.includes('.') && t.includes(',')) { + const decimalChar = t.lastIndexOf('.') > t.lastIndexOf(',') ? '.' : ','; + const thousandsChar = decimalChar === '.' ? ',' : '.'; + t = t.split(thousandsChar).join(''); + if (decimalChar === ',') t = t.replace(',', '.'); + } + const m = t.match(/^(\d+)(?:[.,](\d{1,2}))?$/); + if (!m) return null; + const value = Number(m[2] ? `${m[1]}.${m[2]}` : m[1]); + return Number.isFinite(value) && value > 0 ? value : null; +} + +function approxEq(a: number, b: number): boolean { + return Math.abs(a - b) <= REL_TOL * Math.max(Math.abs(a), Math.abs(b)); +} + +// Does a figure ≈ `target` occur in `text` with the ~40 preceding chars matching `contextRe` and (when +// given) NOT matching `excludeRe`? Returns true on the first qualifying occurrence. +function figureInContext( + text: string, + target: number, + contextRe: RegExp, + excludeRe: RegExp | null, + wideVetoRe: RegExp | null = null, + requireMoneyAnchor = false, +): boolean { + NUMBER_RE.lastIndex = 0; + let m: RegExpExecArray | null; + while ((m = NUMBER_RE.exec(text)) !== null) { + const n = normalizeBgNumber(m[0]); + if (n === null || !approxEq(n, target)) continue; + const before = text.slice(Math.max(0, m.index - 40), m.index); + if (excludeRe && excludeRe.test(before)) continue; + // A wider veto looks further back (~55 chars) for "в размер"/"ресурс"/increment markers that make an + // "…на " an amount, not a total. + if (wideVetoRe && wideVetoRe.test(text.slice(Math.max(0, m.index - 55), m.index))) continue; + if (!contextRe.test(before)) continue; + // #307 — a total needs a monetary marker bracketing the figure, else a bare "…на " matches a + // non-monetary number (days, article nos.) that coincidentally ≈ the target. A value keyword right + // before, OR a currency unit within the ~60 chars after, qualifies. + if (requireMoneyAnchor) { + const after = text.slice(m.index + m[0].length, m.index + m[0].length + 60); + // A non-monetary unit immediately after the figure (days/months/years/count/%) vetoes it before + // a downstream currency token can wrongly anchor it as money (#307). + if (NON_MONEY_UNIT_AFTER.test(after)) continue; + if (!MONEY_BEFORE.test(before) && !MONEY_AFTER.test(after)) continue; + } + return true; + } + return false; +} + +export function classifyAmendmentValue(input: AmendmentValueInput): AmendmentValueTreatment { + const b = input.valueBefore; + const a = input.valueAfter; + const d = input.valueDelta; + if (b === null || a === null || d === null || b <= 0 || a <= 0 || d <= 0) return { kind: 'none' }; + // Source self-consistency (a = b + d) is the precondition of the defect model. + if (!approxEq(a, b + d)) return { kind: 'none' }; + // A single annex whose "increment" is at least the whole prior value (2b ≤ a < 10b). Below 2b is a + // normal increase; ≥10b is a mis-key handled by #299's annex_suspect. + if (a < 2 * b || a >= 10 * b) return { kind: 'none' }; + + const text = input.texts.filter((t): t is string => !!t && t.trim() !== '').join(' '); + + // 1) The delta figure appears as an INCREMENT ("с ") — the value is genuinely correct, don't + // touch it. Checked FIRST so an exact 2× that the text calls a real increase is not mis-restated. + if (text && figureInContext(text, d, INCREMENT_CTX, null)) return { kind: 'genuine_increment' }; + + // 2) The delta figure appears as a TOTAL ("на ", "възлиза на …", "обща стойност … "). + // The true value_after is the delta (the announced new total). TOTAL_VETO rejects "в размер на"/ + // "ресурс"/increment phrasings that name the change amount, not the contract total. A monetary anchor + // is REQUIRED (#307) so a bare "…на " over a non-monetary number (days, article nos.) is rejected. + if (text && figureInContext(text, d, TOTAL_CTX, NOT_TOTAL_CTX, TOTAL_VETO, true)) { + return { kind: 'total_restated', correctedAfter: d }; + } + + // 3) Exact 2× (value_delta ≈ value_before): the "difference" field just echoed the OLD value, so + // value_after = before + before double-counts an UNCHANGED value (currency re-denomination or a + // non-value administrative annex). This restatement to value_before is only SAFE with a positive text + // signal (#307): ЗОП чл.116 ал.1 т.1 permits a genuine in-scope +100% via a pre-announced option/review + // clause that `outsideZop` cannot see, so a text-free rewrite could silently HALVE a legitimate value. + // Require either a "value unchanged / re-denomination" phrasing (RESTATE_UNCHANGED_CTX) or the + // before-value itself announced as the new total. Absent any signal, return `none` and let the + // arithmetic annex_total_suspect flag EXCLUDE the row (an honest gap beats a silent corruption). + // Still skipped for outside-ЗОП exception contracts, where a real +100% is legal. + if (approxEq(a, 2 * b) && !input.outsideZop && text) { + if ( + RESTATE_UNCHANGED_CTX.test(text) || + figureInContext(text, b, TOTAL_CTX, NOT_TOTAL_CTX, TOTAL_VETO, true) + ) { + return { kind: 'unchanged_restated', correctedAfter: b }; + } + } + + return { kind: 'none' }; +} + +// The single value the ETL needs: the corrected value_after when the text confirms a double-count, else +// null (leave value_after as the source gave it). +export function restatedValueAfter(input: AmendmentValueInput): number | null { + const t = classifyAmendmentValue(input); + return t.kind === 'total_restated' || t.kind === 'unchanged_restated' ? t.correctedAfter : null; +} + +export function isGenuineIncrement(input: AmendmentValueInput): boolean { + return classifyAmendmentValue(input).kind === 'genuine_increment'; +} + +// Convenience for the ETL staging: the treatment label to store on the raw amendment row (NULL when no +// signal), and the corrected value_after (NULL unless a double-count was confirmed). A non-null treatment +// tells derive/normalize NOT to arithmetic-flag the row (it is either corrected or confirmed-genuine). +export function amendmentValueTreatment(input: AmendmentValueInput): { + treatment: 'total_restated' | 'unchanged_restated' | 'genuine_increment' | null; + restatedAfter: number | null; +} { + const t = classifyAmendmentValue(input); + return { + treatment: t.kind === 'none' ? null : t.kind, + restatedAfter: + t.kind === 'total_restated' || t.kind === 'unchanged_restated' ? t.correctedAfter : null, + }; +} diff --git a/packages/ingest/src/base.test.ts b/packages/ingest/src/base.test.ts index d4d89b65..cb577d20 100644 --- a/packages/ingest/src/base.test.ts +++ b/packages/ingest/src/base.test.ts @@ -137,6 +137,54 @@ describe('base EOP mapper', () => { expect(baseSqlLiteral('annexes', 'value_delta', row?.value_delta)).toBe('-345.16'); }); + // #305 Tier-2: base.ts runs the validated основание-text heuristic (amendment-total.ts) for annexes and + // persists value_treatment + value_after_restated onto the raw row. A doubled value_after whose text + // announces the NEW TOTAL ("…на ") is restated to that true total; an untreated annex stays NULL. + it('populates value_treatment/value_after_restated for an annex whose text announces a new total', () => { + const row = mapBaseRecord( + 'annexes', + { + uniqueProcurementNumber: '00224-2025-0009', + contractNumber: '990001', + publicationDate: '05.03.2026', + lastContractValue: '442000', + currentContractValue: '981240', // doubled: source put the new TOTAL in the change field + contractValueDifference: '539240', + contractCurrency: 'BGN', + changeReason: 'Общата стойност на договора се променя на 539 240 лв.', + }, + { day: '2026-03-05', fetchedAt: '2026-03-05T00:00:00Z' }, + ); + + expect(row?.value_after).toBe(981240); // raw after left as the source gave it + expect(row?.value_treatment).toBe('total_restated'); + expect(row?.value_after_restated).toBe(539240); // the corrected true total + // The restated total must serialise as a bare number for the staging INSERT, not a quoted string. + expect(baseSqlLiteral('annexes', 'value_after_restated', row?.value_after_restated)).toBe( + '539240', + ); + }); + + it('leaves value_treatment/value_after_restated NULL for a >2× annex with no text total (not exact-2×)', () => { + // 2.5× (not exact 2×) and no announced total in the text ⇒ no confident signal ⇒ left to the flag. + const row = mapBaseRecord( + 'annexes', + { + uniqueProcurementNumber: '00224-2025-0010', + contractNumber: '990002', + publicationDate: '05.03.2026', + lastContractValue: '1000000', + currentContractValue: '2500000', + contractValueDifference: '1500000', + contractCurrency: 'BGN', + }, + { day: '2026-03-05', fetchedAt: '2026-03-05T00:00:00Z' }, + ); + + expect(row?.value_treatment).toBeNull(); + expect(row?.value_after_restated).toBeNull(); + }); + it('coerces signed reals without loosening the magnitude-only fields', () => { expect(toSignedReal('-345,16')).toBe(-345.16); expect(toSignedReal('-1 234,56')).toBe(-1234.56); diff --git a/packages/ingest/src/base.ts b/packages/ingest/src/base.ts index fea2b5ca..4020ee3b 100644 --- a/packages/ingest/src/base.ts +++ b/packages/ingest/src/base.ts @@ -1,5 +1,7 @@ // Base EOP plain-JSON adapter helpers. Pure and Worker-safe: no Node APIs. +import { amendmentValueTreatment } from './amendment-total.ts'; + export type BaseCategory = 'contracts' | 'tenders' | 'annexes'; export type BaseCoercionKind = | 'text' @@ -326,6 +328,10 @@ export const BASE_CATEGORIES: Record = { field('description', 'changeDescription', 'text'), field('reason', 'changeReason', 'text'), field('circumstances', 'changeReasonDescription', 'text'), + // #305 Tier-2 — computed from the основание text after the generic mapping (see mapBaseRecord), not + // read from a source key. key=null keeps the generic loop from touching them; mapBaseRecord sets them. + field('value_treatment', null, 'text'), + field('value_after_restated', null, 'real'), field('outside_zop', 'isExceptionContract', 'bool'), field('exemption_legal_basis', 'directAwardJustification', 'text'), field('correction_number', null, 'text'), @@ -379,9 +385,34 @@ export function mapBaseRecord( if (!cfg.keep(record)) return null; const row: BaseStagingRow = fixedValues(cat, meta); for (const f of cfg.fields) row[f.column] = f.key === null ? null : coerce(f.kind, record[f.key]); + // #305 Tier-2 — EOP annexes only: classify value_delta from the основание free text (the validated + // heuristic in amendment-total.ts) and persist the treatment label + corrected total onto the raw row. + // OCDS annexes never reach here (ocds.ts stages them, and they carry value_after = null anyway). + if (cat === 'annexes') { + const treatment = amendmentValueTreatment({ + valueBefore: numOrNull(row.value_before), + valueAfter: numOrNull(row.value_after), + valueDelta: numOrNull(row.value_delta), + currency: strOrNull(row.currency), + texts: [strOrNull(row.description), strOrNull(row.reason), strOrNull(row.circumstances)], + // #305 — outside-ЗОП exception contracts (isExceptionContract) are not bound by чл.116's +50% cap, + // so the text-free exact-2× restatement must not fire on them (see amendment-total.ts rule 3). + outsideZop: numOrNull(row.outside_zop) === 1, + }); + row.value_treatment = treatment.treatment; + row.value_after_restated = treatment.restatedAfter; + } return row; } +function numOrNull(v: BaseStagingValue | undefined): number | null { + return typeof v === 'number' ? v : null; +} + +function strOrNull(v: BaseStagingValue | undefined): string | null { + return typeof v === 'string' ? v : null; +} + // Hard ceiling on a single text literal's character length. EOP/registry text fields // (subjects, descriptions, names) are well under this; anything larger is corrupt or // hostile and is truncated rather than passed to sqlite (avoids SQLITE_TOOBIG / abuse of diff --git a/packages/ingest/tsconfig.json b/packages/ingest/tsconfig.json index b8ac7d61..9d4ac835 100644 --- a/packages/ingest/tsconfig.json +++ b/packages/ingest/tsconfig.json @@ -1,6 +1,8 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { + // base.ts imports './amendment-total.ts' with an explicit .ts extension (the ETL runtime consumes this + // source under plain Node). allowImportingTsExtensions is set in tsconfig.base.json for every consumer. "types": ["@cloudflare/workers-types"] }, "include": ["src"] diff --git a/scripts/derive-amendments.sql b/scripts/derive-amendments.sql index 23aaff87..23dcee7a 100644 --- a/scripts/derive-amendments.sql +++ b/scripts/derive-amendments.sql @@ -144,8 +144,10 @@ SET AND a.contract_number = raw_contracts.contract_number AND a.rn = 1 ), + -- #305 Tier-2: a text-confirmed double-count carries the corrected total in value_after_restated; use + -- it as the effective after so current_value reflects the true total, not the raw doubled value_after. current_value = ( - SELECT a.value_after FROM dedup a + SELECT COALESCE(a.value_after_restated, a.value_after) FROM dedup a WHERE a.unp = raw_contracts.unp AND a.contract_number = raw_contracts.contract_number AND a.value_after IS NOT NULL diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 964132da..c5ba2c72 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -819,6 +819,7 @@ FROM ( CASE y.value_flag WHEN 'value_suspect' THEN y.proc_est_native WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS display_native, -- value_suspect is repaired directly from proc_est_eur in the outer amount_eur CASE; value_low and @@ -826,6 +827,7 @@ FROM ( CASE y.value_flag WHEN 'value_suspect' THEN NULL WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, -- Keep the companion currency paired with the exact native value chosen above. In particular, @@ -837,6 +839,10 @@ FROM ( WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') ELSE COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') END + WHEN 'annex_total_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') + END ELSE CASE WHEN y.current_value IS NOT NULL THEN COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') @@ -943,6 +949,86 @@ FROM ( WHERE am.unp = c.unp AND am.contract_number = c.contract_number AND am.value_before > 0 AND am.value_after >= 10 * am.value_before ))))) THEN 'annex_suspect' + -- #305 value double-count: a driving annex reports a new TOTAL added to the old instead of + -- replacing it, so value_after ≈ 2× the OLD total. ЗОП чл.116 caps a single amendment at +50%, + -- so one step cannot legally more than double a contract — the ≥2× single step IS the defect + -- signal, wherever it sits in the chain. Scope: value_after in [2×,10×) a base that value_before + -- ties to a KNOWN prior total — signing_value OR a preceding annex's value_after (the multi-annex + -- case) — same currency. Slow legitimate climbs never reach ≥2× so stay untouched; the ≥10× + -- mis-key is #299's annex_suspect above; cross-currency doubles are an FX artefact ('review'); + -- and the ABS(... - current_value) tie binds this to the annex that DRIVES current_value, so a + -- doubled annex later superseded by a correct one is NOT flagged. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + -- #305 Tier-2: a text-treated annex (restated total or confirmed-genuine increment) is NOT an + -- arithmetic suspect — value_treatment IS NOT NULL means the основание text already resolved it. + AND am.value_treatment IS NULL + -- #305 multi-annex: the doubled step need not be the FIRST annex. value_before may be a + -- prior cumulative total (a preceding annex's value_after), not signing. Anchor to signing + -- OR a legitimately-grown prior total (a prior annex that was itself not a double), while a + -- single ≥2× step (ЗОП чл.116 caps one amendment at +50%) is the defect wherever it sits. + AND am.value_before > 0 AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× (value_after ≈ 2× value_before) on an ORPHAN + -- base — value_before ties neither signing NOR any prior annex's value_after (e.g. contract + -- 84818, whose annex base 76.77M is unrelated to the contract's values). A legal +100% in one + -- amendment is impossible (ЗОП чл.116), so flag → signing fallback (EXCLUDE); never REWRITES + -- (that stays gated, #307 HIGH-2). The orphan guard leaves a legitimate compounding-doubling + -- chain (each step's base ties the prior step) untouched, exactly as the legit-prior arm does. + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2: the double-count model presupposes a self-consistent row (value_after ≈ + -- value_before + value_delta). If value_delta is present and contradicts that, the model + -- provably does not apply — do NOT flag (mirrors the TS classifier's a≈b+d precondition). + AND (am.value_delta IS NULL + OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' + -- #305 NEW-HIGH-1 (multi-annex chain contamination): the double-count correction is per-row and + -- does NOT propagate down a chain. When a PRIOR annex was double-count corrected (value_after was + -- doubled, restated down), a LATER annex still arrives from the feed computed on the CONTAMINATED + -- (raw, doubled) base. Its own step ratio is legitimate (<2×) so the gate above misses it, and the + -- prior annex is text-treated so it is excluded too — yet current_value inherited the doubled + -- total. Detect the driving annex whose value_before ties to a prior annex's RAW value_after where + -- that prior was restated to a lower total, and flag → signing fallback (an honest exclusion beats + -- a served overstatement) until per-chain value_before propagation (a follow-up) recomputes it. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_treatment IS NULL + AND am.value_before > 0 + AND ABS(am.value_after - c.current_value) < 0.01 + AND EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after_restated IS NOT NULL + AND prev.value_after_restated < prev.value_after + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1283,6 +1369,86 @@ SELECT 1, WHERE am.unp = c.unp AND am.contract_number = c.contract_number AND am.value_before > 0 AND am.value_after >= 10 * am.value_before ))))) THEN 'annex_suspect' + -- #305 value double-count: a driving annex reports a new TOTAL added to the old instead of + -- replacing it, so value_after ≈ 2× the OLD total. ЗОП чл.116 caps a single amendment at +50%, + -- so one step cannot legally more than double a contract — the ≥2× single step IS the defect + -- signal, wherever it sits in the chain. Scope: value_after in [2×,10×) a base that value_before + -- ties to a KNOWN prior total — signing_value OR a preceding annex's value_after (the multi-annex + -- case) — same currency. Slow legitimate climbs never reach ≥2× so stay untouched; the ≥10× + -- mis-key is #299's annex_suspect above; cross-currency doubles are an FX artefact ('review'); + -- and the ABS(... - current_value) tie binds this to the annex that DRIVES current_value, so a + -- doubled annex later superseded by a correct one is NOT flagged. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + -- #305 Tier-2: a text-treated annex (restated total or confirmed-genuine increment) is NOT an + -- arithmetic suspect — value_treatment IS NOT NULL means the основание text already resolved it. + AND am.value_treatment IS NULL + -- #305 multi-annex: the doubled step need not be the FIRST annex. value_before may be a + -- prior cumulative total (a preceding annex's value_after), not signing. Anchor to signing + -- OR a legitimately-grown prior total (a prior annex that was itself not a double), while a + -- single ≥2× step (ЗОП чл.116 caps one amendment at +50%) is the defect wherever it sits. + AND am.value_before > 0 AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× (value_after ≈ 2× value_before) on an ORPHAN + -- base — value_before ties neither signing NOR any prior annex's value_after (e.g. contract + -- 84818, whose annex base 76.77M is unrelated to the contract's values). A legal +100% in one + -- amendment is impossible (ЗОП чл.116), so flag → signing fallback (EXCLUDE); never REWRITES + -- (that stays gated, #307 HIGH-2). The orphan guard leaves a legitimate compounding-doubling + -- chain (each step's base ties the prior step) untouched, exactly as the legit-prior arm does. + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2: the double-count model presupposes a self-consistent row (value_after ≈ + -- value_before + value_delta). If value_delta is present and contradicts that, the model + -- provably does not apply — do NOT flag (mirrors the TS classifier's a≈b+d precondition). + AND (am.value_delta IS NULL + OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' + -- #305 NEW-HIGH-1 (multi-annex chain contamination): the double-count correction is per-row and + -- does NOT propagate down a chain. When a PRIOR annex was double-count corrected (value_after was + -- doubled, restated down), a LATER annex still arrives from the feed computed on the CONTAMINATED + -- (raw, doubled) base. Its own step ratio is legitimate (<2×) so the gate above misses it, and the + -- prior annex is text-treated so it is excluded too — yet current_value inherited the doubled + -- total. Detect the driving annex whose value_before ties to a prior annex's RAW value_after where + -- that prior was restated to a lower total, and flag → signing fallback (an honest exclusion beats + -- a served overstatement) until per-chain value_before propagation (a follow-up) recomputes it. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_treatment IS NULL + AND am.value_before > 0 + AND ABS(am.value_after - c.current_value) < 0.01 + AND EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after_restated IS NOT NULL + AND prev.value_after_restated < prev.value_after + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1349,6 +1515,7 @@ SELECT 1, ) c WHERE CASE c.value_flag WHEN 'annex_suspect' THEN COALESCE(c.signing_value, c.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(c.signing_value, c.current_value) ELSE COALESCE(c.current_value, c.signing_value) END IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || c.unp) @@ -1376,6 +1543,7 @@ SELECT (SELECT contract_candidates FROM pipeline_stats) AS contract_candidates, (SELECT COUNT(*) FROM contracts WHERE value_flag = 'value_suspect') AS value_suspect, (SELECT COUNT(*) FROM contracts WHERE value_flag = 'annex_suspect') AS annex_suspect, + (SELECT COUNT(*) FROM contracts WHERE value_flag = 'annex_total_suspect') AS annex_total_suspect, (SELECT COUNT(*) FROM contracts WHERE value_flag = 'review') AS review, (SELECT COUNT(*) FROM contracts WHERE fx_converted = 1) AS fx_converted, (SELECT ROUND(SUM(amount_eur) / 1e9, 2) FROM contracts) AS clean_total_eur_bn, diff --git a/scripts/precompute.sql b/scripts/precompute.sql index c79a26ef..f57cbedd 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -22,8 +22,9 @@ -- signing/current in EUR for the contract page's estimated→signing→current strip. -- BGN at the fixed peg (÷1.95583), EUR as-is, foreign at the row's stored fx_rate (eur_per_unit). -- Display rule: NULL where the figure is suspect, so the caller renders „данните се преглеждат", --- never a fabricated number. signing suppressed for value_suspect; current suppressed for value_ or --- annex_suspect (the suspect annex is the bad part). estimated_value_eur is derived per-request on +-- never a fabricated number. signing suppressed for value_suspect; current suppressed for value_, +-- annex_suspect or annex_total_suspect (#305; the suspect annex is the bad part). estimated_value_eur +-- is derived per-request on -- the contract detail loader from the tender (procurement-level, shared across a multi-lot prepiska). UPDATE contracts SET signing_value_eur = CASE @@ -33,7 +34,7 @@ UPDATE contracts SET WHEN fx_rate IS NOT NULL THEN signing_value * fx_rate ELSE NULL END, current_value_eur = CASE - WHEN value_flag IN ('value_suspect','annex_suspect') OR current_value IS NULL THEN NULL + WHEN value_flag IN ('value_suspect','annex_suspect','annex_total_suspect') OR current_value IS NULL THEN NULL WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'EUR' THEN current_value WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'BGN' THEN current_value / 1.95583 WHEN fx_rate IS NOT NULL THEN current_value * fx_rate diff --git a/scripts/promote-amendments.sql b/scripts/promote-amendments.sql index 5d72e367..b7822315 100644 --- a/scripts/promote-amendments.sql +++ b/scripts/promote-amendments.sql @@ -8,7 +8,7 @@ DELETE FROM amendments; INSERT OR REPLACE INTO amendments ( id, natural_key, contract_number, unp, value_before, value_after, value_delta, currency, - published_at, document_number, description, source + published_at, document_number, description, source, value_restated, value_treatment, value_suspect ) WITH keyed AS ( SELECT @@ -40,13 +40,62 @@ SELECT contract_number, unp, value_before, - value_after, - value_delta, + -- #305 Tier-2: serve the effective (text-corrected) after and a self-consistent delta; a restated + -- annex carries the true total, an untreated one is unchanged. + COALESCE(value_after_restated, value_after), + COALESCE(value_after_restated, value_after) - value_before, currency, published_at, document_number, description, - source + source, + CASE WHEN value_after_restated IS NOT NULL THEN 1 ELSE 0 END, + value_treatment, + -- #305 residual: mark a suspected double-count that is NOT already text-treated so the UI suppresses + -- the untrusted value_after. Mirrors normalize-raw.sql's annex_total_suspect arithmetic gate, but + -- joined to raw_contracts for the contract's signing_value/currency (this served INSERT has no + -- contract row to read). value_treatment IS NULL keeps a restated/genuine row out (value_restated + -- already owns those). No current_value tie here: the tie in normalize-raw only decides whether the + -- CONTRACT is flagged; the per-row marker suppresses any row whose after is an unbridgeable double. + CASE WHEN value_treatment IS NULL + AND value_before > 0 + AND value_after >= 2 * value_before AND value_after < 10 * value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (value_delta IS NULL OR ABS(value_after - (value_before + value_delta)) < 0.01 * value_after) + AND EXISTS ( + SELECT 1 FROM raw_contracts rc + WHERE rc.unp = dedup.unp AND rc.contract_number = dedup.contract_number + AND rc.signing_value > 0 + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing. Anchor to signing OR a legitimately-grown prior total (prev not + -- itself a double); a single ≥2× step violates ЗОП чл.116 wherever it sits (see normalize-raw.sql). + AND ( + ABS(dedup.value_before - rc.signing_value) < 0.01 * rc.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = dedup.unp AND prev.contract_number = dedup.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - dedup.value_before) < 0.01 * dedup.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double). + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: EXACT single-step 2× on an ORPHAN base (value_before ties neither signing + -- nor any prior annex) — mark the row suspect; never rewrites (see normalize-raw.sql). The + -- orphan guard leaves compounding chains untouched. + OR ( + ABS(dedup.value_after - 2 * dedup.value_before) < 0.005 * dedup.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = dedup.unp AND prev.contract_number = dedup.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - dedup.value_before) < 0.01 * dedup.value_before + ) + ) + ) + AND COALESCE(NULLIF(dedup.currency, ''), COALESCE(NULLIF(rc.currency, ''), 'BGN')) + = COALESCE(NULLIF(rc.currency, ''), 'BGN') + ) + THEN 1 ELSE 0 END FROM dedup WHERE rn = 1; diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index d74ab266..cdaf8527 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -1063,7 +1063,7 @@ FROM ( ELSE q.signing_value * q.fx_rate END AS signing_value_eur, CASE - WHEN q.value_flag IN ('value_suspect', 'annex_suspect') OR q.current_value IS NULL THEN NULL + WHEN q.value_flag IN ('value_suspect', 'annex_suspect', 'annex_total_suspect') OR q.current_value IS NULL THEN NULL WHEN q.current_value_currency = 'EUR' THEN q.current_value WHEN q.current_value_currency = 'BGN' THEN q.current_value / 1.95583 ELSE q.current_value * q.fx_rate @@ -1073,11 +1073,13 @@ FROM ( CASE y.value_flag WHEN 'value_suspect' THEN y.proc_est_native WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS display_native, CASE y.value_flag WHEN 'value_suspect' THEN NULL WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, CASE y.value_flag @@ -1087,6 +1089,11 @@ FROM ( ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') END + WHEN 'annex_total_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + END ELSE CASE WHEN y.current_value IS NOT NULL THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') @@ -1182,6 +1189,55 @@ FROM ( WHERE am.unp = c.unp AND am.contract_number = c.contract_number AND am.value_before > 0 AND am.value_after >= 10 * am.value_before ))))) THEN 'annex_suspect' + -- #305 value double-count: a driving annex reports a new TOTAL added to the old instead of + -- replacing it, so value_after ≈ 2× the OLD total. ЗОП чл.116 caps a single amendment at +50%, + -- so one step cannot legally more than double a contract — the ≥2× single step IS the defect + -- signal, wherever it sits in the chain. Scope: value_after in [2×,10×) a base that value_before + -- ties to a KNOWN prior total — signing_value OR a preceding annex's value_after (the multi-annex + -- case) — same currency. Slow legitimate climbs never reach ≥2× so stay untouched; the ≥10× + -- mis-key is #299's annex_suspect above; cross-currency doubles are an FX artefact ('review'); + -- and the ABS(... - current_value) tie binds this to the annex that DRIVES current_value, so a + -- doubled annex later superseded by a correct one is NOT flagged. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + -- #305 Tier-2: skip text-treated annexes (restated total or confirmed-genuine increment). + AND am.value_treatment IS NULL + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing. Anchor to signing OR a legitimately-grown prior total (prev + -- not itself a double); a single ≥2× step violates ЗОП чл.116 wherever it sits (see normalize-raw.sql). + AND am.value_before > 0 AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× on an ORPHAN base (value_before ties neither + -- signing nor any prior annex) is the ЗОП чл.116 defect signature — flag (→ signing + -- fallback, EXCLUDE); never rewrites. The orphan guard leaves compounding chains untouched + -- (see normalize-raw.sql). + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (am.value_delta IS NULL OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1351,7 +1407,7 @@ FROM ( ELSE q.signing_value * q.fx_rate END AS signing_value_eur, CASE - WHEN q.value_flag IN ('value_suspect', 'annex_suspect') OR q.current_value IS NULL THEN NULL + WHEN q.value_flag IN ('value_suspect', 'annex_suspect', 'annex_total_suspect') OR q.current_value IS NULL THEN NULL WHEN q.current_value_currency = 'EUR' THEN q.current_value WHEN q.current_value_currency = 'BGN' THEN q.current_value / 1.95583 ELSE q.current_value * q.fx_rate @@ -1361,11 +1417,13 @@ FROM ( CASE y.value_flag WHEN 'value_suspect' THEN y.proc_est_native WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS display_native, CASE y.value_flag WHEN 'value_suspect' THEN NULL WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, CASE y.value_flag @@ -1375,6 +1433,11 @@ FROM ( ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') END + WHEN 'annex_total_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + END ELSE CASE WHEN y.current_value IS NOT NULL THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') @@ -1474,6 +1537,55 @@ FROM ( WHERE am.unp = c.unp AND am.contract_number = c.contract_number AND am.value_before > 0 AND am.value_after >= 10 * am.value_before ))))) THEN 'annex_suspect' + -- #305 value double-count: a driving annex reports a new TOTAL added to the old instead of + -- replacing it, so value_after ≈ 2× the OLD total. ЗОП чл.116 caps a single amendment at +50%, + -- so one step cannot legally more than double a contract — the ≥2× single step IS the defect + -- signal, wherever it sits in the chain. Scope: value_after in [2×,10×) a base that value_before + -- ties to a KNOWN prior total — signing_value OR a preceding annex's value_after (the multi-annex + -- case) — same currency. Slow legitimate climbs never reach ≥2× so stay untouched; the ≥10× + -- mis-key is #299's annex_suspect above; cross-currency doubles are an FX artefact ('review'); + -- and the ABS(... - current_value) tie binds this to the annex that DRIVES current_value, so a + -- doubled annex later superseded by a correct one is NOT flagged. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + -- #305 Tier-2: skip text-treated annexes (restated total or confirmed-genuine increment). + AND am.value_treatment IS NULL + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing. Anchor to signing OR a legitimately-grown prior total (prev + -- not itself a double); a single ≥2× step violates ЗОП чл.116 wherever it sits (see normalize-raw.sql). + AND am.value_before > 0 AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× on an ORPHAN base (value_before ties neither + -- signing nor any prior annex) is the ЗОП чл.116 defect signature — flag (→ signing + -- fallback, EXCLUDE); never rewrites. The orphan guard leaves compounding chains untouched + -- (see normalize-raw.sql). + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (am.value_delta IS NULL OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1605,7 +1717,7 @@ WHERE source LIKE 'ocds:%' INSERT OR REPLACE INTO amendments ( id, natural_key, contract_number, unp, value_before, value_after, value_delta, currency, - published_at, document_number, description, source + published_at, document_number, description, source, value_restated, value_treatment, value_suspect ) WITH keyed AS ( SELECT @@ -1637,13 +1749,62 @@ SELECT contract_number, unp, value_before, - value_after, - value_delta, + -- #305 Tier-2: serve the effective (text-corrected) after and a self-consistent delta; the current_value + -- rollup below reads this served value_after, so a restated annex drives current_value with the true total. + COALESCE(value_after_restated, value_after), + COALESCE(value_after_restated, value_after) - value_before, currency, published_at, document_number, description, - source + source, + CASE WHEN value_after_restated IS NOT NULL THEN 1 ELSE 0 END, + value_treatment, + -- #305 residual: mark a suspected double-count that is NOT already text-treated so the UI suppresses + -- the untrusted value_after. Mirrors normalize-raw.sql's annex_total_suspect arithmetic gate, but + -- joined to raw_contracts for the contract's signing_value/currency (this served INSERT has no + -- contract row to read). value_treatment IS NULL keeps a restated/genuine row out (value_restated + -- already owns those). No current_value tie here: the tie in normalize-raw only decides whether the + -- CONTRACT is flagged; the per-row marker suppresses any row whose after is an unbridgeable double. + CASE WHEN value_treatment IS NULL + AND value_before > 0 + AND value_after >= 2 * value_before AND value_after < 10 * value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (value_delta IS NULL OR ABS(value_after - (value_before + value_delta)) < 0.01 * value_after) + AND EXISTS ( + SELECT 1 FROM raw_contracts rc + WHERE rc.unp = dedup.unp AND rc.contract_number = dedup.contract_number + AND rc.signing_value > 0 + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing. Anchor to signing OR a legitimately-grown prior total (prev not + -- itself a double); a single ≥2× step violates ЗОП чл.116 wherever it sits (see normalize-raw.sql). + AND ( + ABS(dedup.value_before - rc.signing_value) < 0.01 * rc.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = dedup.unp AND prev.contract_number = dedup.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - dedup.value_before) < 0.01 * dedup.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double). + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: EXACT single-step 2× on an ORPHAN base (value_before ties neither signing + -- nor any prior annex) — mark the row suspect; never rewrites (see normalize-raw.sql). The + -- orphan guard leaves compounding chains untouched. + OR ( + ABS(dedup.value_after - 2 * dedup.value_before) < 0.005 * dedup.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = dedup.unp AND prev.contract_number = dedup.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - dedup.value_before) < 0.01 * dedup.value_before + ) + ) + ) + AND COALESCE(NULLIF(dedup.currency, ''), COALESCE(NULLIF(rc.currency, ''), 'BGN')) + = COALESCE(NULLIF(rc.currency, ''), 'BGN') + ) + THEN 1 ELSE 0 END FROM dedup WHERE rn = 1; @@ -1659,7 +1820,10 @@ SET WHERE a.unp = substr(contracts.tender_id, 3) AND a.contract_number = contracts.contract_number AND a.value_after IS NOT NULL - ORDER BY a.published_at DESC, a.id DESC + -- #305: tie-break on natural_key to match the full-rebuild path (derive-amendments.sql), so the + -- driving amendment picked for current_value is identical across full and slice when two annexes + -- share published_at. `id` (row insertion order) diverged from natural_key and could pick a different row. + ORDER BY a.published_at DESC, a.natural_key DESC LIMIT 1 ), current_value_currency = ( @@ -1667,7 +1831,9 @@ SET WHERE a.unp = substr(contracts.tender_id, 3) AND a.contract_number = contracts.contract_number AND a.value_after IS NOT NULL - ORDER BY a.published_at DESC, a.id DESC + -- #305: same natural_key tie-break as current_value above, so the currency comes from the same + -- driving amendment the value does. + ORDER BY a.published_at DESC, a.natural_key DESC LIMIT 1 ) WHERE (id GLOB 'c:[eo]:*' AND EXISTS ( @@ -1734,7 +1900,90 @@ WITH contract_base AS ( WHERE am.unp = substr(c.tender_id, 3) AND am.contract_number = c.contract_number AND am.value_before > 0 AND am.value_after >= 10 * am.value_before - ) AS has_step10 + ) AS has_step10, + -- #305 value double-count: a driving annex whose value_before ≈ a KNOWN prior total — the contract's + -- signing_value OR a preceding annex's value_after (the multi-annex case) — with value_after in + -- [2×,10×) that base, same currency, and matching current_value. Checked against the CUMULATIVE domain + -- amendments, matching where this pass re-rolls current_value from. ЗОП чл.116 caps a single amendment + -- at +50%, so the ≥2× step is the defect signal wherever it sits; slow climbs never reach ≥2×, and the + -- ≥10× mis-key and cross-currency cases are handled elsewhere. + EXISTS ( + SELECT 1 FROM amendments am + WHERE am.unp = substr(c.tender_id, 3) + AND am.contract_number = c.contract_number + -- #305 Tier-2: skip text-treated annexes. Restated totals already stop matching (served value_after + -- is the corrected total, no longer ≈2× before), but the explicit guard also covers confirmed-genuine + -- increments, whose value_after is legitimately ≥2× and must NOT be arithmetic-flagged. + AND am.value_treatment IS NULL + AND am.value_before > 0 AND c.signing_value > 0 + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing — anchor to signing OR a legitimately-grown prior total, prev not + -- itself a double (see normalize-raw.sql). + AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + -- #305 NEW-HIGH-2: this reconciliation reads the CUMULATIVE served `amendments` (a prior-window + -- annex is not in this window's raw_amendments), but the full path anchors on RAW values. For a + -- #305-restated prev the served value_after is the CORRECTED (lower) total, not the raw one, so + -- `value_after < 2*value_before` flips true and the gate would disagree with the full rebuild + -- (flag flips between the daily slice and the next full derive). Restrict the anchor to + -- non-restated prevs, whose served value_after == raw value_after — reproducing the full-path + -- (raw) decision without losing cross-window history. + AND prev.value_restated = 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× on an ORPHAN base (value_before ties neither signing + -- nor any prior served annex) is the ЗОП чл.116 defect signature — flag (→ signing fallback, + -- EXCLUDE); never rewrites. The orphan guard leaves compounding chains untouched (see + -- normalize-raw.sql). + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (am.value_delta IS NULL OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) AS has_double, + -- #305 NEW-HIGH-1 (multi-annex chain contamination), slice mirror. The full path (normalize-raw.sql) + -- detects this on RAW values (prev.value_after_restated < prev.value_after AND am.value_before ≈ raw + -- prev.value_after). The slice reads the CUMULATIVE served `amendments`, which does NOT retain the raw + -- value_after of a restated prev — so this is a CONSERVATIVE approximation: a driving annex whose + -- value_before sits ABOVE a restated prior annex's CORRECTED total (it rode the raw, doubled base) but + -- within a contamination band (< 2× the corrected total, i.e. not a fresh legitimate double). Flags → + -- signing fallback, matching the full path's honest exclusion. Exactness is restored on the next full + -- rebuild; a follow-up value_before-propagation PR removes the approximation entirely. + EXISTS ( + SELECT 1 FROM amendments am + WHERE am.unp = substr(c.tender_id, 3) + AND am.contract_number = c.contract_number + AND am.value_treatment IS NULL + AND am.value_before > 0 + AND ABS(am.value_after - c.current_value) < 0.01 + AND EXISTS ( + SELECT 1 FROM amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_restated = 1 + AND am.value_before > prev.value_after + AND am.value_before < 2 * prev.value_after + ) + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) AS has_contaminated_base FROM contracts c JOIN tenders te ON te.id = c.tender_id WHERE ( @@ -1757,9 +2006,10 @@ WITH contract_base AS ( ), base AS ( SELECT id, currency, signing_value, current_value, current_value_currency, fx_rate, proc_est_eur, proc_est_native, CASE - WHEN c.value_flag <> 'annex_suspect' + WHEN c.value_flag NOT IN ('annex_suspect', 'annex_total_suspect') AND NOT (c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 OR (c.current_value / c.signing_value >= 5 AND c.has_step10))))) + AND NOT (c.current_value IS NOT NULL AND c.signing_value > 0 AND (c.has_double OR c.has_contaminated_base)) THEN c.value_flag WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND (c.eff_eur > 200 * c.proc_est_eur -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly @@ -1771,6 +2021,11 @@ WITH contract_base AS ( -- The step alone is NOT enough — some chains have a huge step that a later annex pulls -- back below signing, and flagging those would RAISE the shown value, not repair it. OR (c.current_value / c.signing_value >= 5 AND c.has_step10)))) THEN 'annex_suspect' + -- #305 single-annex value double-count: the driving annex more than doubled the contract in one + -- step (ЗОП чл.116 caps a single amendment at +50%). has_double already ties to current_value. + -- has_contaminated_base additionally catches a legitimate-looking later annex riding a doubled base + -- (#305 NEW-HIGH-1) — both fall back to signing. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND (c.has_double OR c.has_contaminated_base) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS new_value_flag @@ -1780,11 +2035,13 @@ WITH contract_base AS ( CASE new_value_flag WHEN 'value_suspect' THEN proc_est_native WHEN 'annex_suspect' THEN COALESCE(signing_value, current_value) + WHEN 'annex_total_suspect' THEN COALESCE(signing_value, current_value) ELSE COALESCE(current_value, signing_value) END AS display_native, CASE new_value_flag WHEN 'value_suspect' THEN NULL WHEN 'annex_suspect' THEN COALESCE(signing_value, current_value) + WHEN 'annex_total_suspect' THEN COALESCE(signing_value, current_value) ELSE COALESCE(current_value, signing_value) END AS trusted_native, CASE new_value_flag @@ -1793,13 +2050,17 @@ WITH contract_base AS ( WHEN signing_value IS NOT NULL THEN COALESCE(NULLIF(currency, ''), 'BGN') ELSE COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') END + WHEN 'annex_total_suspect' THEN CASE + WHEN signing_value IS NOT NULL THEN COALESCE(NULLIF(currency, ''), 'BGN') + ELSE COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') + END ELSE CASE WHEN current_value IS NOT NULL THEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') ELSE COALESCE(NULLIF(currency, ''), 'BGN') END END AS trusted_currency, CASE - WHEN new_value_flag IN ('value_suspect', 'annex_suspect') OR current_value IS NULL THEN NULL + WHEN new_value_flag IN ('value_suspect', 'annex_suspect', 'annex_total_suspect') OR current_value IS NULL THEN NULL WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'EUR' THEN current_value WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'BGN' THEN current_value / 1.95583 WHEN fx_rate IS NOT NULL THEN current_value * fx_rate diff --git a/scripts/work-staging-schema.sql b/scripts/work-staging-schema.sql index 9e85b00d..6c403e9c 100644 --- a/scripts/work-staging-schema.sql +++ b/scripts/work-staging-schema.sql @@ -174,6 +174,13 @@ CREATE TABLE raw_amendments ( value_before REAL, -- Стойност преди изменението value_after REAL, -- Стойност след изменението → current_value value_delta REAL, -- Изменение на стойността + -- #305 Tier-2 text-based value correction (computed in TS ingest, packages/ingest/src/amendment-total.ts): + -- value_treatment labels how the основание text reads value_delta ('total_restated' / 'unchanged_restated' + -- / 'genuine_increment', NULL when no signal); value_after_restated is the corrected (true) total when a + -- double-count was confirmed, else NULL. Derive/normalize use COALESCE(value_after_restated, value_after) + -- as the effective after, and skip the arithmetic annex_total_suspect flag when value_treatment IS NOT NULL. + value_treatment TEXT, + value_after_restated REAL, currency TEXT, description TEXT, -- Описание на измененията reason TEXT, -- Причини за изменение (ЗОП основание) diff --git a/tsconfig.base.json b/tsconfig.base.json index 939300bc..bcaee8ac 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -17,6 +17,7 @@ "forceConsistentCasingInFileNames": true, "declaration": false, "sourceMap": true, - "noEmit": true + "noEmit": true, + "allowImportingTsExtensions": true } } From f8973a277f5a333cbc7f84d09b1514f135ab971e Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 14 Aug 2026 11:38:59 +0000 Subject: [PATCH 64/69] =?UTF-8?q?fix(ci):=20=D1=81=D0=B2=D0=B5=D1=80=D1=8F?= =?UTF-8?q?=D0=B2=D0=B0=D0=B9=20=D0=BF=D1=81=D0=B5=D0=B2=D0=B4=D0=BE=D0=BD?= =?UTF-8?q?=D0=B8=D0=BC=D0=B8=D1=82=D0=B5=20=D0=B2=20=D1=81=D0=BE=D0=BD?= =?UTF-8?q?=D0=B4=D0=B0=D1=82=D0=B0=20=D0=B7=D0=B0=20=D0=BA=D0=BE=D0=BB?= =?UTF-8?q?=D0=BE=D0=BD=D0=B8=D1=82=D0=B5=20=D0=BD=D0=B0=20=D0=B0=D0=BD?= =?UTF-8?q?=D0=B5=D0=BA=D1=81=D0=B8=D1=82=D0=B5=20(#310)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Стъпката „Apply amendment restated/suspect columns" от #307 падна на първото си истинско пускане и нямаше как да мине - нито на тази база, нито на никоя. Заявката именуваше колоните has_restated / has_treatment / has_suspect, а read_flag търси реда по низа, с който е извикана ensure_column, тоест value_restated / value_treatment / value_suspect. Object.hasOwn връща false за всяка от трите, сондата чете това като нечетим отговор и излиза с 2, което case-ът третира като фатално. Понеже стъпката стои преди деплоя на Worker-а, деплоят до stage спираше на нея. Сайтът не беше засегнат - нищо не е било разгърнато (стъпки 12-14 прескочени), stage връщаше 200 с изданието отпреди #307. Псевдонимите стават самите имена на колоните, плюс коментар защо не могат да бъдат описателни. Проверено срещу истинската база: заявката връща {value_restated: 0, value_treatment: 0, value_suspect: 0}, а сондата дава статус 1 (добави) при липсващи и 0 (има ги) при налични - никога 2. Пропускът е показателен: deploy.yml не се изпълнява преди мърдж (пуска се само при push към main), нито един тест не го чете и в CI няма линтер за workflow файлове, тъй че първото изпълнение на всяка промяна в деплой стъпка е винаги СЛЕД мърджа. Изнасянето на сондата в проверим скрипт е отделна следваща стъпка. --- .github/workflows/deploy.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 835e208d..2bdfdf2e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -226,12 +226,16 @@ jobs: if: steps.guard.outputs.ok == 'true' run: | node scripts/wrangler-render.mjs apps/web/wrangler.jsonc + # Each alias below MUST be the column name itself: read_flag looks the row up by the very string + # ensure_column is called with. An alias that merely describes the column (has_restated) makes + # `Object.hasOwn(row, key)` false for every column, which the probe treats as an unreadable answer + # and turns into a hard failure — so the step could never succeed, on any database. schema_json="$(pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ --config wrangler.deploy.jsonc --remote --yes --json \ --command "SELECT - (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_restated') AS has_restated, - (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_treatment') AS has_treatment, - (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_suspect') AS has_suspect")" + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_restated') AS value_restated, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_treatment') AS value_treatment, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_suspect') AS value_suspect")" read_flag() { printf '%s' "$schema_json" | node -e ' From b17e70cf7acaf41dbaca2c5d57baf1503bba3d01 Mon Sep 17 00:00:00 2001 From: Stefan Angelov Date: Fri, 14 Aug 2026 15:29:34 +0300 Subject: [PATCH 65/69] =?UTF-8?q?fix(etl):=20=D1=81=D0=B2=D1=8A=D1=80?= =?UTF-8?q?=D0=B6=D0=B8=20=D1=80=D0=B0=D0=B7=D0=BC=D0=B8=D0=BD=D0=B0=D1=82?= =?UTF-8?q?=D0=B8=D1=82=D0=B5=20=D0=BF=D0=BE=20=D0=BD=D0=BE=D0=BC=D0=B5?= =?UTF-8?q?=D1=80=20=D0=B0=D0=BD=D0=B5=D0=BA=D1=81=D0=B8=20=D0=BF=D1=80?= =?UTF-8?q?=D0=B5=D0=B7=20=D1=81=D1=82=D0=BE=D0=B9=D0=BD=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=BA=D0=BE=D1=82=D0=B2=D0=B0=20(#308)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Анексът от ЦАИС носи вътрешен номер (148846), а договорът на същата процедура носи деловодния номер на възложителя (Д-226). Двата са в различни пространства от имена, тъй че съединяването по (УНП, номер на договор) изпускаше около 7% от EOP анексите от всяко навиване анекс → договор → фирма/възложител. Низовата нормализация е пробвана и премерена: възстановява почти нищо, защото номерата са несвързани идентификатори. Връзването е по СТОЙНОСТ: value_before на анекса е стойността на договора към момента на изменението, тоест равна на signing_value на целевия договор. Свързва се само при точно съвпадение (под 0,5 стотинка), при съвпадаща валута и ЕИК на изпълнителя, и само когато съвпада с ЕДИН договор на процедурата. Редовете, които съвпадат с два и повече или с нито един, ОСТАВАТ несвързани - честна дупка бие грешен договор на сайт за прозрачност. Премерено на живия корпус: 1937 разминати EOP анекса, 1563 се свързват (1377 пряко, 186 наследени по веригата), а точността на вече свързаните е 9348/9349, тоест 99,99%. Работи и по двата пътя. Пълният върви от scripts/resolve-amendment-contracts.sql преди derive-amendments.sql; дневният носи корпусно безопасен двойник вътре в refresh-slice.sql, чиито кандидати идват от СЕРВИРАНАТА contracts (целия корпус), обединена с прозореца - тъй че „уникално на процедурата" се пита корпусно, а не в прозореца. Без това дневният цикъл щеше да остави блока бездеен, защото кронът пуска само slice. Подредбата е част от коректността: резолверът тече ПРЕДИ prefer-EOP триенето. Пренаписването на EOP анекс върху договор, който вече е запазил OCDS близнак, би възкресило близнака и би скъсало гейта amendment-twin-dedup от #303. Провенансът се пази: contract_number_raw държи анексния номер отпреди пренаписването, а link_method='value_anchor' прави свързаните по стойност изброими на сервираната страна. Той ключи и natural_key, тъй че вече сервиран несвързан ред се пренаписва на място, вместо да се удвои. Два кръга преглед от наша страна, всяка находка проверена срещу истинските скриптове, не приета по описание: подредбата спрямо близнаците, двусмислените във верига, дневният път, и накрая разминаването между двата пътя при договор с нулева стойност (slice пътят го местеше при съсед, пълният не) - последното днес не хапеше на корпуса, но беше заредено. Затваря #306. --- .github/workflows/deploy.yml | 13 +- docs/README.md | 4 + .../306-amendment-contract-namespace-link.md | 94 +++ .../migrations/0008_amendment_provenance.sql | 9 + .../src/amendments-contract-resolve.test.ts | 634 ++++++++++++++++++ packages/db/src/amendments-ocds-link.test.ts | 3 + .../db/src/amendments-slice-resolve.test.ts | 303 +++++++++ packages/db/src/amendments-sql.test.ts | 3 + .../db/src/amendments-total-restated.test.ts | 3 + .../db/src/amendments-total-suspect.test.ts | 3 + .../db/src/contractor-identity-sql.test.ts | 6 + .../etl-entity-canonicalization-sql.test.ts | 3 + packages/db/src/refresh-slice.test.ts | 9 + .../db/src/value-flag-annex-step-sql.test.ts | 3 + .../db/src/value-flag-stotinki-sql.test.ts | 3 + packages/ingest/src/refresh.ts | 8 +- scripts/derive-amendments.sql | 21 +- scripts/import.mjs | 8 + scripts/normalize-raw.sql | 10 +- scripts/promote-amendments.sql | 13 +- scripts/refresh-slice.sql | 162 ++++- scripts/resolve-amendment-contracts.sql | 150 +++++ scripts/work-staging-schema.sql | 4 +- 23 files changed, 1452 insertions(+), 17 deletions(-) create mode 100644 docs/implementation-plans/306-amendment-contract-namespace-link.md create mode 100644 packages/db/migrations/0008_amendment_provenance.sql create mode 100644 packages/db/src/amendments-contract-resolve.test.ts create mode 100644 packages/db/src/amendments-slice-resolve.test.ts create mode 100644 scripts/resolve-amendment-contracts.sql diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2bdfdf2e..e15e693d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -222,7 +222,11 @@ jobs: # EXISTS). These are pure additions with safe defaults (INTEGER NOT NULL DEFAULT 0 / nullable TEXT), so # no backfill or completion marker is needed — an ALTER populates every existing row. Malformed # responses are fatal. This also makes 0006/0007 replay-safe when they were applied out-of-ledger. - - name: Apply amendment restated/suspect columns + # #306 folds in here rather than adding a second step of its own (as PR #308's own note asked): + # refresh-slice.sql and promote-amendments.sql write contract_number_raw + link_method into served + # `amendments`, so the first cron after release would crash on the missing columns. One probe, one + # mechanism — a second hand-written step is another chance for the alias bug #310 had to fix. + - name: Apply amendment restated/suspect + provenance columns if: steps.guard.outputs.ok == 'true' run: | node scripts/wrangler-render.mjs apps/web/wrangler.jsonc @@ -235,7 +239,9 @@ jobs: --command "SELECT (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_restated') AS value_restated, (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_treatment') AS value_treatment, - (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_suspect') AS value_suspect")" + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_suspect') AS value_suspect, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'contract_number_raw') AS contract_number_raw, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'link_method') AS link_method")" read_flag() { printf '%s' "$schema_json" | node -e ' @@ -276,6 +282,9 @@ jobs: ensure_column value_restated "value_restated INTEGER NOT NULL DEFAULT 0" ensure_column value_treatment "value_treatment TEXT" ensure_column value_suspect "value_suspect INTEGER NOT NULL DEFAULT 0" + # #306 provenance: NULL on both = the row linked by contract_number directly (or is unlinked). + ensure_column contract_number_raw "contract_number_raw TEXT" + ensure_column link_method "link_method TEXT" # `run deploy`, not `deploy` — bare `pnpm deploy` is a pnpm built-in, not our package script. - name: Deploy explorer (sigma) diff --git a/docs/README.md b/docs/README.md index 01cda08a..16f03f97 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,7 +11,11 @@ - [`etl-architecture.md`](etl-architecture.md) — целевата ETL архитектура (RFC): предложение за състоянието и реда на изпълнение. - [`v1-implementation-plan.md`](v1-implementation-plan.md) — precompute слоят и пагинацията (защо rollup-и и keyset вместо per-request GROUP BY / OFFSET). - [`implementation-plans/286-ocds-amendment-unp.md`](implementation-plans/286-ocds-amendment-unp.md) — защо OCDS анексите не се свързват с договор (OCID вместо УНП) и планът за поправка през bridge-а `tender.id → УНП` + prefer-EOP dedup (#286). +<<<<<<< HEAD +- [`implementation-plans/306-amendment-contract-namespace-link.md`](implementation-plans/306-amendment-contract-namespace-link.md) — защо 1 937 EOP анекса не се свързват с договор (номерът на анекса е в друго именно пространство от деловодния номер) и планът за поправка чрез value-anchor (`value_before → signing_value`, 99.99% точност) (#306). +======= - [`implementation-plans/305-amendment-value-double-count.md`](implementation-plans/305-amendment-value-double-count.md) — защо стойността на анекс се удвоява (ЦАИС ЕОП слага новия **тотал** в полето за промяна) и планът за откриване/поправка: `annex_total_suspect` флаг + текстова хеуристика за възстановяване на истинския тотал (#305). +>>>>>>> origin/main - [`integrity-gate.md`](integrity-gate.md) — reconciliation gate-ът: hard asserts върху тоталите при import/CI. - [`anomaly-report.md`](anomaly-report.md) — cross-row аномалии при опресняване: какво `value_flag` не хваща на ниво отделен договор. - [`deploy.md`](deploy.md) — деплой към Cloudflare: двата Worker-а (`sigma`, `sigma-etl`) и споделеният D1 per environment. diff --git a/docs/implementation-plans/306-amendment-contract-namespace-link.md b/docs/implementation-plans/306-amendment-contract-namespace-link.md new file mode 100644 index 00000000..3b23ff19 --- /dev/null +++ b/docs/implementation-plans/306-amendment-contract-namespace-link.md @@ -0,0 +1,94 @@ +# Implementation Plan: #306 — annexes don't link to a contract (contract number in a different namespace) + +## Executive Summary + +| Field | Value | +|---|---| +| Ticket | [midt-bg/sigma#306](https://github.com/midt-bg/sigma/issues/306) — labels `data-quality`, `etl`, `priority: high` | +| Problem | 1,937 EOP annexes (7.2%) don't link to any contract. #286 fixed the OCDS half (procedure axis: OCID vs УНП). This is the other axis: the **contract number**. The annex carries an internal annex-side number (`148846`, `2886`); the contract carries the buyer's filing number (`Д-226`, `388-2020`). The exact `(unp, contract_number)` join fails, so the annex drops out of every annex→contract→company/authority rollup. | +| Root cause | Two genuinely unrelated identifiers in different namespaces — not dirty strings. Measured: `TRIM`+`UPPER` saves 0, digits-only 11, substring 19. String normalisation is a dead end. | +| Approach | Link by **value**: an annex's `value_before` is the contract's value at amendment time, so it equals the target contract's `signing_value`. Rewrite `raw_amendments.contract_number` to the resolved contract when `value_before` **exactly** (< 0.5 стотинка), currency-matched, uniquely matches one contract on the procedure; propagate across a chain sharing the annex-side number. | +| Complexity | Medium — the resolver is a self-contained CTE; the subtlety is full-vs-slice parity (the #305 bug class) and the slice's windowed staging. | +| Risk | Low on the full path (measured 99.99% precision), gated to leave ambiguous/no-match annexes unlinked. | +| Status | **Full-derive resolver implemented as a gated, standalone script (`scripts/resolve-amendment-contracts.sql`) that runs BEFORE `derive-amendments.sql`, full-derive path only. Slice/daily resolver deferred (documented below).** Validated against the live `sigma-dev` corpus; hardened per the PR #308 review (ordering blocker, group-contradiction rule, cumulative-candidate dedup, EIK guard, provenance columns). | + +--- + +## 1. Problem, verified on real data (`sigma-dev`) + +Reproduced the issue's breakdown exactly on the served corpus (an amendment `a` links iff `contracts c` exists with `c.tender_id = 't:'||a.unp AND c.contract_number = a.contract_number`): + +| check | #306 | live `sigma-dev` | +|---|---|---| +| EOP annexes unlinked | 1930 | **1937** | +| empty contract_number | 0 | 0 | +| empty УНП | 0 | 0 | +| УНП not in `tenders` | 0 | 3 | +| procedure has no contract | 12 | 15 | +| procedure has contracts, none match | 1918 | **1919** | +| procedure has exactly 1 contract (safe) | 1012 | **1012** | +| price-changing | 554 | **555** | + +## 2. The value anchor (the insight beyond the issue) + +The issue proposed linking only the 1,012 single-contract procedures by УНП. Measuring `value_before` → `signing_value` unlocks the multi-contract cases too. On the **already-linked** annexes (ground truth), when `value_before` uniquely matches one contract's `signing_value`: + +| gate | unique matches | correct | precision | +|---|---|---|---| +| within 1% | 9061 | 8975 | 99.08% | +| **exact (< 0.5 стотинка)** | **9349** | **9348** | **99.99%** | + +Exact-cent match wins on **both** precision and recall — the 1% band manufactures ambiguity and admits the #305 "matched a smaller sibling's value" errors. Example `00011-2020-0002` (multi-lot): the unlinked `2886` annexes carry `value_before=56000`, which exactly matches contract `388-2020` (lot 2, signing 56000), **not** `387-2020` (64000) — a case УНП-only linking cannot resolve. + +### Recovery under the shipped gate (exact + currency + unique + chain-propagation) + +| tier | rule | links | +|---|---|---| +| single-contract, value-confirmed | procedure has 1 contract, `value_before` == its `signing_value` | 775 | +| multi-contract, value-unique | `value_before` exactly matches one of ≥2 contracts | 604 | +| **combined (before chain propagation)** | | **~1,379 (71%)** | + +Left **unlinked by design**: value-ambiguous (matches 2+, ~115), no-match (~193, target not yet ingested — the #249 class), no-tender/no-contract (~18). An honest gap beats a wrong contract on a transparency site. + +> **Re-measured on the live `sigma-dev` corpus (PR #308 review).** Read-only reproduction against the served corpus confirms the headline numbers hold under the revised rule: **1,937** unlinked, **1,563** recovered (1,377 direct + 186 propagated), precision **9,348/9,349 = 99.99%**. Two review inferences were corrected by the data: the `value_before IS NULL/≤0` class is **empty** (all 1,937 unlinked annexes carry a value; the `1,379+326≠1,937` gap is tier-estimate rounding, not a value-less class), and the only anchor-disagreement group in the whole corpus is the correct 00026 lot-base case (hence the §3.4 rule revision). **Still owed:** the served corpus is deduped one-row-per-contract, so it does not exercise the cumulative-staging path (§3.1) — that must be re-measured on raw cumulative staging via a local full backfill before the dedup fix is fully validated end-to-end. + +## 3. Fix — full-derive path (`scripts/resolve-amendment-contracts.sql`, run before `derive-amendments.sql`) + +The resolver is a **standalone script** that `import.mjs` runs from `runFullDerive` / `runWorkBackfill` **before** `derive-amendments.sql` — i.e. **before** the #286 prefer-EOP dedup DELETE and its diagnostics. Ordering is load-bearing (PR #308 review, todorkolev #1 blocker): rewriting an EOP annex onto a contract that already kept an OCDS twin, if done *after* the dedup, resurrects the twin (`annex_count = 2` on a one-annex contract) and trips the `amendment-twin-dedup` integrity gate (#303), failing the whole derive. Running first — and above the #286 diagnostics — keeps the dedup the sole twin guard and its dropped/excess counts honest bounds. The resolver reads only `source LIKE 'eop:%'` rows + `raw_contracts`, so it is independent of the OCDS bridge and safe to run first. + +1. `contract_candidates` — `raw_contracts` **deduped to one row per `(unp, contract_number)`** (`ROW_NUMBER() … ORDER BY source DESC, id DESC = 1`), mirroring normalize-raw. `raw_contracts` is cumulative (the same contract recurs across daily buckets; the collapse happens later in normalize-raw), so without this `COUNT(*) OVER` would count staging **rows**, not contracts — mass fail-close on a real rebuild, or a match to a superseded value (PR #308 review, nikimilenkov HIGH 2). +2. `grp` — **all** EOP annexes with no `(unp, contract_number)` contract, grouped by the shared annex-side number `(unp, annex_cnum)` = one contract's chain. Value-less members (`value_before` NULL/≤0) are included so they can inherit the chain target (review MEDIUM 2). +3. `vmatch` — join to `contract_candidates` on `unp`, exact value (`ABS(signing_value − value_before) < 0.005`), **explicit** currency on both sides (blank ≠ blank; review LOW 1), null-tolerant contractor-EIK match (a value collision onto a different contractor is refused for free; review MEDIUM 5). `COUNT(*) OVER (PARTITION BY amendment_id)` flags uniqueness. +4. **Group rule** (reviews todorkolev #2, nikimilenkov MEDIUM 1 & 2, revised against the live corpus): a member's OWN unique (`n_match = 1`) exact match always applies — it is individually trustworthy (the 99.99% figure) and is **not** voided when annex-number siblings point elsewhere, because an annex number can be a **lot-base** shared across contracts (live corpus: `20РП-У50А015` → …-Л01 @ 22569.98 **and** …-Л03 @ 28557.50, each annex exactly-uniquely matching its own lot — the sole disagreement group in the whole corpus, and it is correct). Members with **no** own unique match (value-less admin steps, or later steps whose cumulative value matches no `signing_value`) inherit one agreed group target — but **only** when the direct members agree; disagreement withholds *propagation*, never the direct hits. A member that is itself value-ambiguous (`n_match ≥ 2`) carries its own contradicting evidence and never links, directly or by inheritance. + + > An earlier revision refused the whole group on disagreement (nikimilenkov MEDIUM 1 as first stated). Measured on the live corpus that dropped 2 confirmed-correct multi-lot links and prevented **zero** wrong links (the only disagreement group is the benign 00026 lot-base case), so the rule keeps direct hits and gates only propagation. +5. `UPDATE raw_amendments SET contract_number = resolved, contract_number_raw = , link_method = 'value_anchor'` in place — like the #286 УНП bridge, but **preserving provenance** (review MEDIUM 4). The rollup, `promote-amendments.sql`, and the serving join then link with no further change; the original annex number stays in `contract_number_raw`, which also keeps it in the amendment `natural_key` so a resolved row never collides with a native annex sharing `document_number` on the target (review MEDIUM 3). + +Diagnostic printed by wrangler: `annexes_value_linked`, `eop_annexes_still_unlinked` (complementary predicates over the same mismatch population). + +Idempotent: after the rewrite the row links by number, so a re-run's `grp` no longer selects it and the provenance columns are never re-stamped. + +## 4. Slice / daily path — **implemented** (corpus-safe resolver inside `refresh-slice.sql`) + +Originally deferred: the standalone `resolve-amendment-contracts.sql` is **not** run on the slice path (`runSliceDerive`), because it would execute against `refresh-slice.sql`'s **windowed** `raw_contracts`, where "unique on the procedure" means "unique **in the window**" — a corpus-ambiguous annex looks unique in a narrow window and mislinks, and the measured 99.99% precision (a full-corpus number) would not carry (PR #308 review, nikimilenkov HIGH 1). + +The daily/slice + Worker path now runs a corpus-safe equivalent **inside `refresh-slice.sql`** (search `#306: slice-safe value-anchor resolver`), addressing PR #308 review todorkolev "дневните обновявания": the cron runs only `refresh-slice.sql`, so the fix had to live there or stay inert in production. It applies the same value/currency/EIK anchor, chain rules, and provenance stamping as the full path, differing only in the candidate source: + +- Candidate contracts are drawn from the **served `contracts` table** (the whole corpus; `unp` via the `tender_id` suffix, contractor ЕИК via the winning `bidders.eik_normalized`) **UNIONed** with this window's `raw_contracts`, so uniqueness is asked corpus-wide. It is intentionally **not** under a byte-identical lockstep marker — the candidate source differs by construction. Scans are bounded to procedures with an EOP annex in this window (`window_unps`), keeping the served read a keyed lookup (`idx_contracts_tender_id`). +- Resolved targets land in `refresh_touched_contracts` **for free**: the resolver rewrites the target `contract_number` onto `raw_amendments` in the setup batch, and the existing `@refresh-batch amendments` touch join (`raw_amendments` → `contracts` on `contract_number`) then scopes prior-window targets into it. +- It runs in the setup batch **before** the prefer-EOP dedup DELETE, the same ordering the full path uses before `derive-amendments.sql`'s dedup (review todorkolev #1 blocker — otherwise a rewrite resurrects an OCDS twin and trips `amendment-twin-dedup` #303). +- The "is this annex's number already a real contract?" question is asked over a **value-agnostic** `all_contract_numbers` CTE (every contract number on the procedure, from window `raw_contracts` **and** served `contracts`, regardless of `signing_value`) — **not** over `contract_candidates` (which requires `signing_value > 0` for value matching). This keeps the slice path identical to the full path's `NOT EXISTS … raw_contracts`: an annex that matches a **zero-value** contract by number links by number, and is never value-linked to a neighbour (review todorkolev discrepancy). + +**Deployment (review todorkolev blocker).** `refresh-slice.sql` now writes `contract_number_raw` + `link_method` into served `amendments`, but the deployed DBs are populated out-of-band so `wrangler d1 migrations apply` can't run. `deploy.yml` gains an idempotent step (probe `pragma_table_info`, `ALTER` only the missing columns, malformed response fatal) **before** the Worker deploys — otherwise the first cron after release crashes on the missing columns. If #307 merges first, fold these two columns into its provenance step instead. + +Provenance keeps slice and full keys aligned: the amendment `natural_key` (winners dedup + promotion) uses the raw annex number via `COALESCE(NULLIF(contract_number_raw,''), contract_number, '')`, and the served `amendments` promotion carries `contract_number_raw` + `link_method`. A resolved annex re-emitted in a later slice window re-resolves deterministically to the same key — an honest, never-double-counted link. Where a target contract and its namespace-mismatched annex arrive in the **same** window, the annex still links (window `raw_contracts` is in the candidate union); the residual best-effort gap is only a target never served or staged, which the next full rebuild closes (#286 precedent). + +## 5. Blast radius (auto-corrects on a full rebuild) + +Everything is rebuilt from scratch by `precompute.sql` (DELETE/INSERT), so it self-corrects: `company_totals`, `authority_totals`, `sector_totals`, `home_totals`, `facet_counts`, `flow_pairs`, `cpv_division_stats`, `search_index`, and each contract's `annex_count`/`current_value`/timeline. Overall annex counts and sums on the annexes themselves are unchanged. Two things to verify after the run: (1) `cpv_division_stats` p95/p99 bands shift for affected divisions; (2) **merge-order dependency** — ~23 newly-linked annexes have `value_after ≥ 2× value_before` and should trip the #307 double-count flags (`annex_total_suspect`/`value_suspect`). Those flags do **not** exist on this branch (they are #307, in review); if this merges before #307, the newly-linked ≥2× annexes enter the aggregates **unflagged** until #307 lands. Track the merge order explicitly. + +**Migration numbering (PR #308 review todorkolev "сблъсък на номера").** #307 adds `0006_amendment_restated.sql` + `0007_amendment_value_suspect.sql`; this branch's provenance migration is therefore numbered **`0008_amendment_provenance.sql`** to sit after both. This assumes #307 merges first (it should — the ≥2× flags above depend on it). If #308 lands first instead, renumber to `0006` and have #307 shift to `0007`/`0008`. + +## 6. Tests + +`packages/db/src/amendments-contract-resolve.test.ts` runs the real `resolve-amendment-contracts.sql` + `derive-amendments.sql` (the full-derive composition) against SQLite and pins: single-contract link (00017 shape), multi-lot value disambiguation (00011 shape), chain propagation + `current_value` = last step, value-less chain member inherits (MEDIUM 2), ambiguous-in-chain member stays unlinked (todorkolev #2), whole-group refusal on disagreeing anchors (MEDIUM 1), cumulative-staging duplicate counted once (HIGH 2), EIK guard (MEDIUM 5), value-ambiguous / no-match / currency-guard / blank-vs-blank-currency (LOW 1) → unlinked, twin-ordering (the resolver runs before the prefer-EOP dedup, no `annex_count = 2` — todorkolev #1 blocker), gate (derive-alone does not resolve — HIGH 1), idempotency, natural-key collision avoidance (MEDIUM 3), provenance through promote into served `amendments` (MEDIUM 4), already-linked untouched, and the diagnostic counts. diff --git a/packages/db/migrations/0008_amendment_provenance.sql b/packages/db/migrations/0008_amendment_provenance.sql new file mode 100644 index 00000000..fc8d4727 --- /dev/null +++ b/packages/db/migrations/0008_amendment_provenance.sql @@ -0,0 +1,9 @@ +-- #306: provenance for value-anchor-linked annexes. The resolver in +-- scripts/resolve-amendment-contracts.sql rewrites a namespace-mismatched annex's contract_number to its +-- target contract; unlike the #286 OCDS bridge (whose OCID survives in tender_ext_id + source), the annex +-- number would otherwise be destroyed with no trace, making the 99.99%-precision claim unauditable in +-- production and any "this annex isn't ours" complaint uninvestigable (review nikimilenkov MEDIUM 4). +-- Keep the original annex-side number and stamp the link method so value-linked rows stay enumerable on the +-- served side. NULL on both columns = the row linked by contract_number directly (or is unlinked). +ALTER TABLE amendments ADD COLUMN contract_number_raw TEXT; +ALTER TABLE amendments ADD COLUMN link_method TEXT; diff --git a/packages/db/src/amendments-contract-resolve.test.ts b/packages/db/src/amendments-contract-resolve.test.ts new file mode 100644 index 00000000..f0b57d13 --- /dev/null +++ b/packages/db/src/amendments-contract-resolve.test.ts @@ -0,0 +1,634 @@ +// Issue #306 — EOP annexes whose annex-side number is in a different namespace than the contract number +// (the annex carries an internal number like 148846; the contract carries the buyer's filing number like +// Д-226), so the (unp, contract_number) join drops them out of every annex→contract rollup. The resolver in +// scripts/resolve-amendment-contracts.sql links them by the exact, currency- and contractor-matched +// value_before → signing_value anchor (measured 99.99% precision on the real corpus), uniquely, with chain +// propagation, leaving ambiguous / no-match annexes honestly unlinked. +// +// The resolver is a SEPARATE script that runs BEFORE derive-amendments.sql on the full-derive path only +// (review todorkolev #1 blocker: running after the prefer-EOP dedup resurrects OCDS twins; review +// nikimilenkov HIGH 1: the slice path's windowed raw_contracts can't answer "unique on the procedure"). +// These tests run the REAL scripts against SQLite via the sqlite3 CLI, exactly as runFullDerive composes +// them — resolver, then derive — so the value anchor, dedup, group rule, guards, and provenance are +// exercised as shipped. +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const initSchema = resolve(root, 'packages/db/migrations/0000_init.sql'); +// #305: promote-amendments.sql writes value_restated/value_treatment/value_suspect into served amendments. +const restatedMigration = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const valueSuspectMigration = resolve( + root, + 'packages/db/migrations/0007_amendment_value_suspect.sql', +); +const provenanceMigration = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +const workStagingSchema = resolve(root, 'scripts/work-staging-schema.sql'); +const resolveAmendments = resolve(root, 'scripts/resolve-amendment-contracts.sql'); +const deriveAmendments = resolve(root, 'scripts/derive-amendments.sql'); +const promoteAmendments = resolve(root, 'scripts/promote-amendments.sql'); + +function sqlite(dbPath: string, sql: string): void { + execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8', stdio: 'pipe' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function readScriptCapture(dbPath: string, path: string): string { + return execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + encoding: 'utf8', + }); +} + +// The full-derive composition: value resolver first (review todorkolev #1), then the derive rollup. +function runFullDerive(dbPath: string): void { + readScript(dbPath, resolveAmendments); + readScript(dbPath, deriveAmendments); +} + +// The numeric rows the resolver prints; the #306 diagnostic is its only 2-column row +// (annexes_value_linked | eop_annexes_still_unlinked). +function diagRows(out: string): number[][] { + return out + .split('\n') + .map((line) => line.trim()) + .filter((line) => /^\d+(\|\d+)*$/.test(line)) + .map((line) => line.split('|').map(Number)); +} + +let dir: string; +let db: string; + +beforeEach(() => { + dir = mkdtempSync(resolve(tmpdir(), 'amendments-resolve-')); + db = resolve(dir, 'work.sqlite'); + readScript(db, initSchema); + readScript(db, restatedMigration); + readScript(db, valueSuspectMigration); + readScript(db, provenanceMigration); + readScript(db, workStagingSchema); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe('#306 amendment→contract value-anchor resolver', () => { + it('links a single-contract procedure whose annex number differs (00017 shape) by exact value', () => { + // UNP with ONE contract "ОП-3-016/…" signing 5800; annex carries internal number 11725, value_before + // 5800 (exact). The number join fails; the value anchor links it. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00017-2020-0041','ОП-3-016/03.02.2021г.',5800,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, value_delta, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00017-2020-0041','11725','2026-03-05','A1',5800,5800,0,'BGN');`, + ); + runFullDerive(db); + + // The annex's contract_number is rewritten to the real filing number → it now links. + expect( + sqliteJson<{ contract_number: string; contract_number_raw: string; link_method: string }>( + db, + "SELECT contract_number, contract_number_raw, link_method FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([ + { + contract_number: 'ОП-3-016/03.02.2021г.', + contract_number_raw: '11725', + link_method: 'value_anchor', + }, + ]); + // The contract picks up the annex in its rollup. + expect( + sqliteJson<{ annex_count: number }>( + db, + "SELECT annex_count FROM raw_contracts WHERE contract_number='ОП-3-016/03.02.2021г.'", + ), + ).toEqual([{ annex_count: 1 }]); + }); + + it('disambiguates a multi-contract (multi-lot) procedure by value (00011 shape)', () => { + // Two contracts on one procedure: 387-2020 (signing 64000) and 388-2020 (signing 56000). An unlinked + // annex carries internal number 2886, value_before 56000 → must link to 388-2020, NEVER 387-2020. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','387-2020',64000,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','388-2020',56000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, value_delta, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00011-2020-0002','2886','2026-03-05','A1',56000,55552.5,NULL,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '388-2020' }]); + expect( + sqliteJson<{ contract_number: string; annex_count: number }>( + db, + "SELECT contract_number, annex_count FROM raw_contracts WHERE unp='00011-2020-0002' ORDER BY contract_number", + ), + ).toEqual([ + { contract_number: '387-2020', annex_count: 0 }, + { contract_number: '388-2020', annex_count: 1 }, + ]); + }); + + it('propagates the resolved target across a chain sharing the annex number, so current_value is the last step', () => { + // Three annexes share internal number 2886; only the FIRST carries value_before = signing (56000), the + // later two carry the running cumulative. All must attach to 388-2020, and current_value must be the + // LAST step's value_after (53580.21), not the first. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','387-2020',64000,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','388-2020',56000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00011-2020-0002','2886','2026-03-05','A1',56000,55552.5,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00011-2020-0002','2886','2026-03-06','A2',55552.5,55306.05,'BGN'), + ('eop:annexes:2026-03-05','2026-03-07','00011-2020-0002','2886','2026-03-07','A3',55306.05,53580.21,'BGN');`, + ); + runFullDerive(db); + + // All three annexes now carry the resolved contract_number. + expect( + sqliteJson<{ n: number }>( + db, + "SELECT COUNT(*) AS n FROM raw_amendments WHERE unp='00011-2020-0002' AND contract_number='388-2020'", + ), + ).toEqual([{ n: 3 }]); + // annex_count = 3, current_value = the latest step's after-value. + expect( + sqliteJson<{ annex_count: number; current_value: number }>( + db, + "SELECT annex_count, current_value FROM raw_contracts WHERE contract_number='388-2020'", + ), + ).toEqual([{ annex_count: 3, current_value: 53580.21 }]); + }); + + it('propagates the target to a VALUE-LESS chain member so the chain does not break (review MEDIUM 2)', () => { + // A1 carries value_before = signing (unique anchor → 388-2020); A2 is an administrative annex with NO + // value_before (a term/scope change). A2 shares the annex number 2886, so it must INHERIT 388-2020 — + // else the chain breaks and annex_count under-counts. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','387-2020',64000,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','388-2020',56000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00011-2020-0002','2886','2026-03-05','A1',56000,55552.5,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00011-2020-0002','2886','2026-03-06','A2',NULL,NULL,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ document_number: string; contract_number: string; link_method: string | null }>( + db, + 'SELECT document_number, contract_number, link_method FROM raw_amendments ORDER BY document_number', + ), + ).toEqual([ + { document_number: 'A1', contract_number: '388-2020', link_method: 'value_anchor' }, + { document_number: 'A2', contract_number: '388-2020', link_method: 'value_anchor' }, + ]); + expect( + sqliteJson<{ annex_count: number }>( + db, + "SELECT annex_count FROM raw_contracts WHERE contract_number='388-2020'", + ), + ).toEqual([{ annex_count: 2 }]); + }); + + it('does NOT attach an ambiguous chain member to the group target (review todorkolev #2)', () => { + // Procedure: C-1 @ 500, C-2 @ 500, C-3 @ 1000. Annex 999: A1 value_before 1000 → C-3 (unique anchor); + // A2 value_before 500 → matches C-1 AND C-2 (ambiguous). A2 must stay unlinked — it carries its own + // contradicting evidence — and must NOT corrupt C-3's current_value. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00050-2020-0001','C-1',500,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00050-2020-0001','C-2',500,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00050-2020-0001','C-3',1000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00050-2020-0001','999','2026-03-05','A1',1000,900,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00050-2020-0001','999','2026-03-06','A2',500,450,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ document_number: string; contract_number: string }>( + db, + 'SELECT document_number, contract_number FROM raw_amendments ORDER BY document_number', + ), + ).toEqual([ + { document_number: 'A1', contract_number: 'C-3' }, // unique anchor linked + { document_number: 'A2', contract_number: '999' }, // ambiguous → stays unlinked + ]); + // C-3 keeps A1 only; current_value is A1's after-value (900), NOT the ambiguous A2's 450. + expect( + sqliteJson<{ annex_count: number; current_value: number }>( + db, + "SELECT annex_count, current_value FROM raw_contracts WHERE contract_number='C-3'", + ), + ).toEqual([{ annex_count: 1, current_value: 900 }]); + }); + + it('links each annex of a lot-base group to its OWN lot by its own unique match (real 00026 shape)', () => { + // The annex-side number is a LOT-BASE shared across two lots (real corpus: 20РП-У50А015 → …-Л01 @ + // 22569.98 AND …-Л03 @ 28557.50). Each annex exactly-uniquely matches its OWN lot, so BOTH link — the + // "disagreement" between siblings is benign and must NOT void the individually-trustworthy direct hits. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00026-2020-0027','20РП-У50А015-Л01',22569.98,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00026-2020-0027','20РП-У50А015-Л03',28557.5,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00026-2020-0027','20РП-У50А015','2026-03-05','A1',22569.98,22000,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00026-2020-0027','20РП-У50А015','2026-03-06','A2',28557.5,28000,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ document_number: string; contract_number: string }>( + db, + "SELECT document_number, contract_number FROM raw_amendments WHERE unp='00026-2020-0027' ORDER BY document_number", + ), + ).toEqual([ + { document_number: 'A1', contract_number: '20РП-У50А015-Л01' }, + { document_number: 'A2', contract_number: '20РП-У50А015-Л03' }, + ]); + expect( + sqliteJson<{ contract_number: string; annex_count: number }>( + db, + "SELECT contract_number, annex_count FROM raw_contracts WHERE unp='00026-2020-0027' ORDER BY contract_number", + ), + ).toEqual([ + { contract_number: '20РП-У50А015-Л01', annex_count: 1 }, + { contract_number: '20РП-У50А015-Л03', annex_count: 1 }, + ]); + }); + + it('withholds PROPAGATION when the group disagrees, but keeps the direct hits (review MEDIUM 1, revised)', () => { + // Two direct hits disagree (lot-base spread), plus a value-less admin annex A3 sharing the number. The + // two direct hits still link to their own lots; A3 has no agreed target to inherit → stays unlinked. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00060-2020-0001','D-1',100,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00060-2020-0001','D-2',200,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00060-2020-0001','777','2026-03-05','A1',100,90,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00060-2020-0001','777','2026-03-06','A2',200,180,'BGN'), + ('eop:annexes:2026-03-05','2026-03-07','00060-2020-0001','777','2026-03-07','A3',NULL,NULL,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ document_number: string; contract_number: string }>( + db, + "SELECT document_number, contract_number FROM raw_amendments WHERE unp='00060-2020-0001' ORDER BY document_number", + ), + ).toEqual([ + { document_number: 'A1', contract_number: 'D-1' }, // own unique match stands + { document_number: 'A2', contract_number: 'D-2' }, // own unique match stands + { document_number: 'A3', contract_number: '777' }, // no agreed target → no propagation + ]); + expect( + sqliteJson<{ contract_number: string; annex_count: number }>( + db, + "SELECT contract_number, annex_count FROM raw_contracts WHERE unp='00060-2020-0001' ORDER BY contract_number", + ), + ).toEqual([ + { contract_number: 'D-1', annex_count: 1 }, + { contract_number: 'D-2', annex_count: 1 }, + ]); + }); + + it('counts cumulative-staging duplicates of one contract as a SINGLE candidate (review HIGH 2)', () => { + // The same logical contract is staged from TWO daily EOP buckets (raw_contracts is cumulative). Without + // deduping candidates, the lone annex would see n_match = 2 and be refused; the resolver must collapse + // the duplicate to one candidate and link. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00070-2020-0001','FILING-7',12345,'BGN'), + ('eop:contracts:2026-03-06','2026-03-06','00070-2020-0001','FILING-7',12345,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-06','2026-03-06','00070-2020-0001','88001','2026-03-06','A1',12345,12000,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: 'FILING-7' }]); + }); + + it('applies an EIK guard — a value match onto a different contractor is refused (review MEDIUM 5)', () => { + // Same unp, value, currency, but the annex is contractor 222222222 and the only value-matching contract + // is contractor 111111111 → the row's own EIK contradicts the target → no link. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency, contractor_eik) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00080-2020-0001','FILING-8',9000,'BGN','111111111'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency, contractor_eik) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00080-2020-0001','44001','2026-03-05','A1',9000,8500,'BGN','222222222');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '44001' }]); // unchanged → EIK guard held + }); + + it('leaves a value-AMBIGUOUS annex unlinked (two contracts share the signing value)', () => { + // Both contracts on the procedure have signing 5000; the annex value_before 5000 matches BOTH — the + // resolver must refuse (an honest gap beats a coin-flip). + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00099-2020-0001','LOT-A',5000,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00099-2020-0001','LOT-B',5000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00099-2020-0001','7777','2026-03-05','A1',5000,4800,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '7777' }]); // unchanged → stays unlinked + expect( + sqliteJson<{ total: number }>( + db, + "SELECT COALESCE(SUM(annex_count),0) AS total FROM raw_contracts WHERE unp='00099-2020-0001'", + ), + ).toEqual([{ total: 0 }]); + }); + + it('leaves a NO-MATCH annex unlinked (value_before matches no contract on the procedure)', () => { + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00099-2020-0002','30',700000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00099-2020-0002','24035','2026-03-05','A1',123456,120000,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '24035' }]); // unchanged + }); + + it('holds the exact-cent tolerance — a value 3 BGN off does NOT match (pins the 0.005 constant)', () => { + // signing 5000, value_before 5003 → 3.00 apart: far under a naive "within 5" band but far over the + // 0.5-стотинка exact gate. Must stay unlinked; this is the whole basis of the 99.99% precision claim. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00087-2020-0001','FILING-87',5000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00087-2020-0001','47001','2026-03-05','A1',5003,4800,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '47001' }]); // unchanged → 3 BGN is not an exact-cent match + }); + + it('applies a currency guard — a BGN-valued annex does not match a same-number EUR contract', () => { + // signing_value numerically equals value_before but the currencies differ → not a real value match. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00207-2020-0171','FILING-1',66820,'EUR'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00207-2020-0171','9001','2026-03-05','A1',66820,66000,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '9001' }]); // unchanged → currency guard held + }); + + it('does not match blank-currency against blank-currency (review LOW 1)', () => { + // Both sides carry no explicit currency. Under so tight a gate the resolver must NOT silently agree via + // a 'BGN' default — an explicit currency is required on both sides. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00085-2020-0001','FILING-85',7500,''); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00085-2020-0001','45001','2026-03-05','A1',7500,7000,'');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '45001' }]); // unchanged → no blank-vs-blank match + }); + + it('does NOT resurrect an OCDS twin: resolver runs before the prefer-EOP dedup (review todorkolev #1 blocker)', () => { + // Contract Д-226 already has an OCDS annex on it. An EOP annex carries the unlinked internal number + // 148846 and value_before = signing (5000). Because the resolver runs BEFORE the #286 prefer-EOP DELETE, + // the EOP annex is moved onto Д-226 first, the OCDS twin is then dropped, and annex_count = 1. If the + // order were reversed the twin would survive and annex_count would be 2 (the twin-dedup #303 gate would + // then hard-fail the whole derive). + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00090-2020-0001','Д-226',5000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('ocds:annexes:2026-03-05','2026-03-05','00090-2020-0001','Д-226','2026-03-05','ocds-e82gsb-1',5000,NULL,'BGN'), + ('eop:annexes:2026-03-05','2026-03-05','00090-2020-0001','148846','2026-03-05','148846-1',5000,4500,'BGN');`, + ); + runFullDerive(db); + + // Exactly one annex survives on Д-226 (the EOP one), and it drives current_value. + expect( + sqliteJson<{ annex_count: number; current_value: number }>( + db, + "SELECT annex_count, current_value FROM raw_contracts WHERE contract_number='Д-226'", + ), + ).toEqual([{ annex_count: 1, current_value: 4500 }]); + // No twin duplication: exactly one row lands on Д-226 for this procedure. + expect( + sqliteJson<{ n: number }>( + db, + "SELECT COUNT(*) AS n FROM raw_amendments WHERE unp='00090-2020-0001' AND contract_number='Д-226'", + ), + ).toEqual([{ n: 1 }]); + }); + + it('is GATED to the resolver script: derive-amendments.sql alone does not resolve (review HIGH 1)', () => { + // The slice path runs derive-amendments.sql WITHOUT the resolver. Running derive alone must leave a + // resolvable annex untouched — proving the link is gated to the full-derive-only resolver script. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00095-2020-0001','FILING-95',5800,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00095-2020-0001','11725','2026-03-05','A1',5800,5600,'BGN');`, + ); + readScript(db, deriveAmendments); // derive ONLY — no resolver + + expect( + sqliteJson<{ contract_number: string; link_method: string | null }>( + db, + "SELECT contract_number, link_method FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '11725', link_method: null }]); // untouched + }); + + it('is idempotent: a second full derive links nothing new and keeps annex_count stable', () => { + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00096-2020-0001','FILING-96',5800,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00096-2020-0001','11725','2026-03-05','A1',5800,5600,'BGN');`, + ); + runFullDerive(db); + const secondPass = readScriptCapture(db, resolveAmendments); + readScript(db, deriveAmendments); + + // Second pass over already-resolved staging: the row now links by number so grp is empty and no new + // rewrite happens. The diagnostic still reports the one value-linked row (link_method persists) and 0 + // still unlinked — i.e. nothing new linked, nothing left over. + const twoCol = diagRows(secondPass).filter((r) => r.length === 2); + expect(twoCol[twoCol.length - 1]).toEqual([1, 0]); + // contract_number stays resolved; annex_count stable at 1; provenance preserved. + expect( + sqliteJson<{ annex_count: number }>( + db, + "SELECT annex_count FROM raw_contracts WHERE contract_number='FILING-96'", + ), + ).toEqual([{ annex_count: 1 }]); + expect( + sqliteJson<{ contract_number_raw: string; link_method: string }>( + db, + "SELECT contract_number_raw, link_method FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number_raw: '11725', link_method: 'value_anchor' }]); + }); + + it('carries provenance through promote into the served amendments table (review MEDIUM 4)', () => { + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00097-2020-0001','FILING-97',5800,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00097-2020-0001','11725','2026-03-05','A1',5800,5600,'BGN');`, + ); + runFullDerive(db); + readScript(db, promoteAmendments); + + // The served row is enumerable as value-linked, and the original annex number survives as provenance. + expect( + sqliteJson<{ contract_number: string; contract_number_raw: string; link_method: string }>( + db, + "SELECT contract_number, contract_number_raw, link_method FROM amendments WHERE unp='00097-2020-0001'", + ), + ).toEqual([ + { contract_number: 'FILING-97', contract_number_raw: '11725', link_method: 'value_anchor' }, + ]); + // The audit count nikimilenkov asked for is a plain query on the served side. + expect( + sqliteJson<{ n: number }>( + db, + "SELECT COUNT(*) AS n FROM amendments WHERE link_method='value_anchor'", + ), + ).toEqual([{ n: 1 }]); + }); + + it('a resolved annex does not collide with a native annex sharing document_number on the target (review MEDIUM 3)', () => { + // Contract FILING-98 already has a native annex whose document_number is 'DOC-1'. An unlinked annex 55501 + // resolves to FILING-98 and ALSO has document_number 'DOC-1'. Keying the resolved row on its original + // annex number keeps both rows distinct — the real annex is not silently dropped by the dedup. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00098-2020-0001','FILING-98',5000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00098-2020-0001','FILING-98','2026-03-04','DOC-1',6000,5500,'BGN'), + ('eop:annexes:2026-03-05','2026-03-05','00098-2020-0001','55501','2026-03-05','DOC-1',5000,4500,'BGN');`, + ); + runFullDerive(db); + + // Both annexes survive on FILING-98 (the native one and the value-linked one). + expect( + sqliteJson<{ annex_count: number }>( + db, + "SELECT annex_count FROM raw_contracts WHERE contract_number='FILING-98'", + ), + ).toEqual([{ annex_count: 2 }]); + }); + + it('never touches an annex that already links by contract_number, and emits the #306 diagnostic', () => { + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00001-2020-0001','Д-100',5000,'BGN'), -- annex links by value (unlinked number) + ('eop:contracts:2026-03-05','2026-03-05','00002-2020-0001','Д-200',9000,'BGN'); -- annex already links by number + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00001-2020-0001','55501','2026-03-05','A1',5000,4500,'BGN'), + ('eop:annexes:2026-03-05','2026-03-05','00002-2020-0001','Д-200','2026-03-05','A2',9000,8000,'BGN');`, + ); + const out = readScriptCapture(db, resolveAmendments); + readScript(db, deriveAmendments); + + // A1 got resolved; A2 (already linked by number) is untouched and keeps NULL provenance. + expect( + sqliteJson<{ document_number: string; contract_number: string; link_method: string | null }>( + db, + 'SELECT document_number, contract_number, link_method FROM raw_amendments ORDER BY document_number', + ), + ).toEqual([ + { document_number: 'A1', contract_number: 'Д-100', link_method: 'value_anchor' }, + { document_number: 'A2', contract_number: 'Д-200', link_method: null }, + ]); + // Diagnostic: exactly 1 value-linked, 0 still unlinked. The resolver's only 2-column row. + const twoCol = diagRows(out).filter((r) => r.length === 2); + expect(twoCol[twoCol.length - 1]).toEqual([1, 0]); + }); +}); diff --git a/packages/db/src/amendments-ocds-link.test.ts b/packages/db/src/amendments-ocds-link.test.ts index 73dd62a5..e86eb5bb 100644 --- a/packages/db/src/amendments-ocds-link.test.ts +++ b/packages/db/src/amendments-ocds-link.test.ts @@ -16,6 +16,8 @@ const initSchema = resolve(root, 'packages/db/migrations/0000_init.sql'); // #305 Tier-2: promote-amendments.sql writes value_restated/value_treatment to served amendments. const migration6 = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7 = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8 = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const workStagingSchema = resolve(root, 'scripts/work-staging-schema.sql'); const deriveAmendments = resolve(root, 'scripts/derive-amendments.sql'); const promoteAmendments = resolve(root, 'scripts/promote-amendments.sql'); @@ -77,6 +79,7 @@ beforeEach(() => { readScript(db, initSchema); // served `amendments` + `contracts` readScript(db, migration6); // #305 Tier-2 value_restated/value_treatment on served amendments readScript(db, migration7); // #305 residual value_suspect on served amendments + readScript(db, migration8); // #306 provenance columns on served amendments readScript(db, workStagingSchema); // raw_* staging // Two EOP procedures: T1 (contract 90029) already has an EOP annex; T2 (contract 55500) has NONE. diff --git a/packages/db/src/amendments-slice-resolve.test.ts b/packages/db/src/amendments-slice-resolve.test.ts new file mode 100644 index 00000000..931dc449 --- /dev/null +++ b/packages/db/src/amendments-slice-resolve.test.ts @@ -0,0 +1,303 @@ +// Issue #306 (PR #308 review todorkolev "дневните обновявания") — the value-anchor resolver must also run on +// the daily/slice + Worker path, not only the full derive. `scripts/refresh-slice.sql` carries a corpus-safe +// resolver whose candidate contracts come from the served `contracts` table (the whole corpus) UNIONed with +// the current window's raw_contracts, so "unique on the procedure" is asked corpus-wide — closing the gap that +// kept the full-path resolver off the slice (windowed raw_contracts could only answer "unique in the window"). +// +// These tests run the REAL refresh-slice.sql against SQLite via the sqlite3 CLI, exactly as the CLI slice path +// and the Worker compose it: a first window promotes a served corpus, then a later window brings a +// namespace-mismatched annex (its annex-side number is in a different namespace than the contract's filing +// number). The annex links by the exact value_before → signing_value anchor, keeps its provenance, and rolls +// onto the prior-window target — while a corpus-ambiguous annex that would look unique in the window stays +// honestly unlinked. +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migrations = [ + 'packages/db/migrations/0000_init.sql', + 'packages/db/migrations/0001_flow_pairs_bidder_index.sql', + 'packages/db/migrations/0002_current_value_currency.sql', + 'packages/db/migrations/0003_related_persons_foundation.sql', + // #305: refresh-slice.sql writes value_restated/value_treatment/value_suspect into served amendments. + 'packages/db/migrations/0006_amendment_restated.sql', + 'packages/db/migrations/0007_amendment_value_suspect.sql', + 'packages/db/migrations/0008_amendment_provenance.sql', +].map((p) => resolve(root, p)); +const workStagingSchema = resolve(root, 'scripts/work-staging-schema.sql'); +const refreshSlice = resolve(root, 'scripts/refresh-slice.sql'); + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function sqlite(dbPath: string, sql: string): void { + execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8', stdio: 'pipe' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +// Fresh transient staging for a new window, exactly as import.mjs / the Worker do between refreshes. +function resetStaging(dbPath: string): void { + const rows = sqliteJson<{ name: string }>( + dbPath, + "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'raw_%' ORDER BY name DESC", + ); + for (const row of rows) sqlite(dbPath, `DROP TABLE IF EXISTS "${row.name}";`); + readScript(dbPath, workStagingSchema); +} + +// One EOP procedure header (a tender) so contract promotion has an authority + tender to attach to. +function seedTender(dbPath: string, unp: string, authorityEik: string): void { + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject, + cpv_code, cpv_description, contract_kind, estimated_value, currency, authority_name, + authority_eik, authority_type, published_at) + VALUES + ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', '${unp}', 'T-${unp}', 'open', + 'Subject ${unp}', '45000000', 'Construction', 'works', 5000, 'BGN', 'Authority ${unp}', + '${authorityEik}', 'public', '2026-06-01');`, + ); +} + +// An EOP base contract. `cnum` is the contract's filing number; `signing` its signing value. +function seedContract( + dbPath: string, + opts: { unp: string; cnum: string; signing: number; authorityEik: string; contractorEik: string }, +): void { + const { unp, cnum, signing, authorityEik, contractorEik } = opts; + sqlite( + dbPath, + `INSERT INTO raw_contracts + (source, dataset_year, dataset_variant, fetched_at, needs_enrichment, document_number, + published_at, unp, tender_ext_id, procedure_type, procurement_subject, cpv_code, + cpv_description, contract_kind, estimated_value, procurement_currency, authority_name, + authority_eik, authority_type, contract_number, contract_date, signing_value, currency, + contract_subject, awarded_to_group, contractor_eik, contractor_name) + VALUES + ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-${cnum}', + '2026-06-01', '${unp}', 'T-${unp}', 'open', 'Subject ${unp}', '45000000', 'Construction', + 'works', 5000, 'BGN', 'Authority ${unp}', '${authorityEik}', 'public', '${cnum}', + '2026-06-02', ${signing}, 'BGN', 'Contract ${cnum}', 0, '${contractorEik}', 'Bidder ${unp}');`, + ); +} + +// A namespace-mismatched EOP annex: `annexCnum` is the annex-side internal number (matches no contract by +// number); it must link to a contract by the value_before anchor instead. +function seedAnnex( + dbPath: string, + opts: { + unp: string; + annexCnum: string; + valueBefore: number; + valueAfter: number; + authorityEik: string; + contractorEik: string; + }, +): void { + const { unp, annexCnum, valueBefore, valueAfter, authorityEik, contractorEik } = opts; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, contract_number, + contract_date, published_at, unp, authority_eik, authority_name, procurement_subject, + contract_kind, value_before, value_after, value_delta, currency, contractor_eik, description) + VALUES + ('eop:annexes:2026-06-08', 2026, 'eop', '2026-06-08T00:00:00Z', '1', 'AMD-${annexCnum}', + '${annexCnum}', '2026-06-02', '2026-06-09', '${unp}', '${authorityEik}', 'Authority ${unp}', + 'Subject ${unp}', 'works', ${valueBefore}, ${valueAfter}, ${valueAfter - valueBefore}, 'BGN', + '${contractorEik}', 'Namespace-mismatched annex');`, + ); +} + +let dir: string; +let db: string; + +beforeEach(() => { + dir = mkdtempSync(resolve(tmpdir(), 'amendments-slice-resolve-')); + db = resolve(dir, 'work.sqlite'); + for (const m of migrations) readScript(db, m); + readScript(db, workStagingSchema); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +interface AmendmentRow { + contract_number: string; + contract_number_raw: string | null; + link_method: string | null; +} +interface ContractRow { + contract_number: string; + annex_count: number; + current_value: number | null; +} + +describe('refresh-slice #306 value-anchor resolver', () => { + it('links a window annex to a prior-window served contract by the corpus value anchor', () => { + // Window 1: the base contract, filing number Д-226, is served on its own. + seedTender(db, 'UNP-NS', '123456786'); + seedContract(db, { + unp: 'UNP-NS', + cnum: 'Д-226', + signing: 1000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // Window 2: only the annex arrives, carrying the internal number 148846 — matches no contract by number, + // but its value_before (1000) is the served contract's signing_value. + resetStaging(db); + seedAnnex(db, { + unp: 'UNP-NS', + annexCnum: '148846', + valueBefore: 1000, + valueAfter: 1200, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + const amendments = sqliteJson( + db, + "SELECT contract_number, contract_number_raw, link_method FROM amendments WHERE unp='UNP-NS'", + ); + // Rewritten onto the target contract's filing number, with the annex-side number kept as provenance. + expect(amendments).toEqual([ + { contract_number: 'Д-226', contract_number_raw: '148846', link_method: 'value_anchor' }, + ]); + + // The prior-window target was touched and re-rolled: the annex now counts and drives current_value. + const contract = sqliteJson( + db, + "SELECT contract_number, annex_count, current_value FROM contracts WHERE tender_id='t:UNP-NS'", + ); + expect(contract).toEqual([{ contract_number: 'Д-226', annex_count: 1, current_value: 1200 }]); + }); + + it('leaves a corpus-ambiguous annex unlinked even when it is unique within the window', () => { + // Window 1: two served contracts on the SAME procedure, both signing_value 1000 — the annex value cannot + // pick between them across the corpus. + seedTender(db, 'UNP-AMB', '123456786'); + seedContract(db, { + unp: 'UNP-AMB', + cnum: 'Д-1', + signing: 1000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + seedContract(db, { + unp: 'UNP-AMB', + cnum: 'Д-2', + signing: 1000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // Window 2: a THIRD matching contract Д-3 (1000) plus the mismatched annex. A windowed-only resolver would + // see just Д-3 and mislink (n_match = 1); the corpus-aware resolver sees Д-1/Д-2/Д-3 and refuses. + resetStaging(db); + seedContract(db, { + unp: 'UNP-AMB', + cnum: 'Д-3', + signing: 1000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + seedAnnex(db, { + unp: 'UNP-AMB', + annexCnum: '148846', + valueBefore: 1000, + valueAfter: 1200, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // Unlinked: the annex keeps its raw annex-side number and no link_method. + const amendments = sqliteJson( + db, + "SELECT contract_number, contract_number_raw, link_method FROM amendments WHERE unp='UNP-AMB'", + ); + expect(amendments).toEqual([ + { contract_number: '148846', contract_number_raw: null, link_method: null }, + ]); + + // None of the three candidate contracts absorbed the annex. + const amended = sqliteJson<{ n: number }>( + db, + "SELECT COUNT(*) AS n FROM contracts WHERE tender_id='t:UNP-AMB' AND annex_count > 0", + ); + expect(amended).toEqual([{ n: 0 }]); + }); + + it('keeps an annex on its zero-value contract it matches BY NUMBER (never value-links to a neighbour)', () => { + // The full path excludes an annex whose (unp, contract_number) is a real contract, regardless of that + // contract's value. The slice path must agree: Д-1 has signing_value 0, Д-2 has 5000; the annex is numbered + // Д-1 with value_before 5000. It matches Д-1 by number, so it must stay on Д-1 — NOT get value-linked to Д-2 + // just because Д-1 fails the signing_value > 0 candidate filter (review todorkolev: the paths must agree). + seedTender(db, 'UNP-ZERO', '123456786'); + seedContract(db, { + unp: 'UNP-ZERO', + cnum: 'Д-2', + signing: 5000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // Window 2: the zero-value contract Д-1 arrives with the annex that carries its number. + resetStaging(db); + seedContract(db, { + unp: 'UNP-ZERO', + cnum: 'Д-1', + signing: 0, + authorityEik: '123456786', + contractorEik: '987654308', + }); + seedAnnex(db, { + unp: 'UNP-ZERO', + annexCnum: 'Д-1', + valueBefore: 5000, + valueAfter: 9000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // The annex stays on Д-1 by number — not rewritten to the value-neighbour Д-2. + const amendments = sqliteJson( + db, + "SELECT contract_number, contract_number_raw, link_method FROM amendments WHERE unp='UNP-ZERO'", + ); + expect(amendments).toEqual([ + { contract_number: 'Д-1', contract_number_raw: null, link_method: null }, + ]); + + // Д-1 absorbs the annex; the value-neighbour Д-2 is untouched. + const perContract = sqliteJson<{ contract_number: string; annex_count: number }>( + db, + "SELECT contract_number, annex_count FROM contracts WHERE tender_id='t:UNP-ZERO' ORDER BY contract_number", + ); + expect(perContract).toEqual([ + { contract_number: 'Д-1', annex_count: 1 }, + { contract_number: 'Д-2', annex_count: 0 }, + ]); + }); +}); diff --git a/packages/db/src/amendments-sql.test.ts b/packages/db/src/amendments-sql.test.ts index 75ceea4b..e4f56dfa 100644 --- a/packages/db/src/amendments-sql.test.ts +++ b/packages/db/src/amendments-sql.test.ts @@ -20,6 +20,8 @@ const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration6 = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); // #305 residual: AMENDMENTS_SQL also selects am.value_suspect, added by this migration. const migration7 = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8 = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); @@ -71,6 +73,7 @@ function withDb(fn: (dbPath: string) => T): T { readScript(dbPath, migration0); readScript(dbPath, migration6); readScript(dbPath, migration7); + readScript(dbPath, migration8); return fn(dbPath); } finally { rmSync(dir, { recursive: true, force: true }); diff --git a/packages/db/src/amendments-total-restated.test.ts b/packages/db/src/amendments-total-restated.test.ts index 5909f710..2eee4fad 100644 --- a/packages/db/src/amendments-total-restated.test.ts +++ b/packages/db/src/amendments-total-restated.test.ts @@ -25,6 +25,8 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); @@ -63,6 +65,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/amendments-total-suspect.test.ts b/packages/db/src/amendments-total-suspect.test.ts index 2fa3c913..e34fdbab 100644 --- a/packages/db/src/amendments-total-suspect.test.ts +++ b/packages/db/src/amendments-total-suspect.test.ts @@ -26,6 +26,8 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); // #305 residual: served amendments gained value_suspect (promote + refresh-slice write it). const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); @@ -68,6 +70,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index e47c775a..b46d01bb 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -26,6 +26,11 @@ const migration7 = readFileSync( resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'), 'utf8', ); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8 = readFileSync( + resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'), + 'utf8', +); const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); const normalize = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); const precompute = readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8'); @@ -79,6 +84,7 @@ function build(path: 'normalize' | 'refresh'): DatabaseSync { db.exec(migration3); db.exec(migration6); db.exec(migration7); + db.exec(migration8); db.exec(staging); db.exec(seed); if (path === 'normalize') { diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index 7a682ffa..53012248 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -14,6 +14,8 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -45,6 +47,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 3685e1e4..32ecd676 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -17,6 +17,8 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); // #305 residual: served amendments gained value_suspect (refresh-slice promotes it). const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const deriveAmendmentsPath = resolve(root, 'scripts/derive-amendments.sql'); @@ -191,6 +193,7 @@ function initWorkDb(dbPath: string): void { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); } @@ -584,6 +587,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -668,6 +672,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); // An EOP procedure (tender + base contract) with УНП UNP-SLICE / tender.id TENDER-SLICE. An @@ -853,6 +858,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -906,6 +912,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -959,6 +966,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -1098,6 +1106,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, diff --git a/packages/db/src/value-flag-annex-step-sql.test.ts b/packages/db/src/value-flag-annex-step-sql.test.ts index bfa8af32..c4872177 100644 --- a/packages/db/src/value-flag-annex-step-sql.test.ts +++ b/packages/db/src/value-flag-annex-step-sql.test.ts @@ -29,6 +29,8 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const promotePath = resolve(root, 'scripts/promote-amendments.sql'); @@ -65,6 +67,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/value-flag-stotinki-sql.test.ts b/packages/db/src/value-flag-stotinki-sql.test.ts index d2fe07e5..a13808aa 100644 --- a/packages/db/src/value-flag-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-stotinki-sql.test.ts @@ -23,6 +23,8 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -54,6 +56,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration3Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/ingest/src/refresh.ts b/packages/ingest/src/refresh.ts index 45808bf3..0aed14ad 100644 --- a/packages/ingest/src/refresh.ts +++ b/packages/ingest/src/refresh.ts @@ -98,6 +98,12 @@ const LEGACY_TRANSIENT_STAGING_TABLES = [ 'raw_egov_amendments', ] as const; +// Scratch tables that live only for the span of a single derive step. Each is DROP-guarded at the top of +// its own script, so it self-heals on the next run; listing it here also sweeps it after an aborted run so +// it never lingers in D1 (review nikimilenkov LOW 2 — #306's value-resolver scratch table). Not part of +// work-staging-schema.sql, so it stays out of transientStagingStatements' recreate path. +const SCRATCH_TABLES = ['amendment_contract_resolve'] as const; + function touchesTransientStaging(statement: string): boolean { return TRANSIENT_STAGING_TABLES.some((table) => statement.includes(table)); } @@ -148,7 +154,7 @@ export function fullClearTables(normalizeRawSql: string): string[] { } export function dropTransientStagingStatements(): string[] { - return [...TRANSIENT_STAGING_TABLES, ...LEGACY_TRANSIENT_STAGING_TABLES] + return [...SCRATCH_TABLES, ...TRANSIENT_STAGING_TABLES, ...LEGACY_TRANSIENT_STAGING_TABLES] .reverse() .map((table) => `DROP TABLE IF EXISTS ${table}`); } diff --git a/scripts/derive-amendments.sql b/scripts/derive-amendments.sql index 23dcee7a..217490ea 100644 --- a/scripts/derive-amendments.sql +++ b/scripts/derive-amendments.sql @@ -1,8 +1,9 @@ -- Sigma — roll raw_amendments up onto raw_contracts. -- Run AFTER scripts/load-eop.mjs (which stages the EOP base + in-bucket OCDS amendments). --- Re-runnable. First recovers the УНП for OCDS amendments via the tender.id bridge (#286), then --- prefers the EOP annex over its OCDS twin (dropping the twin), then matches amendments by --- (unp, contract_number). +-- Re-runnable. On the full-derive path scripts/resolve-amendment-contracts.sql (#306) has already rewritten +-- namespace-mismatched EOP annexes onto their target contract_number BEFORE this file runs. This file then +-- recovers the УНП for OCDS amendments via the tender.id bridge (#286), prefers the EOP annex over its OCDS +-- twin (dropping the twin), and matches amendments by (unp, contract_number). -- current_value = the after-value of the LATEST amendment; annex_count = how many. -- Contracts without amendments keep annex_count = 0 and current_value = NULL (the -- convention downstream is COALESCE(current_value, signing_value)). @@ -110,12 +111,24 @@ WHERE source LIKE 'ocds:%' AND e.contract_number = raw_amendments.contract_number ); +-- #306: the value-anchor resolver that links namespace-mismatched EOP annexes (annex-side number ≠ the +-- contract's filing number) by rewriting raw_amendments.contract_number lives in +-- scripts/resolve-amendment-contracts.sql and runs BEFORE this file on the full-derive path only (see that +-- file's header for the ordering-blocker and slice-gating rationale — reviews todorkolev #1/#3, nikimilenkov +-- HIGH 1). By the time this rollup runs, resolved annexes already carry their target contract_number (and +-- keep the original annex number in contract_number_raw for the natural_key below), so the rollup links them +-- with no further change. + UPDATE raw_contracts SET annex_count = 0, current_value = NULL; WITH keyed AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + -- #306: key on the ORIGINAL annex-side number (contract_number_raw) when the value resolver rewrote a + -- row, so a resolved annex keeps a stable identity and never collides with a native annex that shares + -- document_number on the target contract (review nikimilenkov MEDIUM 3). NULL raw → the plain number, + -- so unresolved rows and the slice path (which never resolves) produce byte-identical keys. + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), diff --git a/scripts/import.mjs b/scripts/import.mjs index a841759c..fe2aba16 100644 --- a/scripts/import.mjs +++ b/scripts/import.mjs @@ -285,6 +285,12 @@ function assertDeriveWindowSafe(mode, from) { } async function runFullDerive() { + // #306: link namespace-mismatched EOP annexes by value BEFORE derive-amendments.sql (its prefer-EOP + // dedup would otherwise resurrect OCDS twins — review todorkolev #1). This STANDALONE script is the + // full-derive form (candidates from the whole re-staged raw_contracts). The daily/slice + Worker path runs + // an equivalent value anchor inside refresh-slice.sql that draws candidates from the served `contracts` + // corpus instead of the window (so "unique on the procedure" is corpus-wide — review nikimilenkov HIGH 1). + execSql(resolve(root, 'scripts/resolve-amendment-contracts.sql')); execSql(resolve(root, 'scripts/derive-amendments.sql')); run('node', ['scripts/load-fx.mjs', '--apply', ...passthru]); execSql(resolve(root, 'scripts/load-nuts.sql')); @@ -362,6 +368,8 @@ async function runWorkBackfill() { `--out=${resolve(workDir, `${stem}.eop-load.sql`)}`, ...loadFlags, ]); + // #306: value-anchor resolver runs first on the full-derive path — see runFullDerive. + sqliteFile(workDb, resolve(root, 'scripts/resolve-amendment-contracts.sql')); sqliteFile(workDb, resolve(root, 'scripts/derive-amendments.sql')); run('node', [ 'scripts/load-fx.mjs', diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index c5ba2c72..1c1f7a11 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -732,7 +732,10 @@ INSERT OR IGNORE INTO contracts -- travels alongside the value it minted — computed ONCE over raw_amendments, not per-row). WITH amendment_dedup AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + -- #306: key on the original annex-side number (contract_number_raw) when the value resolver rewrote a + -- row — must match derive-amendments.sql / promote-amendments.sql byte-for-byte (review nikimilenkov + -- MEDIUM 3). NULL raw → the plain number, so unresolved rows are unaffected. + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), @@ -1278,7 +1281,10 @@ CREATE TABLE IF NOT EXISTS pipeline_stats ( DELETE FROM pipeline_stats; WITH amendment_dedup AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + -- #306: key on the original annex-side number (contract_number_raw) when the value resolver rewrote a + -- row — must match derive-amendments.sql / promote-amendments.sql byte-for-byte (review nikimilenkov + -- MEDIUM 3). NULL raw → the plain number, so unresolved rows are unaffected. + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), diff --git a/scripts/promote-amendments.sql b/scripts/promote-amendments.sql index b7822315..c5e70557 100644 --- a/scripts/promote-amendments.sql +++ b/scripts/promote-amendments.sql @@ -7,13 +7,18 @@ DELETE FROM amendments; INSERT OR REPLACE INTO amendments ( - id, natural_key, contract_number, unp, value_before, value_after, value_delta, currency, - published_at, document_number, description, source, value_restated, value_treatment, value_suspect + id, natural_key, contract_number, contract_number_raw, link_method, unp, + value_before, value_after, value_delta, currency, + published_at, document_number, description, source, + value_restated, value_treatment, value_suspect ) WITH keyed AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + -- #306: key on the original annex-side number (contract_number_raw) when the value resolver rewrote a + -- row — MUST match derive-amendments.sql's rollup key byte-for-byte, else annex_count and the served + -- timeline disagree (review nikimilenkov MEDIUM 3). NULL raw → the plain number. + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), @@ -38,6 +43,8 @@ SELECT natural_key, natural_key, contract_number, + contract_number_raw, -- #306 provenance: the annex-side number before the value resolver rewrote it + link_method, -- #306 provenance: 'value_anchor' for value-linked rows, else NULL unp, value_before, -- #305 Tier-2: serve the effective (text-corrected) after and a self-consistent delta; a restated diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index cdaf8527..5bd90796 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -21,6 +21,7 @@ DROP TABLE IF EXISTS refresh_unp_prefix_authorities; DROP TABLE IF EXISTS refresh_joint_authority_members; DROP TABLE IF EXISTS refresh_joint_tender_sources; DROP TABLE IF EXISTS refresh_amendment_winners; +DROP TABLE IF EXISTS refresh_amendment_contract_resolve; CREATE TABLE refresh_touched_contracts (id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_bidders (bidder_id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_authorities (authority_id TEXT PRIMARY KEY); @@ -64,6 +65,155 @@ WHERE source LIKE 'ocds:%' ); -- @bridge-lockstep end +-- #306: slice-safe value-anchor resolver. Links EOP annexes whose annex-side number is in a different +-- namespace than the contract's filing number (annex carries an internal number like 148846; the contract +-- carries Д-226), by the exact value_before → signing_value anchor — the same 99.99%-precision link the +-- full path applies in scripts/resolve-amendment-contracts.sql. It runs HERE, before the prefer-EOP dedup +-- DELETE below and the amendment promotion further down, for the SAME reason the full path runs it before +-- derive-amendments.sql's dedup: rewriting an EOP annex onto a contract that already kept an OCDS twin would +-- resurrect the twin and trip amendment-twin-dedup (#303, review todorkolev #1). +-- +-- The full path is gated OFF the slice because its candidates came from the WINDOWED raw_contracts, where +-- "unique on the procedure" means "unique in the window" and a corpus-ambiguous annex would mislink (review +-- nikimilenkov HIGH 1). This slice version closes that gap the way the deferred plan §4 prescribes: candidate +-- contracts are drawn from the CUMULATIVE served `contracts` (the whole corpus) UNIONed with this window's +-- raw_contracts, so uniqueness is asked over the corpus, not the window — the measured precision carries. It +-- is intentionally NOT under a byte-identical lockstep marker with the full-path script: the candidate source +-- differs by construction. Scans are bounded to the procedures that actually have an EOP annex in this window +-- (window_unps), so the served-corpus read stays a keyed lookup, not a full scan (idx_contracts_tender_id). +-- Resolved prior-window targets need no extra touch-wiring: the rewrite below lands the target contract_number +-- on raw_amendments, and the existing `@refresh-batch amendments` touch join (raw_amendments → contracts on +-- contract_number) then scopes those targets into refresh_touched_contracts for free (plan §4). +DROP TABLE IF EXISTS refresh_amendment_contract_resolve; +CREATE TABLE refresh_amendment_contract_resolve AS +WITH window_unps AS ( + SELECT DISTINCT unp FROM raw_amendments + WHERE source LIKE 'eop:%' AND unp IS NOT NULL AND contract_number IS NOT NULL +), +-- Every contract NUMBER that exists on the affected procedures — from BOTH this window's raw_contracts and the +-- served corpus, REGARDLESS of signing_value. `grp` below asks this to decide "is the annex's number already a +-- real contract on the procedure". It MUST be value-agnostic: the full path asks only "does such a contract +-- exist" (NOT EXISTS over raw_contracts), so keying the slice question off contract_candidates (which requires +-- signing_value > 0) would diverge — an annex that points to a ZERO-value contract by number would look +-- namespace-mismatched and get value-linked to a neighbour, contradicting "links by number, not value" and +-- making the two paths disagree (review todorkolev). +all_contract_numbers AS ( + SELECT unp, contract_number FROM raw_contracts + WHERE contract_number IS NOT NULL AND unp IN (SELECT unp FROM window_unps) + UNION + SELECT substr(tender_id, 3) AS unp, contract_number FROM contracts + WHERE contract_number IS NOT NULL AND tender_id IN (SELECT 't:' || unp FROM window_unps) +), +-- One row per LOGICAL contract on the affected procedures, for VALUE matching (signing_value > 0). Two sources, +-- deduped: (0) this window's raw_contracts — cumulative EOP buckets repeat a contract across days, so collapse +-- to the latest source-day then highest id, mirroring normalize-raw; (1) the served `contracts` corpus (unp = +-- the tender_id suffix, contractor ЕИК via the winning bidder). A contract present in BOTH is one logical +-- contract — src_rank keeps the window row (freshest signing_value) and the COUNT below never double-counts it +-- into a false ambiguity. +contract_candidates AS ( + SELECT unp, contract_number, signing_value, currency, contractor_eik FROM ( + SELECT unp, contract_number, signing_value, currency, contractor_eik, + ROW_NUMBER() OVER ( + PARTITION BY unp, contract_number ORDER BY src_rank, cand_source DESC, ord DESC + ) AS rn + FROM ( + SELECT c.unp, c.contract_number, c.signing_value, c.currency, + TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) + AS contractor_eik, + 0 AS src_rank, c.source AS cand_source, c.id AS ord + FROM raw_contracts c + WHERE c.contract_number IS NOT NULL + AND c.signing_value IS NOT NULL AND c.signing_value > 0 + AND c.unp IN (SELECT unp FROM window_unps) + UNION ALL + SELECT substr(c.tender_id, 3) AS unp, c.contract_number, c.signing_value, c.currency, + b.eik_normalized AS contractor_eik, + 1 AS src_rank, '' AS cand_source, '' AS ord + FROM contracts c + LEFT JOIN bidders b ON b.id = c.bidder_id + WHERE c.contract_number IS NOT NULL + AND c.signing_value IS NOT NULL AND c.signing_value > 0 + AND c.tender_id IN (SELECT 't:' || unp FROM window_unps) + ) + ) WHERE rn = 1 +), +-- EOP annexes on the affected procedures that match NO REAL contract by (unp, contract_number) — the +-- namespace-mismatch group. Value-less members (admin/term steps mid-chain) are included so they can inherit +-- the chain's target (review nikimilenkov MEDIUM 2). The NOT EXISTS is over all_contract_numbers (every real +-- contract number on the procedure, value-agnostic), so an annex that DOES match a corpus contract directly is +-- correctly excluded — it links by number, not value — even when that target has signing_value <= 0, matching +-- the full path exactly (review todorkolev). +grp AS ( + SELECT a.id AS amendment_id, a.unp, a.contract_number AS annex_cnum, + a.value_before, a.currency, + TRIM(CASE WHEN a.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(a.contractor_eik, 5) ELSE a.contractor_eik END) + AS contractor_eik + FROM raw_amendments a + WHERE a.source LIKE 'eop:%' + AND a.unp IS NOT NULL AND a.contract_number IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM all_contract_numbers c + WHERE c.unp = a.unp AND c.contract_number = a.contract_number + ) +), +-- Exact, currency- and contractor-matched value anchor (< 0.5 стотинка). Currency must be EXPLICIT on both +-- sides (no blank-vs-blank agreement via a default). The EIK guard is null-tolerant and refuses a value +-- collision onto a different contractor's contract for free (review nikimilenkov LOW 1 / MEDIUM 5). +vmatch AS ( + SELECT g.amendment_id, g.unp, g.annex_cnum, c.contract_number AS resolved_cnum, + COUNT(*) OVER (PARTITION BY g.amendment_id) AS n_match + FROM grp g + JOIN contract_candidates c + ON c.unp = g.unp + AND ABS(c.signing_value - g.value_before) < 0.005 + AND NULLIF(c.currency, '') IS NOT NULL + AND NULLIF(g.currency, '') IS NOT NULL + AND c.currency = g.currency + AND (g.contractor_eik IS NULL OR c.contractor_eik IS NULL OR g.contractor_eik = c.contractor_eik) + WHERE g.value_before IS NOT NULL AND g.value_before > 0 +), +-- A member's OWN unique (n_match = 1) exact match — trustworthy on its own, so it always applies and is NOT +-- voided when annex-number siblings point elsewhere (a lot-base number shared across contracts; review +-- nikimilenkov MEDIUM 1). Propagation is withheld on disagreement, never the direct hits. +direct AS ( + SELECT amendment_id, unp, annex_cnum, resolved_cnum FROM vmatch WHERE n_match = 1 +), +-- Propagate one AGREED target across a (unp, annex-number) chain to value-less members, only when the direct +-- members agree on a single contract (review MEDIUM 2). +group_target AS ( + SELECT unp, annex_cnum, MIN(resolved_cnum) AS resolved_cnum + FROM direct GROUP BY unp, annex_cnum HAVING COUNT(DISTINCT resolved_cnum) = 1 +) +-- Link to the own unique match, else the agreed group target — UNLESS the member is itself value-ambiguous +-- (n_match >= 2), which never links directly or by inheritance (review todorkolev #2). +SELECT g.amendment_id, + COALESCE( + (SELECT d.resolved_cnum FROM direct d WHERE d.amendment_id = g.amendment_id), + (SELECT gt.resolved_cnum FROM group_target gt WHERE gt.unp = g.unp AND gt.annex_cnum = g.annex_cnum) + ) AS resolved_cnum +FROM grp g +WHERE NOT EXISTS ( + SELECT 1 FROM vmatch v WHERE v.amendment_id = g.amendment_id AND v.n_match >= 2 +); + +CREATE INDEX idx_refresh_amendment_contract_resolve_id + ON refresh_amendment_contract_resolve(amendment_id); + +-- Rewrite in place, preserving provenance: keep the annex-side number in contract_number_raw and stamp +-- link_method so value-linked rows stay enumerable through promotion into served `amendments`. The raw number +-- also keeps the amendment natural_key stable (see the promotion below and derive/promote on the full path), +-- so a resolved row never collides with a native annex sharing document_number on the target (MEDIUM 3). +UPDATE raw_amendments +SET + contract_number_raw = contract_number, + link_method = 'value_anchor', + contract_number = ( + SELECT r.resolved_cnum FROM refresh_amendment_contract_resolve r WHERE r.amendment_id = raw_amendments.id + ) +WHERE id IN (SELECT amendment_id FROM refresh_amendment_contract_resolve WHERE resolved_cnum IS NOT NULL); + +DROP TABLE IF EXISTS refresh_amendment_contract_resolve; + -- #286: prefer the EOP annex — drop OCDS twins so annex_count and the served timeline aren't doubled. -- The full path (derive-amendments.sql) only checks raw_amendments because it re-stages the whole corpus -- every run. The slice path must ALSO consult the cumulative served `amendments`: an EOP annex promoted by @@ -90,7 +240,7 @@ WHERE source LIKE 'ocds:%' CREATE TABLE refresh_amendment_winners AS WITH keyed AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), NULLIF(seq_no, ''), 'content:' || COALESCE(published_at, '') || ':' || @@ -1716,13 +1866,15 @@ WHERE source LIKE 'ocds:%' ); INSERT OR REPLACE INTO amendments ( - id, natural_key, contract_number, unp, value_before, value_after, value_delta, currency, - published_at, document_number, description, source, value_restated, value_treatment, value_suspect + id, natural_key, contract_number, contract_number_raw, link_method, unp, + value_before, value_after, value_delta, currency, + published_at, document_number, description, source, + value_restated, value_treatment, value_suspect ) WITH keyed AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), @@ -1747,6 +1899,8 @@ SELECT natural_key, natural_key, contract_number, + contract_number_raw, -- #306 provenance: the annex-side number before the value resolver rewrote it + link_method, -- #306 provenance: 'value_anchor' for value-linked rows, else NULL unp, value_before, -- #305 Tier-2: serve the effective (text-corrected) after and a self-consistent delta; the current_value diff --git a/scripts/resolve-amendment-contracts.sql b/scripts/resolve-amendment-contracts.sql new file mode 100644 index 00000000..3498505d --- /dev/null +++ b/scripts/resolve-amendment-contracts.sql @@ -0,0 +1,150 @@ +-- Sigma — #306: link EOP annexes whose annex-side number is in a different namespace than the contract +-- number. The annex carries an internal number (e.g. 148846) while the contract on the same procedure +-- carries the buyer's filing number (e.g. Д-226), so the (unp, contract_number) join drops ~7% of EOP +-- annexes out of every annex→contract→company/authority rollup. String normalisation recovers almost none +-- of them (measured), because the two numbers are genuinely unrelated identifiers. Resolve by VALUE +-- instead: an annex's value_before is the contract's value at amendment time, so it equals the target +-- contract's signing_value. Link only when value_before matches EXACTLY (< 0.5 стотинка), currency- and +-- contractor-matched, uniquely one contract on the procedure — measured 99.99% precision on the already- +-- linked corpus (9348/9349). Rows that match 2+ contracts (value-ambiguous) or none (target not yet +-- ingested — the #249 class) are LEFT unlinked: an honest gap beats a wrong contract on a transparency site. +-- +-- FULL-PATH FORM (review nikimilenkov HIGH 1 + todorkolev #3). This file runs from runFullDerive / +-- runWorkBackfill in scripts/import.mjs, BEFORE derive-amendments.sql, with candidates drawn from the whole +-- re-staged raw_contracts. It is deliberately NOT run on the slice path (runSliceDerive): the slice's +-- raw_contracts holds only the current window, so "unique on the procedure" would mean "unique in the window", +-- not in the corpus — a corpus-ambiguous annex would look unique in a narrow window and mislink. The daily/ +-- slice + Worker path instead runs a corpus-safe equivalent INSIDE scripts/refresh-slice.sql (search "#306: +-- slice-safe value-anchor resolver"), whose candidates come from the served `contracts` table plus this +-- window's raw_contracts, so uniqueness is asked corpus-wide and the measured precision carries. Both forms +-- share the same value/currency/EIK anchor, chain rules, and provenance stamping; see +-- docs/implementation-plans/306-amendment-contract-namespace-link.md §4. +-- +-- ORDER (review todorkolev #1, the blocker): this runs BEFORE the #286 prefer-EOP dedup DELETE in +-- derive-amendments.sql. Rewriting an EOP annex onto a contract that already kept an OCDS twin would +-- resurrect that twin (annex_count = 2 on a one-annex contract) and trip the amendment-twin-dedup integrity +-- gate (#303), failing the whole derive. Running first — and above the #286 diagnostics, so their dropped/ +-- excess counts stay honest bounds on what the dedup actually removes — keeps the dedup the sole twin guard. +-- The resolver reads only source LIKE 'eop:%' rows + raw_contracts, so it has no dependency on the OCDS +-- bridge and is safe to run first. + +-- Candidate contracts, deduped to one row per LOGICAL contract. raw_contracts is CUMULATIVE — the EOP daily +-- open-data buckets repeat the same contract across consecutive days, and the collapse to one row per +-- (unp, contract_number) happens later in normalize-raw.sql, NOT in staging (review nikimilenkov HIGH 2). +-- Without this dedup, COUNT(*) OVER below would count staging ROWS, not contracts: a contract present in N +-- daily buckets would read as n_match = N and fail-close every real link, and a superseded row could match +-- an annex to a stale value. Mirror normalize-raw's rule (latest source-day, then highest id, wins). +DROP TABLE IF EXISTS amendment_contract_resolve; +CREATE TABLE amendment_contract_resolve AS +WITH contract_candidates AS ( + SELECT unp, contract_number, signing_value, currency, contractor_eik + FROM ( + SELECT c.unp, c.contract_number, c.signing_value, c.currency, c.contractor_eik, + ROW_NUMBER() OVER ( + PARTITION BY c.unp, c.contract_number + ORDER BY c.source DESC, c.id DESC + ) AS rn + FROM raw_contracts c + WHERE c.contract_number IS NOT NULL + AND c.signing_value IS NOT NULL AND c.signing_value > 0 + ) + WHERE rn = 1 +), +-- Every EOP annex whose (unp, contract_number) matches no contract — the namespace-mismatch group. Includes +-- value-less members (value_before NULL/≤0): an admin/term annex mid-chain carries no signing value of its +-- own but must still inherit the chain's target (review nikimilenkov MEDIUM 2), else the chain breaks and +-- current_value stops early. Grouped by the shared annex-side number (unp, annex_cnum) = one contract's chain. +grp AS ( + SELECT a.id AS amendment_id, a.unp, a.contract_number AS annex_cnum, + a.value_before, a.currency, a.contractor_eik + FROM raw_amendments a + WHERE a.source LIKE 'eop:%' + AND a.unp IS NOT NULL AND a.contract_number IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM raw_contracts c + WHERE c.unp = a.unp AND c.contract_number = a.contract_number + ) +), +-- Exact, currency- and contractor-matched value anchor. Only members that carry a usable value_before +-- participate in matching. The currency guard requires an EXPLICIT currency on both sides (review +-- nikimilenkov LOW 1): a blank-vs-blank pair must not silently agree via a 'BGN' default under so tight a +-- gate. The EIK guard (review nikimilenkov MEDIUM 5) is null-tolerant — the annex already carries the +-- contractor, so a value collision onto a DIFFERENT contractor's contract is refused for free. +vmatch AS ( + SELECT g.amendment_id, g.unp, g.annex_cnum, c.contract_number AS resolved_cnum, + COUNT(*) OVER (PARTITION BY g.amendment_id) AS n_match + FROM grp g + JOIN contract_candidates c + ON c.unp = g.unp + AND ABS(c.signing_value - g.value_before) < 0.005 + AND NULLIF(c.currency, '') IS NOT NULL + AND NULLIF(g.currency, '') IS NOT NULL + AND c.currency = g.currency + AND (g.contractor_eik IS NULL OR c.contractor_eik IS NULL OR g.contractor_eik = c.contractor_eik) + WHERE g.value_before IS NOT NULL AND g.value_before > 0 +), +-- `direct` — each member's OWN unique (n_match = 1) exact value match. This is trustworthy on its own +-- (exact cent + unique on the whole procedure = the measured 99.99% precision), so a direct hit always +-- applies. Crucially it is NOT voided when its annex-number siblings point elsewhere: an annex-side number +-- can be a LOT-BASE shared across contracts (real corpus: `20РП-У50А015` → …-Л01 @ 22569.98 AND …-Л03 @ +-- 28557.50, each annex exactly-uniquely matching its own lot). The one disagreement group in the whole live +-- corpus is exactly this benign multi-lot case — refusing it (an earlier revision, review nikimilenkov +-- MEDIUM 1) dropped 2 confirmed-correct links and prevented zero wrong ones, so disagreement withholds +-- PROPAGATION only, never the direct hits themselves. +direct AS ( + SELECT amendment_id, unp, annex_cnum, resolved_cnum FROM vmatch WHERE n_match = 1 +), +-- `group_target` — propagate ONE agreed target across a `(unp, annex-number)` chain to members that have no +-- own unique match (value-less admin steps, or later steps whose cumulative value matches no signing_value), +-- but only when the direct members AGREE (a lot-base spread disagrees → no propagation, and the direct hits +-- still stand). So later chain steps link and `current_value` reflects the last step (review MEDIUM 2). +group_target AS ( + SELECT unp, annex_cnum, MIN(resolved_cnum) AS resolved_cnum + FROM direct GROUP BY unp, annex_cnum HAVING COUNT(DISTINCT resolved_cnum) = 1 +) +-- A member links to its OWN unique match if it has one; else to the agreed group target — UNLESS the member +-- is itself value-ambiguous (n_match >= 2), which carries its own contradicting evidence and never links, +-- directly or by inheritance (review todorkolev #2). Members with neither a direct hit nor an agreed group +-- target keep resolved_cnum NULL and stay unlinked. +SELECT g.amendment_id, + COALESCE( + (SELECT d.resolved_cnum FROM direct d WHERE d.amendment_id = g.amendment_id), + (SELECT gt.resolved_cnum FROM group_target gt WHERE gt.unp = g.unp AND gt.annex_cnum = g.annex_cnum) + ) AS resolved_cnum +FROM grp g +WHERE NOT EXISTS ( + SELECT 1 FROM vmatch v WHERE v.amendment_id = g.amendment_id AND v.n_match >= 2 +); + +-- The rewrite UPDATE below correlates raw_amendments.id to amendment_contract_resolve.amendment_id once per +-- unlinked row; index the resolve table so that is a lookup, not a scan (review nikimilenkov LOW 4). +CREATE INDEX IF NOT EXISTS idx_amendment_contract_resolve_id + ON amendment_contract_resolve(amendment_id); + +-- Rewrite contract_number in place — exactly like the #286 УНП bridge — and PRESERVE PROVENANCE (review +-- nikimilenkov MEDIUM 4): keep the original annex-side number in contract_number_raw and stamp link_method +-- so the value-linked rows stay enumerable in staging, through promote, and in the served `amendments` +-- table. contract_number_raw also keeps the annex number in the amendment natural_key (see derive- +-- amendments.sql / promote-amendments.sql), so a resolved row never collides with a native annex that +-- happens to share document_number on the target contract (review nikimilenkov MEDIUM 3). +UPDATE raw_amendments +SET + contract_number_raw = contract_number, + link_method = 'value_anchor', + contract_number = ( + SELECT r.resolved_cnum FROM amendment_contract_resolve r WHERE r.amendment_id = raw_amendments.id + ) +WHERE id IN (SELECT amendment_id FROM amendment_contract_resolve WHERE resolved_cnum IS NOT NULL); + +-- #306 diagnostic (printed by wrangler): annexes linked by the value anchor, and those still unlinked. The +-- two predicates are complementary — both count over the SAME namespace-mismatch population (review +-- nikimilenkov LOW 3) — so linked + still_unlinked = the original mismatch count on a healthy run. +SELECT + (SELECT COUNT(*) FROM raw_amendments WHERE link_method = 'value_anchor') AS annexes_value_linked, + (SELECT COUNT(*) FROM raw_amendments a + WHERE a.source LIKE 'eop:%' AND a.unp IS NOT NULL AND a.contract_number IS NOT NULL + AND a.link_method IS NULL + AND NOT EXISTS (SELECT 1 FROM raw_contracts c + WHERE c.unp = a.unp AND c.contract_number = a.contract_number)) AS eop_annexes_still_unlinked; + +DROP TABLE IF EXISTS amendment_contract_resolve; diff --git a/scripts/work-staging-schema.sql b/scripts/work-staging-schema.sql index 6c403e9c..0e9a581d 100644 --- a/scripts/work-staging-schema.sql +++ b/scripts/work-staging-schema.sql @@ -159,7 +159,9 @@ CREATE TABLE raw_amendments ( fetched_at TEXT NOT NULL, seq_no TEXT, document_number TEXT, - contract_number TEXT, -- ← link to raw_contracts + contract_number TEXT, -- ← link to raw_contracts (rewritten in place by the #306 value resolver) + contract_number_raw TEXT, -- #306: the annex-side number before the value resolver rewrote it (provenance; NULL = never rewritten) + link_method TEXT, -- #306: 'value_anchor' when the resolver linked this row by value; NULL = matched by number / unlinked contract_date TEXT, published_at TEXT, -- amendment publication date (ordering key) unp TEXT, -- ← link to raw_contracts From 0a97059bf290c0629466f00e964e9d4456694180 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 14 Aug 2026 12:34:30 +0000 Subject: [PATCH 66/69] =?UTF-8?q?fix(db,ci,docs):=20=D0=BF=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BE=D0=BC=D0=B5=D1=80=D0=B8=D1=80=D0=B0=D0=B9=20=D0=BC?= =?UTF-8?q?=D0=B8=D0=B3=D1=80=D0=B0=D1=86=D0=B8=D0=B8=D1=82=D0=B5=20=D0=BD?= =?UTF-8?q?=D0=B0=200009/0010=20=D0=B8=20=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=20=D0=BA=D0=BE=D0=BC=D0=B5=D0=BD=D1=82=D0=B0=D1=80=D0=B0?= =?UTF-8?q?=20=D0=B7=D0=B0=200010?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Преномериране: #307 взе 0006/0007, #308 взе 0008, тъй че тези две се местят на 0009 и 0010. Обновени са всички препратки - тестове, deploy.yml, related-persons-data.yml, scripts/cacbg/load.mjs, docs/deploy.md. Коментарът над прилагането на 0010 в deploy.yml беше останал от предишния замисъл и твърдеше три неверни неща: че 0003 обявявала CHECK-овете (тя е върната непокътната и не обявява нищо), че се слагало control_hash NOT NULL (остава NULL-ируема нарочно - регистърът я пропуска за част от декларациите, а индексът по естествен ключ ги сгъва с COALESCE) и че миграцията е „rebuild-based" (вече е тригерна; заглавието ѝ обяснява защо пресъздаването е опасно - оголва външните ключове и обира доказателствените печати). Последното беше най-неприятно: обещаваше на следващия четец точно операцията, която самата миграция отхвърля, а „0003 declares them now" щеше да го прати обратно да добавя CHECK-ове в приложена миграция. docs/deploy.md вече изброява и стъпките, които сондират таблицата и добавят само липсващите колони, за да не изглежда, че --file е единственият механизъм. db 424 зелени, typecheck и prettier чисти, scripts/tr 134/134. --- .github/workflows/deploy.yml | 22 ++++++++++++------- .github/workflows/related-persons-data.yml | 4 ++-- docs/deploy.md | 8 ++++--- ...ce.sql => 0009_interest_link_evidence.sql} | 0 ...l => 0010_publishing_gate_constraints.sql} | 0 .../db/src/contractor-identity-sql.test.ts | 2 +- .../etl-entity-canonicalization-sql.test.ts | 2 +- packages/db/src/integrity-checks.test.ts | 2 +- packages/db/src/migrations.test.ts | 4 ++-- packages/db/src/refresh-slice.test.ts | 2 +- packages/db/src/related-persons-sql.test.ts | 2 +- packages/db/src/search-sql.test.ts | 2 +- packages/db/src/seed-surface.test.ts | 2 +- .../db/src/value-flag-annex-step-sql.test.ts | 2 +- .../db/src/value-flag-stotinki-sql.test.ts | 2 +- packages/ingest/src/refresh-officials.test.ts | 2 +- scripts/cacbg/load.mjs | 2 +- scripts/ship-e2e.test.mjs | 2 +- scripts/ship-reseed.test.mjs | 2 +- 19 files changed, 36 insertions(+), 28 deletions(-) rename packages/db/migrations/{0006_interest_link_evidence.sql => 0009_interest_link_evidence.sql} (100%) rename packages/db/migrations/{0007_publishing_gate_constraints.sql => 0010_publishing_gate_constraints.sql} (100%) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 86567ef8..30ffb5bb 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -145,20 +145,26 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0003_related_persons_foundation.sql - # 0006 attaches the Trade Register evidence seal (#279, ADR-0033). Both migrations are + # 0009 attaches the Trade Register evidence seal (#279, ADR-0033). Both migrations are # applied by name here because wrangler's migration ledger is empty on this D1 (the base # schema was created out-of-band), so `d1 migrations apply` would collide on 0000. pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ --config wrangler.deploy.jsonc --remote --yes \ - --file ../../packages/db/migrations/0006_interest_link_evidence.sql - # 0007 retrofits the publishing-gate CHECK constraints and control_hash NOT NULL onto an - # ALREADY-PROVISIONED D1 (#279 §2). 0003 declares them now, but `CREATE TABLE IF NOT EXISTS` - # never revisits an existing table and the ship step wipes ROWS, not definitions — so without - # this a live database keeps the unconstrained shape no matter how often the data reloads. - # Rebuild-based and therefore idempotent: re-applying copies an already-constrained table. + --file ../../packages/db/migrations/0009_interest_link_evidence.sql + # 0010 owns the publishing-gate enforcement for EVERY database (#279 §2) — fresh and already + # deployed alike. It is deliberately NOT declared in 0003: that migration is already applied + # everywhere, `CREATE TABLE IF NOT EXISTS` never revisits an existing table, and the ship step + # wipes ROWS, not definitions — so an in-place CHECK would exist only on databases built after + # the edit, and be absent on exactly the database that serves the site. Enforced with BEFORE + # INSERT/UPDATE triggers, NOT a table rebuild: the create-copy-drop-rename route would expose + # the foreign keys and strip every evidence seal, emptying the public surface until the next + # monthly run (see the migration header). `control_hash` stays NULLABLE by design — the register + # omits it on some declarations; the natural-key index folds those with COALESCE instead. + # Idempotent: every statement is `IF NOT EXISTS` over a converging definition, so re-applying it + # on each deploy is a no-op. Verified by three consecutive applications, gate still enforcing. pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ --config wrangler.deploy.jsonc --remote --yes \ - --file ../../packages/db/migrations/0007_publishing_gate_constraints.sql + --file ../../packages/db/migrations/0010_publishing_gate_constraints.sql # The base schema was created out-of-band with `d1 execute --file`, so wrangler's migration # ledger is empty and `d1 migrations apply` would collide on 0000. Probe the actual table # instead. SQLite has no `ADD COLUMN IF NOT EXISTS`; a completion-marker table is created only diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index 40dfb5ac..2beb2706 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -311,7 +311,7 @@ jobs: # schema was created out-of-band), so `d1 migrations apply` would collide on 0000. pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote --yes \ - --file ../../packages/db/migrations/0006_interest_link_evidence.sql + --file ../../packages/db/migrations/0009_interest_link_evidence.sql # 0007 retrofits the publishing-gate CHECK constraints and control_hash NOT NULL onto an # ALREADY-PROVISIONED D1 (#279 §2). 0003 declares them now, but `CREATE TABLE IF NOT EXISTS` # never revisits an existing table and the ship step wipes ROWS, not definitions — so without @@ -319,7 +319,7 @@ jobs: # Rebuild-based and therefore idempotent: re-applying copies an already-constrained table. pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote --yes \ - --file ../../packages/db/migrations/0007_publishing_gate_constraints.sql + --file ../../packages/db/migrations/0010_publishing_gate_constraints.sql - name: Ship свързани-лица tables → D1 # ship-related-persons.mjs shells out to `wrangler d1 execute` with cwd=apps/web. Two things it diff --git a/docs/deploy.md b/docs/deploy.md index eb25a430..e296eda8 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -174,11 +174,13 @@ domain таблиците и преизчислява rollup-ите + FTS. > данни**. > > **Уточнение (беше неточно):** тук пишеше и „деплоите не мигрират". Вече не е вярно — -> [deploy.yml](../.github/workflows/deploy.yml) прилага изрично `0003`, `0006` и `0007` с +> [deploy.yml](../.github/workflows/deploy.yml) прилага изрично `0003`, `0009` и `0010` с > `d1 execute --file` при всеки деплой. Причината е, че ledger-ът на wrangler на тези бази е празен > (базовата схема е създадена out-of-band), тъй че `d1 migrations apply` би се сблъскал с `0000`. И трите -> са безопасни за повторно прилагане: `CREATE … IF NOT EXISTS` и, при `0007`, тригери и индекси, които -> се пресъздават до същото състояние. Останалите схемни промени наистина си остават out-of-band. +> са безопасни за повторно прилагане: `CREATE … IF NOT EXISTS` и, при `0010`, тригери и индекси, които +> се пресъздават до същото състояние. Отделно от тях има и стъпки, които сондират таблицата и добавят +> само липсващите колони — за `0002` и за колоните на анексите. Останалите схемни промени наистина си +> остават out-of-band. ## 3. Конфигуриране на GitHub Environments diff --git a/packages/db/migrations/0006_interest_link_evidence.sql b/packages/db/migrations/0009_interest_link_evidence.sql similarity index 100% rename from packages/db/migrations/0006_interest_link_evidence.sql rename to packages/db/migrations/0009_interest_link_evidence.sql diff --git a/packages/db/migrations/0007_publishing_gate_constraints.sql b/packages/db/migrations/0010_publishing_gate_constraints.sql similarity index 100% rename from packages/db/migrations/0007_publishing_gate_constraints.sql rename to packages/db/migrations/0010_publishing_gate_constraints.sql diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index ea4de1ce..ce1961ac 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -18,7 +18,7 @@ const migration3 = readFileSync( ); // …and 0006, which those same blocks now join for the Trade Register evidence gate (#279, ADR-0033). const migration6 = readFileSync( - resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'), + resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'), 'utf8', ); const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index 9ffdea5b..da3ed90f 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -12,7 +12,7 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ // refresh-slice.sql's officials block reads interest_links (0003); build it so the script doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). -const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index 645af634..415fe751 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -30,7 +30,7 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ // precompute.sql's officials block reads interest_links (0003); build it so precompute doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). -const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const precomputePath = resolve(root, 'scripts/precompute.sql'); function sqlite(dbPath: string, sql: string): void { diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index 25e5ef12..a879754a 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -11,8 +11,8 @@ const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); const migration2 = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); -const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); -const migration7 = resolve(root, 'packages/db/migrations/0007_publishing_gate_constraints.sql'); +const migration6 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration7 = resolve(root, 'packages/db/migrations/0010_publishing_gate_constraints.sql'); const backfill = resolve(root, 'scripts/backfill-current-value-currency.sql'); const precompute = resolve(root, 'scripts/precompute.sql'); diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 9c76fb83..232307c9 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -14,7 +14,7 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ // refresh-slice.sql / precompute.sql officials block reads interest_links (0003) — build it in every chain. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). -const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const deriveAmendmentsPath = resolve(root, 'scripts/derive-amendments.sql'); diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index f389d525..66cee0eb 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -26,7 +26,7 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006: SURFACED_OWNERSHIP now requires a Trade Register evidence seal (#279, ADR-0033). -const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }).trim(); diff --git a/packages/db/src/search-sql.test.ts b/packages/db/src/search-sql.test.ts index f371cd1d..4b412d21 100644 --- a/packages/db/src/search-sql.test.ts +++ b/packages/db/src/search-sql.test.ts @@ -16,7 +16,7 @@ import { SEARCH_HITS_SQL, SEARCH_HITS_SQL_NO_CONFLICT } from './queries/search'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); -const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function readScript(dbPath: string, path: string): void { execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${path}\n`, stdio: 'pipe' }); diff --git a/packages/db/src/seed-surface.test.ts b/packages/db/src/seed-surface.test.ts index e9484d25..78a70aaa 100644 --- a/packages/db/src/seed-surface.test.ts +++ b/packages/db/src/seed-surface.test.ts @@ -21,7 +21,7 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migrations = [ 'packages/db/migrations/0000_init.sql', 'packages/db/migrations/0003_related_persons_foundation.sql', - 'packages/db/migrations/0006_interest_link_evidence.sql', + 'packages/db/migrations/0009_interest_link_evidence.sql', ].map((m) => resolve(root, m)); const seed = resolve(root, 'scripts/seed.sql'); diff --git a/packages/db/src/value-flag-annex-step-sql.test.ts b/packages/db/src/value-flag-annex-step-sql.test.ts index c2afb684..c5cbb2a1 100644 --- a/packages/db/src/value-flag-annex-step-sql.test.ts +++ b/packages/db/src/value-flag-annex-step-sql.test.ts @@ -29,7 +29,7 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script // only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it // ever reaches the value_flag CASE these tests are about. -const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const promotePath = resolve(root, 'scripts/promote-amendments.sql'); diff --git a/packages/db/src/value-flag-stotinki-sql.test.ts b/packages/db/src/value-flag-stotinki-sql.test.ts index eb41d860..615ffc3a 100644 --- a/packages/db/src/value-flag-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-stotinki-sql.test.ts @@ -23,7 +23,7 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script // only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it // ever reaches the value_flag CASE these tests are about. -const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], diff --git a/packages/ingest/src/refresh-officials.test.ts b/packages/ingest/src/refresh-officials.test.ts index 6f3eb4c6..2c0ebb8f 100644 --- a/packages/ingest/src/refresh-officials.test.ts +++ b/packages/ingest/src/refresh-officials.test.ts @@ -22,7 +22,7 @@ const migrations = [ '0003_related_persons_foundation.sql', // The officials batch now joins the Trade Register evidence seal (#279, ADR-0033) — without 0006 the // real refresh-slice.sql this test executes cannot parse. - '0006_interest_link_evidence.sql', + '0009_interest_link_evidence.sql', ].map((f) => resolve(root, 'packages/db/migrations', f)); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 18a806f1..ca7c030c 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -48,7 +48,7 @@ const MIGRATION = path.join(ROOT, 'packages/db/migrations/0003_related_persons_f // missing from the work DB would make every evidence write fail at ship time instead of at load. const MIGRATION_EVIDENCE = path.join( ROOT, - 'packages/db/migrations/0006_interest_link_evidence.sql', + 'packages/db/migrations/0009_interest_link_evidence.sql', ); const REPORT = path.join(STAGING, 'findings.md'); // Bumped for #279: classify-2 (КДА added to the joint-stock bar) + tr-1 (identity now rests on a diff --git a/scripts/ship-e2e.test.mjs b/scripts/ship-e2e.test.mjs index 60d6d67e..1d150914 100644 --- a/scripts/ship-e2e.test.mjs +++ b/scripts/ship-e2e.test.mjs @@ -31,7 +31,7 @@ const MIG = resolve(ROOT, 'packages/db/migrations/0003_related_persons_foundatio // 0006 too: the evidence seal is one of the shipped tables (#279, ADR-0033), so it appears in the ship's // TABLES and WIPE_ORDER. A target built from 0003 alone makes the generated wipe abort on „no such table" // before a single row moves — the schema under test has to be the schema the ship writes. -const MIG_EVIDENCE = resolve(ROOT, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const MIG_EVIDENCE = resolve(ROOT, 'packages/db/migrations/0009_interest_link_evidence.sql'); const D1_NAME = 'sigma-test-local'; // Derived from BOTH production constants, and the run below does NOT override either: one row past diff --git a/scripts/ship-reseed.test.mjs b/scripts/ship-reseed.test.mjs index 9c5b6617..44020c5c 100644 --- a/scripts/ship-reseed.test.mjs +++ b/scripts/ship-reseed.test.mjs @@ -17,7 +17,7 @@ const HERE = dirname(fileURLToPath(import.meta.url)); const MIG = resolve(HERE, '..', 'packages/db/migrations/0003_related_persons_foundation.sql'); // 0006 adds interest_link_evidence, which references interest_links — so it is part of the FK graph // this test exists to police, and the wipe must clear it before its parent (#279, ADR-0033). -const MIG_EVIDENCE = resolve(HERE, '..', 'packages/db/migrations/0006_interest_link_evidence.sql'); +const MIG_EVIDENCE = resolve(HERE, '..', 'packages/db/migrations/0009_interest_link_evidence.sql'); // Stub only the two 0000 tables 0002's FKs reference (bidders, authorities) — a PK column is all a FK needs — // then apply the real 0002 and seed one row down every FK chain so a parent delete has live children. From 0d8be1e1a0d9553ea91ebfa4d5a232b9ab9795e2 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 14 Aug 2026 12:39:11 +0000 Subject: [PATCH 67/69] =?UTF-8?q?refactor(db):=20=D0=BF=D1=80=D0=B5=D0=B8?= =?UTF-8?q?=D0=BC=D0=B5=D0=BD=D1=83=D0=B2=D0=B0=D0=B9=20=D0=BF=D1=80=D0=BE?= =?UTF-8?q?=D0=BC=D0=B5=D0=BD=D0=BB=D0=B8=D0=B2=D0=B8=D1=82=D0=B5=20=D0=BD?= =?UTF-8?q?=D0=B0=20=D0=BC=D0=B8=D0=B3=D1=80=D0=B0=D1=86=D0=B8=D0=B8=D1=82?= =?UTF-8?q?=D0=B5=209/10=20=D0=B2=20=D1=82=D0=B5=D1=81=D1=82=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D1=82=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Преномерирането смени пътищата, но остави имената migration6Path/migration7Path, което е подвеждащо и се сблъсква с едноименните променливи, които #307 и #308 въведоха за 0006/0007/0008. --- .../db/src/contractor-identity-sql.test.ts | 4 ++-- .../etl-entity-canonicalization-sql.test.ts | 4 ++-- packages/db/src/integrity-checks.test.ts | 4 ++-- packages/db/src/migrations.test.ts | 24 +++++++++---------- packages/db/src/refresh-slice.test.ts | 16 ++++++------- packages/db/src/related-persons-sql.test.ts | 4 ++-- packages/db/src/search-sql.test.ts | 4 ++-- .../db/src/value-flag-annex-step-sql.test.ts | 4 ++-- .../db/src/value-flag-stotinki-sql.test.ts | 4 ++-- 9 files changed, 34 insertions(+), 34 deletions(-) diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index ce1961ac..b463831c 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -17,7 +17,7 @@ const migration3 = readFileSync( 'utf8', ); // …and 0006, which those same blocks now join for the Trade Register evidence gate (#279, ADR-0033). -const migration6 = readFileSync( +const migration9 = readFileSync( resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'), 'utf8', ); @@ -71,7 +71,7 @@ function build(path: 'normalize' | 'refresh'): DatabaseSync { const db = new DatabaseSync(':memory:'); db.exec(schema); db.exec(migration2); - db.exec(migration3 + migration6); + db.exec(migration3 + migration9); db.exec(staging); db.exec(seed); if (path === 'normalize') { diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index da3ed90f..4c895088 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -12,7 +12,7 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ // refresh-slice.sql's officials block reads interest_links (0003); build it so the script doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). -const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -42,7 +42,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index 415fe751..f6d9c7ff 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -30,7 +30,7 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ // precompute.sql's officials block reads interest_links (0003); build it so precompute doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). -const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const precomputePath = resolve(root, 'scripts/precompute.sql'); function sqlite(dbPath: string, sql: string): void { @@ -74,7 +74,7 @@ function freshDb(): string { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); sqlite(dbPath, CLEAN_FIXTURE); return dbPath; } diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index a879754a..a37050b2 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -11,8 +11,8 @@ const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); const migration2 = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); -const migration6 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); -const migration7 = resolve(root, 'packages/db/migrations/0010_publishing_gate_constraints.sql'); +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration10 = resolve(root, 'packages/db/migrations/0010_publishing_gate_constraints.sql'); const backfill = resolve(root, 'scripts/backfill-current-value-currency.sql'); const precompute = resolve(root, 'scripts/precompute.sql'); @@ -133,7 +133,7 @@ describe('served migrations', () => { ); readScript(dbPath, migration2); readScript(dbPath, migration3); - readScript(dbPath, migration6); + readScript(dbPath, migration9); readScript(dbPath, backfill); readScript(dbPath, precompute); @@ -163,7 +163,7 @@ describe('served migrations', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-0006-')); const dbPath = resolve(dir, 'test.sqlite'); try { - for (const m of [migration0, migration1, migration2, migration3, migration6]) + for (const m of [migration0, migration1, migration2, migration3, migration9]) readScript(dbPath, m); expect( @@ -188,7 +188,7 @@ describe('served migrations', () => { (link_key, evidence_kind, matched_fact, lookup_date, rules_version, live_status) VALUES ('k', 'document', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live');`, ); - readScript(dbPath, migration6); // idempotent re-apply + readScript(dbPath, migration9); // idempotent re-apply expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_link_evidence;').trim()).toBe('1'); // The FK is real: a seal for a link that does not exist is rejected. D1 enforces foreign keys, @@ -215,7 +215,7 @@ describe('served migrations', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-')); const dbPath = resolve(dir, 'test.sqlite'); try { - for (const m of [migration0, migration3, migration6, migration7]) readScript(dbPath, m); + for (const m of [migration0, migration3, migration9, migration10]) readScript(dbPath, m); const seedLink = (status: string) => sqlite( @@ -315,11 +315,11 @@ describe('served migrations', () => { const legacy = resolve(dir, 'legacy.sqlite'); try { // (1) fresh: the migration chain exactly as a new environment applies it. - for (const m of [migration0, migration3, migration6, migration7]) readScript(fresh, m); + for (const m of [migration0, migration3, migration9, migration10]) readScript(fresh, m); // (2) legacy: 0003 + 0006 as they were BEFORE this PR, then 0007 retrofits. `interest_links` is // recreated without constraints because that is the shape every already-deployed database holds. - for (const m of [migration0, migration3, migration6]) readScript(legacy, m); + for (const m of [migration0, migration3, migration9]) readScript(legacy, m); sqlite( legacy, `DROP TABLE interest_link_evidence; @@ -341,7 +341,7 @@ describe('served migrations', () => { lookup_date TEXT NOT NULL, rules_version TEXT NOT NULL, live_status TEXT NOT NULL, sealed_at TEXT NOT NULL DEFAULT (datetime('now')));`, ); - readScript(legacy, migration7); + readScript(legacy, migration10); const link = (db: string, id: string, status: string, cls = 'private_ownership') => sqlite( @@ -393,7 +393,7 @@ describe('served migrations', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-')); const dbPath = resolve(dir, 'test.sqlite'); try { - for (const m of [migration0, migration3, migration6]) readScript(dbPath, m); + for (const m of [migration0, migration3, migration9]) readScript(dbPath, m); // Simulate a database provisioned BEFORE today: 0003 now declares the constraints, so the legacy // shape has to be recreated explicitly. This is the state every already-deployed environment is in // — `CREATE TABLE IF NOT EXISTS` never revisited it and ship-related-persons only wipes rows. @@ -441,7 +441,7 @@ describe('served migrations', () => { 'private_ownership', 'published ');`, ); - readScript(dbPath, migration7); + readScript(dbPath, migration10); // The valid rows survive — nothing is rebuilt, so nothing can be lost… expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( 'd:keep', @@ -490,7 +490,7 @@ describe('served migrations', () => { // Migrations here are applied by a bare `d1 execute --file` with no applied-migrations tracking, // so a second application MUST be a no-op rather than an error or a data loss. - readScript(dbPath, migration7); + readScript(dbPath, migration10); expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('2'); expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( 'd:keep', diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 232307c9..f5d8dc25 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -14,7 +14,7 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ // refresh-slice.sql / precompute.sql officials block reads interest_links (0003) — build it in every chain. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). -const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const deriveAmendmentsPath = resolve(root, 'scripts/derive-amendments.sql'); @@ -187,7 +187,7 @@ function initWorkDb(dbPath: string): void { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); } @@ -579,7 +579,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -664,7 +664,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); // 0006 too: refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279), so the // script cannot parse against a DB that stops at 0003 — every site here applies both. - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); // An EOP procedure (tender + base contract) with УНП UNP-SLICE / tender.id TENDER-SLICE. An @@ -848,7 +848,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -900,7 +900,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -952,7 +952,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -1090,7 +1090,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index 66cee0eb..ddcf4287 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -26,7 +26,7 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006: SURFACED_OWNERSHIP now requires a Trade Register evidence seal (#279, ADR-0033). -const migration6 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }).trim(); @@ -143,7 +143,7 @@ describe('свързани-лица SQL (real SQLite)', () => { try { readScript(dbPath, migration0); readScript(dbPath, migration2); - readScript(dbPath, migration6); + readScript(dbPath, migration9); sqlite(dbPath, FIXTURE); return fn(dbPath); } finally { diff --git a/packages/db/src/search-sql.test.ts b/packages/db/src/search-sql.test.ts index 4b412d21..59f9a95c 100644 --- a/packages/db/src/search-sql.test.ts +++ b/packages/db/src/search-sql.test.ts @@ -16,7 +16,7 @@ import { SEARCH_HITS_SQL, SEARCH_HITS_SQL_NO_CONFLICT } from './queries/search'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); -const migration6 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function readScript(dbPath: string, path: string): void { execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${path}\n`, stdio: 'pipe' }); @@ -141,7 +141,7 @@ function withDb(fn: (dbPath: string) => void): void { try { readScript(dbPath, migration0); readScript(dbPath, migration2); - readScript(dbPath, migration6); + readScript(dbPath, migration9); exec(dbPath, FIXTURE); exec(dbPath, POPULATE_INDEX); fn(dbPath); diff --git a/packages/db/src/value-flag-annex-step-sql.test.ts b/packages/db/src/value-flag-annex-step-sql.test.ts index c5cbb2a1..1ffd29f3 100644 --- a/packages/db/src/value-flag-annex-step-sql.test.ts +++ b/packages/db/src/value-flag-annex-step-sql.test.ts @@ -29,7 +29,7 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script // only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it // ever reaches the value_flag CASE these tests are about. -const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const promotePath = resolve(root, 'scripts/promote-amendments.sql'); @@ -64,7 +64,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/value-flag-stotinki-sql.test.ts b/packages/db/src/value-flag-stotinki-sql.test.ts index 615ffc3a..3a722541 100644 --- a/packages/db/src/value-flag-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-stotinki-sql.test.ts @@ -23,7 +23,7 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script // only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it // ever reaches the value_flag CASE these tests are about. -const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -53,7 +53,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, stagingPath); run(dbPath); } finally { From 25d189ee3b2acf92d522dbd1cfa07b84f82f83e9 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 14 Aug 2026 13:44:46 +0000 Subject: [PATCH 68/69] =?UTF-8?q?fix(etl):=20=D1=85=D0=B2=D0=B0=D1=89?= =?UTF-8?q?=D0=B0=D0=B9=20=D1=81=D1=82=D0=BE=D1=82=D0=B8=D0=BD=D0=BA=D0=B8?= =?UTF-8?q?=20=D0=B3=D1=80=D0=B5=D1=88=D0=BA=D0=B0=D1=82=D0=B0=20=D0=B8=20?= =?UTF-8?q?=D0=BF=D0=BE=20=D0=BF=D1=80=D0=BE=D0=B3=D0=BD=D0=BE=D0=B7=D0=B0?= =?UTF-8?q?=D1=82=D0=B0=20=D0=BD=D0=B0=20=D0=BF=D0=BE=D0=B7=D0=B8=D1=86?= =?UTF-8?q?=D0=B8=D1=8F=D1=82=D0=B0=20(#304)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #298 хвана пропуснатия десетичен знак, като мери спрямо ПРОЦЕДУРНАТА прогноза. При многопозиционна поръчка обаче подписът на грешката е 100x спрямо прогнозата на САМАТА позиция, а съотношението към цялата процедура пада където го изведе делът на позицията. Договорът от доклада (00621-2020-0008) стои на 89,8x спрямо процедурата - под лентата 95-105x, тъй че минаваше отдолу. Добавено е второ рамо в същата лента: стойността е 95-105x собствената прогноза на реда И поне 10x процедурната. Второто условие носи тежест - при рамкови и единични цени прогнозата по позиция е ЕДИНИЧНА цена и цял call-off законно я надхвърля стократно; такъв call-off стои на ~1x процедурната прогноза и се пощадява. И двете прогнози минават под 1000 EUR, защото при такива размери 100x е шум, а не изпусната запетая. Премерено върху корпуса: 7 договора, 40,3 млн. EUR показвани, 1,75 млн. след поправката към процедурната прогноза. Осмият, който лентата хваща, вече е value_suspect по правилото за 200x - рамото се съгласява с предшественика си там, където се застъпват. Най-ясният случай: доставка на учебници при прогноза 2 938 EUR, показвана като 293 778 EUR. Логиката влиза в петте копия на флаговата CASE. Тестът кара истинските скриптове по двата derive пътя и покрива двата ръба на лентата, двата пода поотделно и ред без собствена прогноза. Мутации: 95→90, 105→110 и махането на който и да е от двата пода валят по два теста; махането на рамото само от normalize-raw.sql вали 6, само от refresh-slice.sql - 7. По прегледа на cefothe реконсилиационният пас се изравни с четирите INSERT места: собствената прогноза се вади от самия ред (NULL, когато няма такава) и се превръща по СВОЯТА си валута с датирано fx търсене, а не по валутата на договора. Второто има видима цена - прогноза в евро при договор в лева се делеше на 1,95583, падаше извън лентата и договорът се показваше с 51 129 EUR вместо с 500 000. Има тест точно за това. Липсата на fallback-а към процедурната прогноза е изчистване на смисъла, не промяна в поведението, и е записана като такава: мутация, която го връща, не вали нито един тест. docs/etl.md вече изброява и трите повода за value_suspect, включително двете форми на стотинки лентата, плюс annex_total_suspect от #305, който липсваше в подредения списък. По предимството между това рамо и annex_total_suspect: договори, палещи и двете - нула, но целият корпус още няма нито един annex_total_suspect, защото флагът се пълни при стейджване на анекса, а кронът пуска само прозореца. Въпросът остава непремерен до пълен derive; по конструкция value_suspect печели, защото е първи в CASE-а. Затваря #247. --- docs/etl.md | 26 +- .../src/value-flag-lot-stotinki-sql.test.ts | 324 ++++++++++++++++++ scripts/normalize-raw.sql | 58 +++- scripts/refresh-slice.sql | 114 +++++- 4 files changed, 512 insertions(+), 10 deletions(-) create mode 100644 packages/db/src/value-flag-lot-stotinki-sql.test.ts diff --git a/docs/etl.md b/docs/etl.md index b3fbf50b..4f34a507 100644 --- a/docs/etl.md +++ b/docs/etl.md @@ -253,8 +253,22 @@ Verdict-ът се определя от две прогнозни стойнос CASE-ът се оценява по ред; печели първото съвпадение: -1. **`value_suspect`** — записана стойност неправдоподобно висока: `eff > 2 000 000 000` **или** - (`procEst >= 1000` **и** `eff > 200 * procEst`). Редът се **поправя, не се хвърля**: `amount_eur` +1. **`value_suspect`** — записана стойност неправдоподобно висока. Три отделни повода, всеки от + които стига сам по себе си: + - `eff > 2 000 000 000` — извън всякакъв мащаб за български договор; + - `procEst >= 1000` **и** `eff > 200 * procEst` — общият праг за неправдоподобно надвишаване; + - **стотинки лента** (#247, #298): пропуснат десетичен знак вдига стойността точно ~100 пъти. + Хваща се на две места, защото грешката може да е спрямо коя да е от двете прогнози: + - спрямо **процедурната**: `procEst >= 1000` **и** `95 * procEst <= eff <= 105 * procEst`; + - спрямо **прогнозата по позиция**: `ownEst >= 1000` **и** `procEst >= 1000` **и** + `eff >= 10 * procEst` **и** `95 * ownEst <= eff <= 105 * ownEst`. + + Лентата е тясна нарочно — измерването върху корпуса показва изолирано струпване при ~100x и + нула договора между 105x и 200x. Второто рамо иска и `eff >= 10 * procEst`, защото при рамкови + и единични цени прогнозата по позиция е ЕДИНИЧНА цена и цял call-off законно я надхвърля + стократно; условието за процедурата отсява точно тези случаи. + + Редът се **поправя, не се хвърля**: `amount_eur` става `procEst` (собственият документиран бюджет на поръчката), а показваната нативна сума става нативната процедурна прогноза. Пада до NULL (изключва се) само когато процедурата няма прогноза, от която да се поправи. Guard-ът `procEst >= 1000` пази редовете, чиято *прогноза* е грешката @@ -270,9 +284,13 @@ CASE-ът се оценява по ред; печели първото съвп свършил `>= 5x` над подписаната (сбъркано число в анекс — само стъпката не стига, защото има вериги, при които по-късен анекс сваля грешката обратно под подписаната стойност). Договорът **се връща към подписаната стойност** за `amount_eur`; надутата текуща стойност се потиска. -4. **`review`** — надхвърляне в сива зона: `eff >= 10 * procEst`. Запазен, флагнат и **брои се по +4. **`annex_total_suspect`** (#305) — водещият анекс е удвоил договора в една стъпка (ЗОП чл. 116 + ограничава единично изменение до +50%), а текстът на основанието не дава сигнал, по който + стойността да се поправи. Договорът **се връща към подписаната стойност**, както при + `annex_suspect`. +5. **`review`** — надхвърляне в сива зона: `eff >= 10 * procEst`. Запазен, флагнат и **брои се по face value**. -5. **`ok`** — всичко останало, брои се по `eff`. +6. **`ok`** — всичко останало, брои се по `eff`. Водещият принцип е **поправка пред изключване**: само наистина невъзстановими редове напускат тоталите. Където записаната стойност е недвусмислен боклук, заместваме с най-добрия документиран diff --git a/packages/db/src/value-flag-lot-stotinki-sql.test.ts b/packages/db/src/value-flag-lot-stotinki-sql.test.ts new file mode 100644 index 00000000..388d3172 --- /dev/null +++ b/packages/db/src/value-flag-lot-stotinki-sql.test.ts @@ -0,0 +1,324 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +// Issue #247, second reported contract. The стотинки band added in #298 measures against the PROCEDURE +// estimate, so it only fires when the whole procedure is one lot. On a multi-lot procedure the dropped +// decimal point still lands at exactly 100x the LOT's own estimate, while the ratio to the procedure +// total lands wherever that lot's share puts it - 00621-2020-0008 sits at 89.8x and is served at +// 14,212,416 EUR instead of ~142,000 EUR. +// +// The own-row estimate CANNOT be used on its own: docs/etl.md is explicit that for framework and +// unit-price procedures (medicines, fuel) it is a UNIT price, and a whole call-off legitimately dwarfs +// it. So the arm is a conjunction - 95x..105x of the own estimate AND at least 10x the procedure +// estimate, the threshold that already means "implausibly large for this procedure". A unit-price +// call-off sits at ~1x the procedure estimate and is spared. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// The амендмент columns refresh-slice.sql/promote-amendments.sql now write into served `amendments` +// (#305 value_restated/value_treatment/value_suspect, #306 contract_number_raw/link_method). Without +// them sqlite3 aborts on the amendment promotion long before it reaches the value_flag CASE under test. +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const etlPaths = [ + ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], + ['refresh-slice', resolve(root, 'scripts/refresh-slice.sql')], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-lotband-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** The lot's own estimate: 195_583 BGN = exactly 100_000 EUR, so the multiples read directly. */ +const OWN_BGN = 195_583; +const AUTH_EIK = '000695114'; +const BIDDER_EIK = '831646048'; + +interface Case { + unp: string; + /** Contract value, in BGN. */ + valueBgn: number; + /** The lot's own estimate on the contract row, in BGN. `null` = the row carries none. */ + ownEstBgn: number | null; + /** The whole procedure's estimate, in BGN. */ + procEstBgn: number; + /** Currency the row's OWN estimate is denominated in (`procurement_currency`). Defaults to BGN. */ + ownEstCurrency?: string; +} + +function seed(dbPath: string, cases: Case[]): void { + const tenders = cases + .map( + (c) => + `('eop:tenders:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'public', ${c.procEstBgn}, 'BGN')`, + ) + .join(',\n'); + const contracts = cases + .map( + (c) => + `('eop:contracts:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'C-${c.unp}', '2026-06-01', ${c.valueBgn}, 'BGN', ${c.ownEstBgn ?? 'NULL'}, '${c.ownEstCurrency ?? 'BGN'}', '${BIDDER_EIK}', 'Тестов изпълнител')`, + ) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type, estimated_value, currency) + VALUES ${tenders}; + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, estimated_value, procurement_currency, + contractor_eik, contractor_name) + VALUES ${contracts};`, + ); +} + +/** + * An annex on the contract. With `afterBgn` omitted it is a no-op (before = after), which leaves + * `current_value` on the signing value and `eff_eur` unchanged; its only job then is to make the row + * visible to refresh-slice.sql's reconciliation pass, whose `contract_base` requires the contract to + * HAVE an amendment. With a raised `afterBgn` it also trips the annex condition that makes that pass + * actually recompute the flag rather than keep the one the INSERT path assigned. + */ +function seedAnnex(dbPath: string, unp: string, beforeBgn: number, afterBgn = beforeBgn): void { + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, contract_number, + contract_date, published_at, unp, authority_eik, authority_name, value_before, value_after, + value_delta, currency, contractor_eik, description) + VALUES + ('eop:annexes:${unp}', 2026, 'eop', '2026-06-08T00:00:00Z', '1', 'AMD-${unp}', 'C-${unp}', + '2026-06-01', '2026-06-09', '${unp}', '${AUTH_EIK}', 'Тестов възложител', ${beforeBgn}, + ${afterBgn}, ${afterBgn - beforeBgn}, 'BGN', '${BIDDER_EIK}', 'Изменение');`, + ); +} + +interface Row { + id: string; + value_flag: string; + amount_eur: number | null; +} + +const flagsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT id, value_flag, ROUND(amount_eur) AS amount_eur FROM contracts`, + ); + const out = new Map(); + for (const r of rows) out.set(r.id.split(':')[2]!, r); + return out; +}; + +describe('стотинки band measured against the lot estimate (issue #247)', () => { + for (const [label, scriptPath] of etlPaths) { + it(`${label}: catches the reported multi-lot case the procedure band misses`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // The shape of 00621-2020-0008: exactly 100x the lot estimate, but only 89.8x the procedure + // estimate, so the #298 band (which measures against the procedure) never sees it. + { + unp: 'UNP-LOT100', + valueBgn: OWN_BGN * 100, + ownEstBgn: OWN_BGN, + procEstBgn: Math.round(OWN_BGN * (100 / 89.8)), + }, + ]); + readScript(dbPath, scriptPath); + const row = flagsByUnp(dbPath).get('UNP-LOT100'); + expect(row, 'the seeded contract must reach the served table').toBeDefined(); + expect(row?.value_flag).toBe('value_suspect'); + // Repaired to the procedure estimate, the established anchor for value_suspect. + expect(row?.amount_eur).toBe(111_358); + }); + }); + + it(`${label}: spares a unit-price call-off at 100x its own per-unit estimate`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // A framework/unit-price procedure: the row estimate is a UNIT price, the call-off is 100x it + // in absolute terms but only a fifth of the procedure ceiling. Repairing this would DESTROY a + // real contract - it must stay untouched. + { + unp: 'UNP-UNIT', + valueBgn: OWN_BGN * 100, + ownEstBgn: OWN_BGN, + procEstBgn: OWN_BGN * 500, + }, + ]); + readScript(dbPath, scriptPath); + const row = flagsByUnp(dbPath).get('UNP-UNIT'); + // Without this the `.not.toBe` below passes vacuously on an unseeded row (review cefothe #1). + expect(row, 'the seeded contract must reach the served table').toBeDefined(); + expect(row?.value_flag).not.toBe('value_suspect'); + // Served at face value: 100 x 100_000 EUR. + expect(row?.amount_eur).toBe(10_000_000); + }); + }); + + it(`${label}: pins the 95x/105x edges of the band`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // Exactly on each edge — both inside, because the band is inclusive on purpose. + { unp: 'UNP-AT95', valueBgn: OWN_BGN * 95, ownEstBgn: OWN_BGN, procEstBgn: OWN_BGN * 9 }, + { + unp: 'UNP-AT105', + valueBgn: OWN_BGN * 105, + ownEstBgn: OWN_BGN, + procEstBgn: OWN_BGN * 10, + }, + // One step outside each edge — a dropped decimal lands AT ~100x, never at 94x or 106x, so + // these are ordinary large contracts and must keep their money. + { unp: 'UNP-AT94', valueBgn: OWN_BGN * 94, ownEstBgn: OWN_BGN, procEstBgn: OWN_BGN * 9 }, + { + unp: 'UNP-AT106', + valueBgn: OWN_BGN * 106, + ownEstBgn: OWN_BGN, + procEstBgn: OWN_BGN * 10, + }, + ]); + readScript(dbPath, scriptPath); + const flags = flagsByUnp(dbPath); + for (const unp of ['UNP-AT95', 'UNP-AT105', 'UNP-AT94', 'UNP-AT106']) + expect(flags.get(unp), `seeded contract ${unp} missing`).toBeDefined(); + expect(flags.get('UNP-AT95')?.value_flag).toBe('value_suspect'); + expect(flags.get('UNP-AT105')?.value_flag).toBe('value_suspect'); + expect(flags.get('UNP-AT94')?.value_flag).not.toBe('value_suspect'); + expect(flags.get('UNP-AT106')?.value_flag).not.toBe('value_suspect'); + expect(flags.get('UNP-AT94')?.amount_eur).toBe(9_400_000); + expect(flags.get('UNP-AT106')?.amount_eur).toBe(10_600_000); + }); + }); + + it(`${label}: pins the 1000 EUR floors on both estimates`, () => { + withEtlDb(label, (dbPath) => { + const TINY = 1_000; // ≈ 511 EUR — under the floor + seed(dbPath, [ + // Own estimate under 1000 EUR: at these sizes a 100x ratio is noise, not a dropped decimal. + { unp: 'UNP-TINYOWN', valueBgn: TINY * 100, ownEstBgn: TINY, procEstBgn: TINY * 10 }, + // Procedure estimate under 1000 EUR: same reasoning on the other half of the conjunction. + { unp: 'UNP-TINYPROC', valueBgn: OWN_BGN * 100, ownEstBgn: OWN_BGN, procEstBgn: TINY }, + ]); + readScript(dbPath, scriptPath); + const flags = flagsByUnp(dbPath); + expect(flags.get('UNP-TINYOWN'), 'seeded contract missing').toBeDefined(); + expect(flags.get('UNP-TINYPROC'), 'seeded contract missing').toBeDefined(); + expect(flags.get('UNP-TINYOWN')?.value_flag).not.toBe('value_suspect'); + expect(flags.get('UNP-TINYPROC')?.value_flag).not.toBe('value_suspect'); + }); + }); + + it(`${label}: a row with NO own estimate is not caught by the own-row arm`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // No own estimate at all. The own-row arm has nothing to measure against and must not fire; + // the procedure-level band (#298) is the only thing that could still speak here, and at 10x + // the procedure estimate it does not. This is the one place the INSERT and the UPDATE path + // could disagree, because the reconciliation pass reads the estimate from a different column. + { unp: 'UNP-NOOWN', valueBgn: OWN_BGN * 100, ownEstBgn: null, procEstBgn: OWN_BGN * 10 }, + ]); + seedAnnex(dbPath, 'UNP-NOOWN', OWN_BGN * 100); + readScript(dbPath, scriptPath); + const row = flagsByUnp(dbPath).get('UNP-NOOWN'); + expect(row, 'the seeded contract must reach the served table').toBeDefined(); + expect(row?.value_flag).not.toBe('value_suspect'); + expect(row?.amount_eur).toBe(10_000_000); + }); + }); + + it(`${label}: pins the procedure-level floor of the conjunction at 10x`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // Exactly at the floor: 100x the lot estimate and exactly 10x the procedure estimate. + { + unp: 'UNP-AT10', + valueBgn: OWN_BGN * 100, + ownEstBgn: OWN_BGN, + procEstBgn: OWN_BGN * 10, + }, + // Just under it: 100x the lot estimate but only ~8.9x the procedure estimate. + { + unp: 'UNP-UNDER10', + valueBgn: OWN_BGN * 100, + ownEstBgn: OWN_BGN, + procEstBgn: Math.round(OWN_BGN * 11.2), + }, + ]); + readScript(dbPath, scriptPath); + const flags = flagsByUnp(dbPath); + expect(flags.get('UNP-AT10'), 'seeded contract missing').toBeDefined(); + expect(flags.get('UNP-UNDER10'), 'seeded contract missing').toBeDefined(); + expect(flags.get('UNP-AT10')?.value_flag).toBe('value_suspect'); + expect(flags.get('UNP-UNDER10')?.value_flag).not.toBe('value_suspect'); + // Served at face value — the spared row keeps its money, which is the thing that matters. + expect(flags.get('UNP-UNDER10')?.amount_eur).toBe(10_000_000); + }); + }); + } + + // refresh-slice only. The reconciliation pass keeps whatever flag the INSERT path computed unless the + // contract ALSO trips an annex condition — so that is the only shape in which its own copy of the band + // decides anything, and the only way to pin it. (review cefothe #4/#5) + it(`refresh-slice reconciliation: converts the own estimate through ITS currency, not the contract's`, () => { + withEtlDb('reconcile-currency', (dbPath) => { + const AFTER_BGN = OWN_BGN * 100; // 10 000 000 EUR + const SIGNING_BGN = 100_000; // an annex ratio ≥ 100x, which forces the recompute + seed(dbPath, [ + { + unp: 'UNP-RECCUR', + valueBgn: SIGNING_BGN, + ownEstBgn: 100_000, // …denominated in EUR: the contract value is exactly 100x it + ownEstCurrency: 'EUR', + procEstBgn: 977_915, // 500 000 EUR, so the 10x procedure floor is cleared + }, + ]); + seedAnnex(dbPath, 'UNP-RECCUR', SIGNING_BGN, AFTER_BGN); + readScript(dbPath, resolve(root, 'scripts/refresh-slice.sql')); + const row = flagsByUnp(dbPath).get('UNP-RECCUR'); + expect(row, 'the seeded contract must reach the served table').toBeDefined(); + // Converting the estimate with the CONTRACT's currency divides an already-EUR figure by 1.95583, + // lands at ~196x instead of 100x, drops out of the band — and the row falls through to + // annex_suspect, which repairs to the signing value: 51 129 EUR shown instead of 500 000. + expect(row?.value_flag).toBe('value_suspect'); + expect(row?.amount_eur).toBe(500_000); + }); + }); +}); diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 1c1f7a11..246e91ec 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -895,7 +895,16 @@ FROM ( -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with -- nothing between 105x and 200x, so the band is narrow on purpose. - OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) THEN 'value_suspect' + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) + -- The same dropped decimal point, but on a LOT of a multi-lot procedure: there the signature + -- is 100x the lot's OWN estimate, and the ratio to the whole procedure lands wherever the + -- lot's share puts it (issue #247 reports one at 89.8x). Measuring against the own-row + -- estimate alone would be unsafe - for framework and unit-price procedures it is a UNIT + -- price and a whole call-off legitimately dwarfs it - so it is paired with the procedure + -- level condition that already means "implausibly large for this procedure" (>= 10x, the + -- review threshold). A unit-price call-off sits at ~1x the procedure estimate and is spared. + OR (c.own_est_eur >= 1000 AND c.proc_est_eur >= 1000 AND c.eff_eur >= 10 * c.proc_est_eur + AND c.eff_eur >= 95 * c.own_est_eur AND c.eff_eur <= 105 * c.own_est_eur) THEN 'value_suspect' -- value_low: zero/negative, OR a tiny signed value (< 1000 EUR) that is also < 5% of the -- estimate. Large legitimate framework call-offs (small share of a huge ceiling but big in -- absolute terms) are NOT caught — the < 1000 EUR floor keeps them OUT of value_low. @@ -1084,6 +1093,24 @@ FROM ( LIMIT 1 ) END AS proc_est_eur, + -- Own-row (per-lot) estimate in EUR. The "too high" flags deliberately measure against the + -- PROCEDURE estimate, because for framework and unit-price procedures this per-row number is a + -- UNIT price and a whole call-off legitimately dwarfs it. It is computed here ONLY for the + -- стотинки band, which pairs it with a procedure-level condition for exactly that reason. + CASE + WHEN c.estimated_value IS NULL THEN NULL + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.estimated_value + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.estimated_value / 1.95583 + ELSE c.estimated_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END AS own_est_eur, t.estimated_value AS proc_est_native, aw.currency AS amendment_currency FROM raw_contracts c @@ -1321,7 +1348,16 @@ SELECT 1, -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with -- nothing between 105x and 200x, so the band is narrow on purpose. - OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) THEN 'value_suspect' + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) + -- The same dropped decimal point, but on a LOT of a multi-lot procedure: there the signature + -- is 100x the lot's OWN estimate, and the ratio to the whole procedure lands wherever the + -- lot's share puts it (issue #247 reports one at 89.8x). Measuring against the own-row + -- estimate alone would be unsafe - for framework and unit-price procedures it is a UNIT + -- price and a whole call-off legitimately dwarfs it - so it is paired with the procedure + -- level condition that already means "implausibly large for this procedure" (>= 10x, the + -- review threshold). A unit-price call-off sits at ~1x the procedure estimate and is spared. + OR (c.own_est_eur >= 1000 AND c.proc_est_eur >= 1000 AND c.eff_eur >= 10 * c.proc_est_eur + AND c.eff_eur >= 95 * c.own_est_eur AND c.eff_eur <= 105 * c.own_est_eur) THEN 'value_suspect' WHEN COALESCE(c.current_value, c.signing_value) <= 0 THEN 'value_low' WHEN c.estimated_value > 0 AND c.signing_value IS NOT NULL AND ( CASE @@ -1500,6 +1536,24 @@ SELECT 1, LIMIT 1 ) END AS proc_est_eur, + -- Own-row (per-lot) estimate in EUR. The "too high" flags deliberately measure against the + -- PROCEDURE estimate, because for framework and unit-price procedures this per-row number is a + -- UNIT price and a whole call-off legitimately dwarfs it. It is computed here ONLY for the + -- стотинки band, which pairs it with a procedure-level condition for exactly that reason. + CASE + WHEN c.estimated_value IS NULL THEN NULL + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.estimated_value + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.estimated_value / 1.95583 + ELSE c.estimated_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END AS own_est_eur, t.estimated_value AS proc_est_native, aw.currency AS amendment_currency FROM raw_contracts c diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 5bd90796..0f6fac95 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -1283,7 +1283,15 @@ FROM ( -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with -- nothing between 105x and 200x, so the band is narrow on purpose. - OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) THEN 'value_suspect' + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) + -- The same dropped decimal point on a LOT of a multi-lot procedure: 100x the lot's OWN + -- estimate, while the ratio to the whole procedure lands wherever the lot's share puts it + -- (issue #247 reports one at 89.8x). The own-row estimate alone is unsafe - for framework + -- and unit-price procedures it is a UNIT price and a whole call-off legitimately dwarfs it - + -- so it is paired with the procedure-level condition that already means "implausibly large + -- for this procedure" (>= 10x, the review threshold). + OR (c.own_est_eur >= 1000 AND c.proc_est_eur >= 1000 AND c.eff_eur >= 10 * c.proc_est_eur + AND c.eff_eur >= 95 * c.own_est_eur AND c.eff_eur <= 105 * c.own_est_eur) THEN 'value_suspect' -- value_low: zero/negative, OR a tiny signed value (< 1000 EUR) that is also < 5% of the -- estimate. The < 1000 EUR floor keeps large legitimate framework call-offs OUT. WHEN COALESCE(c.current_value, c.signing_value) <= 0 THEN 'value_low' @@ -1443,6 +1451,23 @@ FROM ( LIMIT 1 ) END AS proc_est_eur, + -- Own-row (per-lot) estimate in EUR - see the note at the стотинки band below: used ONLY + -- there, paired with a procedure-level condition, because for framework and unit-price + -- procedures this number is a UNIT price. + CASE + WHEN c.estimated_value IS NULL THEN NULL + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.estimated_value + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.estimated_value / 1.95583 + ELSE c.estimated_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END AS own_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c JOIN contractor_identity ci @@ -1631,7 +1656,15 @@ FROM ( -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with -- nothing between 105x and 200x, so the band is narrow on purpose. - OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) THEN 'value_suspect' + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) + -- The same dropped decimal point on a LOT of a multi-lot procedure: 100x the lot's OWN + -- estimate, while the ratio to the whole procedure lands wherever the lot's share puts it + -- (issue #247 reports one at 89.8x). The own-row estimate alone is unsafe - for framework + -- and unit-price procedures it is a UNIT price and a whole call-off legitimately dwarfs it - + -- so it is paired with the procedure-level condition that already means "implausibly large + -- for this procedure" (>= 10x, the review threshold). + OR (c.own_est_eur >= 1000 AND c.proc_est_eur >= 1000 AND c.eff_eur >= 10 * c.proc_est_eur + AND c.eff_eur >= 95 * c.own_est_eur AND c.eff_eur <= 105 * c.own_est_eur) THEN 'value_suspect' -- value_low: zero/negative, OR a tiny signed value (< 1000 EUR) that is also < 5% of the -- estimate. The < 1000 EUR floor keeps large legitimate framework call-offs OUT. WHEN COALESCE(c.current_value, c.signing_value) <= 0 THEN 'value_low' @@ -1791,6 +1824,23 @@ FROM ( LIMIT 1 ) END AS proc_est_eur, + -- Own-row (per-lot) estimate in EUR - see the note at the стотинки band below: used ONLY + -- there, paired with a procedure-level condition, because for framework and unit-price + -- procedures this number is a UNIT price. + CASE + WHEN c.estimated_value IS NULL THEN NULL + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.estimated_value + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.estimated_value / 1.95583 + ELSE c.estimated_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END AS own_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c JOIN contractor_identity ci @@ -2047,6 +2097,39 @@ WITH contract_base AS ( ORDER BY rc.source DESC, rc.id DESC LIMIT 1 ), te.estimated_value) AS classifier_estimated_value, + c.signed_at, + -- The contract row's OWN estimate and the currency it is denominated in, for the стотинки band's + -- own-row arm (#247). Deliberately WITHOUT the procedure fallback classifier_estimated_value carries: + -- the arm asks "is this 100× the row's own estimate", and a row with no own estimate has no answer. + -- Falling back would make the arm fire exactly where the procedure band already fires, i.e. mean + -- something other than its name. The four INSERT sites read raw_contracts.estimated_value the same way. + ( + SELECT rc.estimated_value + FROM raw_contracts rc + WHERE rc.unp = substr(c.tender_id, 3) + AND rc.contract_number = c.contract_number + AND ( + (c.id LIKE 'c:e:%' AND rc.source LIKE 'eop:%') + OR (c.id LIKE 'c:o:%' AND rc.source LIKE 'ocds:%') + ) + ORDER BY rc.source DESC, rc.id DESC + LIMIT 1 + ) AS own_est_native, + -- Same row, same order — the estimate's OWN currency chain, byte-for-byte the one the four INSERT + -- sites use (procurement_currency → currency → BGN). Reading it off contracts.currency instead would + -- convert a foreign-currency estimate at the contract's currency (review cefothe #4). + ( + SELECT COALESCE(NULLIF(rc.procurement_currency, ''), NULLIF(rc.currency, ''), 'BGN') + FROM raw_contracts rc + WHERE rc.unp = substr(c.tender_id, 3) + AND rc.contract_number = c.contract_number + AND ( + (c.id LIKE 'c:e:%' AND rc.source LIKE 'eop:%') + OR (c.id LIKE 'c:o:%' AND rc.source LIKE 'ocds:%') + ) + ORDER BY rc.source DESC, rc.id DESC + LIMIT 1 + ) AS own_est_currency, -- One annex step jumped ≥10× — half of the mis-keyed-annex conjunction below. Checked against -- the CUMULATIVE domain amendments, matching where this pass re-rolls current_value from. EXISTS ( @@ -2169,7 +2252,10 @@ WITH contract_base AS ( -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with -- nothing between 105x and 200x, so the band is narrow on purpose. - OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) THEN 'value_suspect' + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) + -- Own-row (per-lot) arm, mirroring the two INSERT-time copies above. + OR (c.own_est_eur >= 1000 AND c.proc_est_eur >= 1000 AND c.eff_eur >= 10 * c.proc_est_eur + AND c.eff_eur >= 95 * c.own_est_eur AND c.eff_eur <= 105 * c.own_est_eur) THEN 'value_suspect' WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 -- Mis-keyed annex: a single step jumped ≥10× AND the aggregate ended ≥5× over signing. -- The step alone is NOT enough — some chains have a huge step that a later annex pulls @@ -2183,7 +2269,27 @@ WITH contract_base AS ( WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS new_value_flag - FROM contract_base c + FROM ( + -- Per-lot estimate in EUR for the стотинки band's own-row arm (see the note at the band). Mirrors + -- the four INSERT sites: the row's OWN estimate (NULL when it has none), converted through the + -- estimate's own currency with a dated fx lookup — not through the contract's currency or fx_rate. + SELECT cb.*, + CASE + WHEN cb.own_est_native IS NULL THEN NULL + WHEN cb.own_est_currency = 'EUR' THEN cb.own_est_native + WHEN cb.own_est_currency = 'BGN' THEN cb.own_est_native / 1.95583 + ELSE cb.own_est_native * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = cb.own_est_currency + AND f.rate_date <= cb.signed_at + AND f.rate_date >= date(cb.signed_at, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END AS own_est_eur + FROM contract_base cb + ) c ), calc AS ( SELECT id, new_value_flag, proc_est_eur, CASE new_value_flag From 215123debc93b5b73573bc3faf09a61120902ccc Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 14 Aug 2026 14:04:44 +0000 Subject: [PATCH 69/69] Merge origin/main into feat/registry-evidence-links (#304) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Единственият конфликт е познатият: новият тест на #304 строи схемата без 0009, а refresh-slice.sql чете interest_link_evidence. Добавена е миграцията в неговата верига. db 487, web 493, ingest 84, etl 20, shared 45, config 10. Typecheck и prettier чисти. --- packages/db/src/value-flag-lot-stotinki-sql.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/db/src/value-flag-lot-stotinki-sql.test.ts b/packages/db/src/value-flag-lot-stotinki-sql.test.ts index 388d3172..67fea352 100644 --- a/packages/db/src/value-flag-lot-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-lot-stotinki-sql.test.ts @@ -28,6 +28,8 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +// #279/ADR-0033: refresh-slice.sql's свързани-лица block reads interest_link_evidence, so 0009 too. +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -60,6 +62,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); + readScript(dbPath, migration9Path); readScript(dbPath, stagingPath); run(dbPath); } finally {