From a40df206668454efe8c9e60de25084ba297f6153 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 12:07:28 +0300 Subject: [PATCH 01/50] =?UTF-8?q?docs:=20ADR-0033=20=E2=80=94=20registry?= =?UTF-8?q?=20evidence=20replaces=20the=20name-distinctiveness=20tier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records the decision before any code: a link publishes only against a checkable Trade Register fact (#279). If review rejects the grounds, the whole approach falls — which is why this ADR comes first rather than last. Seven decisions, each separately contestable: the evidence ladder (six outcomes, first match wins); a heuristic that grounds an assertion — the explicit exception to ADR-0007 decisions 2 and 3, together with what justifies it (corroboration against the official's own filing, never a standalone name match) and the filters that can only withhold; the joint-stock bar as a union of three signals, where an unknown legal-form code withholds; reconciliation of a terminated stake against the live deed, reversing ADR-0021 E11 (the "и към днешна дата" labels defer to phase 2); the deed cache, extending ADR-0010 decision 6 with a 35-day retention; monotonicity as a gate rather than a store — §8 of the issue is false as written; and the launch gates. Supersedes ADR-0009, ADR-0015, ADR-0017 (headers and index synchronised). Amends ADR-0007, ADR-0010, ADR-0021 (E11 only), ADR-0028 — accepted ADRs are not rewritten, so the amendment is recorded here. Cross-checked against issue #279 and the earlier spike; carries the three facts the issue does not: the sustained 429 at ~50 requests, the open question of the ЕАД legal-form code, and the deliberate non-use of the beneficial-ownership fields (C-37/20). --- ...name-uniqueness-guard-and-publish-tiers.md | 3 +- docs/adr/0015-tr-name-uniqueness-census.md | 3 +- docs/adr/0017-name-collision-tier-gate.md | 3 +- ...-evidence-replaces-name-distinctiveness.md | 309 ++++++++++++++++++ docs/adr/README.md | 7 +- 5 files changed, 319 insertions(+), 6 deletions(-) create mode 100644 docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md diff --git a/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md b/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md index 7803e901a..8783b3ac3 100644 --- a/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md +++ b/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md @@ -1,8 +1,9 @@ # ADR-0009: Name-uniqueness is not absolute → single-ЕИК guard + publish tiers -- Status: Accepted +- Status: Superseded by [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Date: 2026-07-05 - Deciders: lb, Claude +- Superseded by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Related: spec §5; refines [ADR-0008](0008-deterministic-name-to-eik-resolution.md) ## Context diff --git a/docs/adr/0015-tr-name-uniqueness-census.md b/docs/adr/0015-tr-name-uniqueness-census.md index b3fb27891..c86d8eece 100644 --- a/docs/adr/0015-tr-name-uniqueness-census.md +++ b/docs/adr/0015-tr-name-uniqueness-census.md @@ -1,8 +1,9 @@ # ADR-0015: TR name-uniqueness census (promoting tier-C generic-name matches) -- Status: Accepted (design; implemented as a Phase-1 pipeline step) +- Status: Superseded by [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Date: 2026-07-05 - Deciders: lb, Claude +- Superseded by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Related: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md); spec §5 ## Context diff --git a/docs/adr/0017-name-collision-tier-gate.md b/docs/adr/0017-name-collision-tier-gate.md index 8d4a234be..38ed31b52 100644 --- a/docs/adr/0017-name-collision-tier-gate.md +++ b/docs/adr/0017-name-collision-tier-gate.md @@ -1,8 +1,9 @@ # ADR-0017: A globally non-unique name cannot ride the name-distinctive tier — even with a certain ЕИК -- Status: Accepted +- Status: Superseded by [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Date: 2026-07-05 - Deciders: lb, Claude +- Superseded by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — the name-distinctive tier it gates leaves the publishing path; the `nameGloballyUnique` precondition survives, narrowed to the weakest evidence rung - Related: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md), [ADR-0016](0016-free-text-entity-resolution.md); `scripts/cacbg/load.mjs`, `scripts/cacbg/audit.mjs` ## Context diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md new file mode 100644 index 000000000..e4d8ff2d0 --- /dev/null +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -0,0 +1,309 @@ +# ADR-0033: A link publishes only against a Trade Register fact — the evidence ladder replaces the name-distinctiveness tier + +- Status: Accepted (design; implemented as the #279 pipeline change) +- Date: 2026-08-05 +- Deciders: Todor (maintainer), Claude +- Supersedes: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md), [ADR-0015](0015-tr-name-uniqueness-census.md), [ADR-0017](0017-name-collision-tier-gate.md) +- Amends: [ADR-0007](0007-scope-and-certainty-bar.md) (decisions 2 and 3), [ADR-0010](0010-pii-posture.md) (decisions 3 and 6), [ADR-0021](0021-methodology-page-and-temporal-freshness.md) (E11 only — E10 stands and is strengthened), [ADR-0028](0028-declared-eik-is-a-determining-identifier.md) +- Related: [ADR-0008](0008-deterministic-name-to-eik-resolution.md), [ADR-0011](0011-host-scoped-tls-pinning.md), [ADR-0026](0026-person-grain-name-institution.md), [ADR-0027](0027-overmerge-gate-is-telemetry-not-a-gate.md), [ADR-0031](0031-suppressions-version-controlled-fingerprinted.md), [ADR-0032](0032-family-ownership-published-under-public-interest.md); spec §3.3/§5/§8/§9, [related-persons-lia.md](../spec/related-persons-lia.md); `scripts/tr/`, `scripts/cacbg/{load,classify}.mjs`, `packages/db/src/queries/related-persons.ts`, `apps/web/app/routes/conflict*.tsx` + +## Context + +Today the identity of the company behind a declared name is decided by a heuristic. `B_distinctive` +(ADR-0009) publishes a link when the name is *structurally* distinctive — a digit, a Latin token, or ≥3 +content words. It fails in both directions: it publishes on a bare name coincidence, and it withholds +everything else. Of the 101 links published today, **36 rest on name coincidence alone**. + +The Trade Register answers the identity question directly, over a public endpoint with no authentication: +`GET https://portal.registryagency.bg/CR/api/Deeds/{ЕИК}`. Either the declarant is named in the company's +live deed, or facts they declared match the ones registered against that ЕИК. + +Four things must be on the table before the decision, because each of them changes what is being decided. + +**1. This is an expansion, not a tightening.** The measured surface grows from 101 links to **329 links / +311 people / 239 companies**. „From held / published / withdrawn = 163 / 65 / 101" means **264 links we +deliberately withhold today become named public claims**. At 329 links a 1% error rate is three falsely +named people. The precision bar has to go **up**, not stay flat — which is why the acceptance gate below is +a hand-labelled sample and not the control totals. + +**2. The second rung is a heuristic used to assert — stated precisely.** What the registry evidence +establishes is **the identity of the company**, not that the official owns it: the ownership claim comes +from the official's own filed declaration and is not a heuristic at all. Rung 2 („Документ") confirms that +*the company this declared name refers to is the same legal entity as the winner we matched*, by finding +the declarant's full name inside that company's live fields. A homonym failure therefore does not invent an +ownership claim — it **attaches the official to the wrong company's ЕИК, contracts and money**. That is a +different error from the one the adversarial reading suggests, and it is still a false public claim about a +named person. + +A three-name subset match against free text, in a register that carries no ЕГН, is **not deterministic**. +ADR-0007's decision list rules it out twice: item 2 (published only when the official↔company↔ЕИК +resolution is *deterministic*) and item 3 (heuristics may *withhold* or *triage*, **never assert**). #279 +replaces a weaker heuristic with a stronger one and promotes it to grounds for assertion. That is the +decision this ADR exists to make explicit; it cannot be smuggled in as an implementation detail. + +**3. The shape of the access is the permitted one.** Spec §3.3 permits „bounded per-ЕИК … thousands, not +~900k" and forbids bulk scraping; ADR-0007 decision 1 already anticipated the „per-ЕИК lookup only, later +leg". The crawl is one lookup per candidate ЕИК — on the order of hundreds — with a closed candidate set: +the crawler never follows a link out of a deed. This is that later leg, not the bulk reuse. + +**4. The endpoint rate-limits, and the limiter is a stated preference.** An earlier spike against this same +API triggered **HTTP 429 at roughly 50 cumulative requests ending in a burst**, and the block was then +*sustained* — every subsequent request returned 429, including simple `/Deeds/{eik}` calls that had worked +seconds earlier. There is no `Retry-After` and no `X-RateLimit-*` header, so a client can only avoid +tripping the limit, never pace against a published one; 25 spaced requests were fine, so pacing matters more +than total volume. #279 §3's „~400 companies, about 20 minutes" is therefore feasible only at its stated +1 request / 3 seconds with no burst, and only if the crawl is resumable. The limiter is the operator's only +available way to express a rate preference, and tuning around it empirically is precisely the posture spec +§3.3's „NEVER bulk-scrape" exists to forbid. This is the evidence behind decision 7's crawl hygiene. + +**5. Parts of the source description in #279 were wrong, and were corrected by live probing** (9 sequential +requests, 2026-08-04 — well inside the pacing above, so no 429 was encountered): + +- `?entryDate=` **is** honoured — a bogus-parameter control returned byte-identical output to the + plain request, so the difference is the parameter, not noise. `…/Fields/{ident}/History` **does** return + JSON, not application HTML. #279 §3 states the opposite for both. +- A wrong `subUIC` yields **HTTP 500 `GL_ERROR_L`**, not 404 — so a 500 can never be read as „no history" + or „outside the register". +- `legalForm` 4 = ООД and 10 = ЕООД (as #279 says); the nomenclature endpoint does **not** settle the enum + — `legalForm=4` and `legalForm=10` return a byte-identical catalogue. +- The envelope's `fullName` carries the legal form (`"ПИМК" ООД`) while `CR_F_2_L` is bare. The joint-stock + bar can therefore be derived from the ЗТРРЮЛНЦ-mandated suffix, at zero extra requests. +- Erasure is **structural, not textual**: an erased entity is a `record-container` block carrying the class + `field-text--erased`. Entities inside one field are separated by + `
` blocks delimited by `
`. +- Erased versions are **live in the current deed**: the first company sampled carries a fully-erased + `CR_F_23_L` dated 2013-07-16, which read naïvely becomes „latest ownership entry: 2013-07-16" and feeds + the refutation rule. +- Seats move. The same company's `CR_F_5_L` shows entry dates 2010-11-04 and 2014-01-23. + +The as-of capability that `?entryDate=` unlocks would fix the two weakest rungs, but it **re-baselines every +control number in #279 §10**. It is therefore deliberately out of scope here and becomes a separate change +under a new rules version — which decision 6 makes the only sanctioned way the surface may move. + +## Decision + +### 1. The evidence ladder replaces the publish tiers + +For every resolved link (person × ЕИК) with a fetched deed, **the first matching rung wins**: + +| # | Outcome (data label) | Condition | Effect | +|---|---|---|---| +| 1 | **„Бар: акционерна форма"** — joint-stock bar | the company is a joint-stock form (АД / ЕАД / КДА) | never published, whatever follows | +| 2 | **„Документ"** — documented | the declarant's full name appears in a live `CR_F_7_L` / `CR_F_18_L` / `CR_F_19_L` / `CR_F_23_L` | published; the role (owner vs manager only) is kept for the label | +| 3 | **„Потвърдено"** — confirmed | the normalized declared seat equals the registered seat for that ЕИК, **or** the declarant wrote the ЕИК in the declaration | published | +| 4 | **„Оборена"** — refuted | *own* stake only: the person appears in no live field, **and** the latest entry date across the live ownership fields is strictly before the first declared year | link withdrawn | +| 5 | **„Неизвестна"** — unknown | anything else | held | +| 6 | **„Извън ТР"** — outside the register | the ЕИК is not in the register (ДЗЗД, БУЛСТАТ associations) | held | + +The Bulgarian labels are the persisted vocabulary — they reach the data, the audit and the methodology +page — so they are fixed here rather than translated at each layer. + +Erased versions are skipped everywhere the live state is read — otherwise the date on an erased record can +„certify" a state that is not in force. + +`publish_tier` carries the evidence kind. **`B_distinctive` leaves the publishing path entirely**; +name distinctiveness survives only as an ordering signal for the review queue. Rung 3's seat leg takes the +declared seat **only from declarations by the same person for the same company**: 4.9% of company-name keys +carry more than one distinct declared seat, so a company-only key would let one person's seat confirm +another person's link. + +This supersedes ADR-0009 (the tier ladder) and ADR-0015 (the name census, whose only job was to unblock +`C_hold`; `tr-census.mjs` and its `promote()` are deleted). ADR-0028's holding survives — a declared ЕИК +*is* the identity — but it is now rung 3 rather than a tier of its own, it is subject to rung 1, and its +census-exemption clause is moot. + +### 2. A heuristic may ground an assertion here — the argued exception to ADR-0007 + +Rung 2 asserts on a name match. We accept it, on these grounds and no wider: + +- **It is a full-subset match, patronymic included.** Partial (2-of-3) matching is refused outright: of 301 + matches measured, 46 were two-token only — precisely the homonym risk. A declarant name with fewer than + three tokens can never earn „Документ". +- **It is scoped to a single registry entity.** Tokens are matched only within one `record-container` + block, after erased blocks are dropped and after HTML entities are decoded. Matching against a whole + field's text would combine the given name of one person with the surname of another — the defect that + ships a libel. +- **The company identity behind it passed preventive control.** The candidate ЕИК comes from the + deterministic exact-name resolution of ADR-0008/#226, not from fuzzy matching. +- **It is corroborated, and that is the whole difference.** A bare TR name is spec §4's *weakest* join — + no ЕГН, no birthdate, not even ADR-0026's `(name, ведомство)` grain. The registry-graph spike that + explored this API concluded, for its own use, that no person-derived edge may reach a user-visible + surface at all, and it named the one cheap corroborator that would change that: *an officer who also + appears as a declarant with a declared stake in the same ЕИК, confirmed by an independent source*. That + is exactly what rung 2 is. The register supplies the name-in-this-company fact; the official's own filing + supplies the stake. Neither alone would publish. **The corroboration is what licenses the assertion, so + the rule must never be extended to a name-only join** — matching a person across two companies, or + treating a registry name as an identity in its own right, stays forbidden. +- **The filters that can only withhold are retained, not removed.** `nameGloballyUnique` and + `nameDistinctiveness` stay in the pipeline as an **AND-gate on the weakest rung only** („Потвърдено"). + They cost near-zero recall and preserve ADR-0017's outcome even though its subject — the name-distinctive + tier — is gone. +- **Homoglyphs are not folded.** `company-name-key.ts` deliberately does not fold Cyrillic↔Latin; person + names take the same posture. A Latin letter inside a name means no „Документ", and the occurrence is + counted rather than silently dropped. + +**The residual collision rate is estimated, not measured, and we say so.** Bulgarian names are three-part +by statute (ЗГР чл. 9) and the API renders full triples consistently. A triple of common components — +Георги · Иванов · Петров — is on the order of 10⁻⁵ of ~3.2M men, so roughly 29 people nationally share it; +but the population that can produce a false link is company officers, a ~1% slice, which squares the +probability to ~10⁻⁴ per colliding pair. That is an argument for plausibility, not a measurement, and the +residual concentrates on exactly the common names. It is therefore **not** the basis on which this +publishes — the hand-labelled sample of decision 7 is. Hyphenated surnames count as one token; Latin-script +names are counted the same way but flagged, since a three-token foreign name is not a patronymic triple. + +The tie-breaker for every rung is the repo's own sentence, from +`packages/shared/src/company-name-key.ts`: **„When in doubt the key stays MORE specific (a recall miss is +safe; an over-merge is not)."** + +This **amends ADR-0007 decisions 2 and 3**: a published claim may now rest on a disclosed, bounded, +entity-scoped full-name match against a public register, in addition to deterministic facts. Everything +else in ADR-0007 stands — in particular that every heuristic is disclosed on the methodology page and +labelled in the data, which decision 7 makes a launch condition rather than a follow-up. + +### 3. The joint-stock bar is a union of three independent signals + +The bar exists because the shareholder book is not public (so the claim is unverifiable) and a parcel of +listed shares is not a material ownership conflict (the „11 Trace shares → €88M" trap). It fires if **any** +of the following says joint-stock: + +1. `closelyHeldForm` on the **declared** name (`load.mjs`) — a different, earlier stage; it **stays**, and + is not redundant with the two below; +2. the фирма suffix taken from the deed envelope's `fullName`, through that same tested predicate; +3. the numeric `legalForm` code from the deed. + +**An unknown `legalForm` code withholds and is reported** — it never falls through to publication. The enum +is known only from four observed values and the nomenclature endpoint does not settle it (Context 5), so +fail-open here would be a bar that silently stops barring. `КДА` — which rung 1 explicitly requires barring +— is absent from both `JOINT_STOCK` and `FORM_TOKENS` in `classify.mjs` today and is added. + +**The named open question is ЕАД.** #279 §3 lists 5 = АД, and the spike's catalogue-derived table also +reads 5 as АД/ЕАД jointly — but ООД and ЕООД turned out **not** to share a code (4 and 10), so a separate +ЕАД code is the likelier reading and no observation settles it. A single-owner joint-stock company is +precisely the shape a declarant is most likely to hold and rung 1 most needs to bar, so this is the one +place where a fail-open enum would do real damage. Signal 2 (the ЗТРРЮЛНЦ suffix on `fullName`) covers it +independently of the code, which is why the bar is a union and not a lookup. + +### 4. Termination is reconciled against the live deed — reversing ADR-0021 E11 for own stakes + +ADR-0021 E11 marks an ownership link `withdrawn` when the company is absent from the person's latest +ownership filing. That is an inference from silence, and its commonest cause is a finished mandate rather +than a sale. Before the withdrawal takes effect, every terminated **own** stake is reconciled against the +live deed: a person still named in a live ownership field (`CR_F_18/19/23_L`) has not divested, and the +link surfaces. Family stakes are never reconciled — the relative's name is neither stored nor checked +(ADR-0010 decision 4, ADR-0032 decision 2), so declared termination applies to them directly. + +**Phase 1 uses the reconciliation only to avoid withdrawing the link.** The „и към днешна дата" labels of +#279 §7, and the post-period contracts they license, are **deferred to phase 2 behind an LIA addendum**: +those labels assert a present tense about a named person, on evidence whose freshness is bounded by the +cache refresh cycle, and they change the claim's shape enough to need the balancing assessment updated +first. The derived live status is therefore recomputed on every run and **never sealed**. + +ADR-0021 **E10 is untouched and strengthened** — see decision 7. + +### 5. The deed cache — amends ADR-0010 + +A deed contains third-party personal data: addresses of natural persons, and the names of people who are +not office holders. + +- **Extraction stays inside ADR-0010 decision 3.** `CR_F_5_L` yields only the „Населено място:" segment; + no parser function returns an address. Decision 3 is re-affirmed, not overridden. +- **Storage is what changes, and that is ADR-0010 decision 6.** Raw deeds are cached under git-ignored + `scratch/tr/`, behind the same refuse-to-run guard as the declaration cache, with a **35-day retention** + (one refresh cycle plus slack) and a purge step in the same job. Decision 6's scope extends from „raw + declaration XML … deleted post-spike" to a second source with a stated TTL. +- **The cache index holds no name at all** — only ЕИК, dates, codes and verdicts, plus a body hash instead + of any content excerpt. Names exist solely in the raw JSON files, are read only to produce a boolean, and + never enter a public table, a response, or a log. ЕГН was absent from every payload examined, and the + index additionally refuses any ten-digit run in a text column — sound because an ЕИК is 9 or 13 digits, + never 10. +- **The deed's beneficial-ownership fields are deliberately not used.** A deed also carries `CR_F_550_L` + (действителни собственици, чл. 63 ЗМИП) and the control fields `CR_F_537/538_L`. Using them would make a + beneficial-ownership claim, which ADR-0007 decision 1 parked after CJEU C-37/20. Rung 2 reads the four + fields #279 names and no others; widening it is a new decision, not an improvement. +- **The sealed `matched_fact` is a closed vocabulary** (`seat:`, `role:owner:`, `eik`), + enforced by a pattern check in the audit. Without that, the matched *name* eventually gets stored there + as a convenience, which is exactly what #279 §9 forbids. + +### 6. Monotonicity is a gate, not a store — and #279 §8 is corrected + +#279 §8 requires the evidence seal to be kept „forever" and recomputation to be strictly additive. As +written that is false, and the implementation must not pretend otherwise: + +- §7's labels flip. A person who leaves the register must lose the label *and* the post-period contracts it + licensed. (Phase 1 avoids this by not shipping the labels at all — decision 4.) +- A permanent seal and a live, expiring cache contradict each other. +- A **court-annulled entry** (чл. 29 ЗТРРЮЛНЦ) invalidates the evidence without any rules change. It is + wired to the correction path of ADR-0031 and named as a ground in the suppression runbook. + +Therefore: seals are **re-derived deterministically** on every run — a seal is written for *every* link, +including held ones, so the review queue explains itself — and monotonicity is enforced as a **gate**. The +audit compares against the pre-wipe export and raises a hard finding when a previously published link +disappears under an **unchanged** `rules_version`; under a changed version it degrades to a printed diff. +Removal remains an intentional event: a rules-version bump, or a correction of wrong input. + +### 7. Launch gates + +None of these are follow-ups. + +- **Precision is proven by a hand-labelled sample, not by the control totals.** A human verifies against the + portal: every reconciled link, every refuted link, and random samples of „Документ" and „Потвърдено". + **Pass mark: zero wrong-company and zero wrong-person errors.** The control numbers of #279 §10 are a + reproducibility check — and they are usable as one only after the §5/§10 discrepancy is resolved with the + author (§5's „102 по седалище, 3 по ЕИК" leaves those three ЕИК confirmations in no bucket of the §10 + row, which sums to exactly 568 without them), and re-measured on top of the §12 phantom-row fix, which + changed the corpus — that fix has landed as #281 (87 phantom rows across 15 sets; 256,286 announced − + 87 = 256,199, covered exactly by 255,582 fetched + 617 missing at source). +- **Every anti-false-zero control is a positive control.** „Sofia seat confirmations: 0" is indistinguishable + from a broken normalizer without one; so is a joint-stock bar with no marginal effect over + `closelyHeldForm`, and a matcher that always returns false (ADR-0027's lesson). +- **A partial cache must fail closed.** A missing *or* incomplete registry cache makes the loader throw. An + 80%-restored cache would otherwise yield roughly 80 published links — above the ship floor of 50 — and so + would ship a decimated surface and wipe the rest from production. The ship floor rises accordingly and is + passed explicitly rather than defaulted. +- **Crawl hygiene.** „Outside the register" is permanent only from a documented positive response; 429, + 5xx and timeouts are transient and are never cached as a negative. A 429 stops the run rather than + marking anything, and is never retried — „5 retries with growing backoff" and „429 stops the run" are + consistent only if retries exclude 429. Given Context 4, the crawl is sequential, paced, resumable, and + ends the run on the first 429 rather than backing off into it. **The deed's returned UIC must equal the + requested ЕИК**, or the deed is refused: an ЕИК is TEXT everywhere because public bodies' identifiers are + exactly the `000…` shape that loses its leading zeros to a numeric round-trip, and the failure mode of + losing them is fetching, and then publishing against, a different company's deed. +- **The methodology page carries the rule verbatim** (ADR-0021 E10): the ladder, the ≥3-token requirement, + the entity-boundary rule, the disclosure that identity rests on a name match **without ЕГН**, the homonym + and seat caveats, the lookup date, the refresh cadence and the retention. E10's existing promise that + held links „се показват едва след като регистърът стане достъпен" is finally kept by this change. +- **The PII rails are asserted, not assumed** — over the shipped dump and over the loader's own output. + +## Consequences + +- The surface roughly triples, and 264 links that are withheld today become named public claims. The + failure mode of the whole change is a **wrong-company** attribution, not a wrong publication decision. +- `tr-census.mjs` and the open-data census pipeline of ADR-0015 are removed. Its premise — that a name + proven nationally unique identifies the company — is the premise this ADR abandons. +- The pipeline gains a hard dependency on an external registry. Every path that lacked one before now has a + fail-closed branch: no cache, partial cache, unknown legal form, rate limit. +- We accept a slower, more fragile refresh in exchange for evidence. Decisions stay a pure, zero-network + function of declarations, cached deeds and contracts, but they no longer run on the 6-hourly ETL cycle + (#279 §9 assumes they do; they do not — the loader needs `node:sqlite`, the declaration corpus and the + full winner set, none of which exist inside a Worker). Cadence becomes two scheduled workflows: decisions + daily, registry lookups monthly. +- Two capabilities are deliberately left on the table and become separate changes under new rules versions: + as-of evidence via `?entryDate=`, and the §7 „и към днешна дата" labels with their post-period contracts. +- The seat rung is structurally capped and we accept the cap: a declared seat is captured **only** from the + ООД/ЕООД holdings table of an *asset* declaration (`parse.mjs`), so it can never rescue a link declared + solely in an interests declaration. This is probably consistent with the measured 102 — rung 1 bars the + joint-stock cases anyway — but it is confirmed against the corpus before launch, not assumed. + +### Measured outcome — recorded on completion + +This ADR is accepted on the design. Exactly one amendment is permitted afterwards: this subsection is +filled in with the measured result, and nothing else in the file is rewritten (repo convention — an +accepted ADR is superseded, not edited). To be recorded: + +- the realized ladder split against the #279 §10 control row, with each number naming the SQL that produced + it, and the 54-link gap between identified (281 + 102) and surfaced (329) attributed bucket by bucket; +- the true candidate-ЕИК count (#279's „~400" is an assumption; the figure is `COUNT(DISTINCT eik)` over + all links, not just published ones); +- the hand-labelled sample result; +- the marginal effect of the registry joint-stock bar over `closelyHeldForm`, and the count of unknown + `legalForm` codes encountered. diff --git a/docs/adr/README.md b/docs/adr/README.md index 0bd84ab6a..ecaa107fd 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -16,15 +16,15 @@ | [0006](0006-eop-wins-dedup.md) | Dedup на два източника: EOP печели по `contract_number` | Прието | | [0007](0007-scope-and-certainty-bar.md) | Свързани лица: обхват и праг на сигурност (само 100% детерминистични съвпадения) | Прието | | [0008](0008-deterministic-name-to-eik-resolution.md) | Детерминистично разрешаване име→ЕИК (авто-публикуване; ръчна опашка само при двусмислие) | Прието | -| [0009](0009-name-uniqueness-guard-and-publish-tiers.md) | Пазач за уникалност на името + нива на публикуване (A_seat / B_distinctive / C_hold) | Прието | +| [0009](0009-name-uniqueness-guard-and-publish-tiers.md) | Пазач за уникалност на името + нива на публикуване (A_seat / B_distinctive / C_hold) | Заменено от [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) | | [0010](0010-pii-posture.md) | Позиция за лични данни: без ЕГН/адреси; третите лица — само за вътрешна проверка | Прието | | [0011](0011-host-scoped-tls-pinning.md) | TLS pinning само за хоста (счупена верига на register.cacbg.bg), не глобален байпас | Прието | | [0012](0012-crawler-and-persistence-architecture.md) | Архитектура на обхождането и съхранението (resumable; кеш по xml_file + ControlHash) | Прието | | [0013](0013-two-declaration-templates.md) | Два шаблона декларации (имущество + интереси) в един парсер | Прието | | [0014](0014-match-output-layers-and-interpretation.md) | Слоеве на съвпадението и тълкуване (собственост/контрол, времеви, собствена институция) | Прието | -| [0015](0015-tr-name-uniqueness-census.md) | Преброяване за уникалност на имена от ТР — промотира глобално уникални tier-C връзки | Прието | +| [0015](0015-tr-name-uniqueness-census.md) | Преброяване за уникалност на имена от ТР — промотира глобално уникални tier-C връзки | Заменено от [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) | | [0016](0016-free-text-entity-resolution.md) | Разрешаване на субекти от свободен текст (деклариран ЕИК + извличане от проза) | Прието | -| [0017](0017-name-collision-tier-gate.md) | Гейт срещу колизия на имена извън отличаващото ниво | Прието | +| [0017](0017-name-collision-tier-gate.md) | Гейт срещу колизия на имена извън отличаващото ниво | Заменено от [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) | | [0018](0018-folder-discovery-and-republication-dedup.md) | Откриване на папки от индекса + dedup на препубликувани декларации (ControlHash) | Прието | | [0019](0019-private-interest-vs-ex-officio-classification.md) | Разделяне на частен финансов интерес от служебни борд-роли (multi-declarant tell) | Прието | | [0020](0020-conflict-explorer-surface-posture.md) | Повърхност на експлорера — само interest_links, noindex до одобрение, произход на всеки ред | Прието | @@ -40,5 +40,6 @@ | [0030](0030-family-ownership-withheld-nameless-aggregate.md) | Дял на свързано лице не влиза в поименната повърхност (v1): събира се и се одитира, но се отчита само като безименен сбор (GDPR C-37/20; заменя ADR-0023) | Заменено от [ADR-0032](0032-family-ownership-published-under-public-interest.md) | | [0031](0031-suppressions-version-controlled-fingerprinted.md) | Свалянията на връзки са версиониран списък с HMAC-отпечатък (salt = CI secret), прилаган при зареждане — не служебна таблица; не изтича сигнала „кой е свален" към прод | Прието | | [0032](0032-family-ownership-published-under-public-interest.md) | Дял на свързано лице се публикува на поименната повърхност наравно със собствения — на основание надделяващ обществен интерес (ЗДОИ чл. 41и; C-184/20 забранява само поименните данни); близкият никога не се назовава, връзката не се твърди; заменя ADR-0030 | Прието | +| [0033](0033-registry-evidence-replaces-name-distinctiveness.md) | Връзка се публикува само срещу проверим факт от Търговския регистър — доказателственият ред заменя нивата по отличителност на името; евристиката вече обосновава твърдение (изрично изключение от ADR-0007), барът за акционерни форми е обединение от три сигнала, монотонността е гейт, не хранилище; заменя ADR-0009/0015/0017 | Прието (дизайн) | Свързан проектен документ: [spec/related-persons-foundation.md](../spec/related-persons-foundation.md). From 4fde3778d08d9f4c268243a876a05b3751170db4 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 12:20:20 +0300 Subject: [PATCH 02/50] docs: ADR-0023 carries its superseded status The index has marked it superseded by ADR-0030 since PR #226, but the file header still read "Accepted". A header/index divergence, spotted while synchronising the index for ADR-0033. --- docs/adr/0023-anonymized-family-ownership-surface.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/adr/0023-anonymized-family-ownership-surface.md b/docs/adr/0023-anonymized-family-ownership-surface.md index 3ead38243..d5dcea50b 100644 --- a/docs/adr/0023-anonymized-family-ownership-surface.md +++ b/docs/adr/0023-anonymized-family-ownership-surface.md @@ -1,8 +1,9 @@ # ADR-0023: Anonymized close-relative (family) ownership surface -- Status: Accepted +- Status: Superseded by [ADR-0030](0030-family-ownership-withheld-nameless-aggregate.md) - Date: 2026-07-07 - Deciders: lb, Claude +- Superseded by: [ADR-0030](0030-family-ownership-withheld-nameless-aggregate.md) - Related: [ADR-0007](0007-scope-and-certainty-bar.md), [ADR-0010](0010-pii-posture.md), [ADR-0019](0019-private-interest-vs-ex-officio-classification.md), [ADR-0022](0022-public-surface-private-ownership-only.md); `scripts/cacbg/parse.mjs`, `scripts/cacbg/load.mjs`, `packages/db/src/queries/related-persons.ts`, `apps/web/app/routes/conflict*.tsx` ## Context From ea00781d3622fa8889f658cc87bf2ca597c4e527 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 16:54:28 +0300 Subject: [PATCH 03/50] test(seed): conflict-of-interest smoke fixture for local development MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /conflicts rendered an empty surface locally — seed.sql carried only authorities, tenders and bidders, so the свързани-лица routes had nothing to show and the feature could not be verified in a browser at all. Adds two invented officials, three invented companies and the rows the read gate actually requires (packages/db/src/queries/related-persons.ts): published status with a surfaced ownership class, a live contract for the ЕИК, and signed_at inside the declared window so the „в периода" split is exercised. One link per outcome — a self stake, a family stake, and a joint-stock link left held — so a change to the publishing rule shows up as a before/after rather than as an empty page either way. Every person and company is invented; no real declarant, no real winner. The ЕИК pass the real 9-digit checksum (normalize-raw.sql's eik_valid), so the fixture cannot teach a wrong ЕИК shape. --- scripts/seed.sql | 79 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/scripts/seed.sql b/scripts/seed.sql index 2ff25add1..9da64854b 100644 --- a/scripts/seed.sql +++ b/scripts/seed.sql @@ -16,3 +16,82 @@ INSERT OR IGNORE INTO bidders (id, name, bulstat) VALUES ('bidder-a', 'Алфа ЕООД', '111111111'), ('bidder-b', 'Бета АД', '222222222'), ('bidder-c', 'Гама ООД', '333333333'); + +-- ── свързани лица (conflict-of-interest) smoke fixture ──────────────────────────────────────────── +-- Without these the /conflicts routes render an empty surface locally, so the whole feature is +-- unverifiable in a browser. Every person and company below is INVENTED — no real declarant, no real +-- winner. The ЕИК pass the real 9-digit checksum (scripts/normalize-raw.sql's eik_valid), so the +-- fixture cannot teach a wrong ЕИК shape to anyone reading it. +-- +-- The rows are shaped by what the read gate actually requires (packages/db/src/queries/related-persons.ts): +-- • status='published' AND interest_class IN (private_ownership, family_ownership) — SURFACED_OWNERSHIP +-- • a LIVE contract for that ЕИК — the read-time zero-contract gate +-- • signed_at inside [first_declared_year, last_declared_year] — IN_WINDOW ⇒ the „в периода" chip +-- • no published SELF stake on the same (official, ЕИК) for a family row — NOT_REDUNDANT_FAMILY +-- One link per outcome, including the non-surfaced ones, so a change to the publishing rule is +-- visible as a before/after rather than as an empty page either way. + +INSERT OR IGNORE INTO authorities (id, name, bulstat, region) VALUES + ('auth-plovdiv', 'Община Пример', '131223340', 'Пловдив'); + +INSERT OR IGNORE INTO tenders + (id, source_id, title, authority_id, cpv_code, estimated_value, currency, procedure_type, status, published_at, deadline_at) +VALUES + ('t-conflict-01', 'AOP-2022-0101', 'Строителен надзор на общински обекти', 'auth-plovdiv', '71520000', 900000, 'BGN', 'открита процедура', 'closed', '2022-01-10', '2022-02-10'), + ('t-conflict-02', 'AOP-2022-0102', 'Консултантски услуги по проект', 'auth-sofia', '79400000', 400000, 'BGN', 'открита процедура', 'closed', '2022-04-05', '2022-05-05'), + ('t-conflict-03', 'AOP-2022-0103', 'Доставка на офис оборудване', 'auth-mrrb', '30190000', 250000, 'BGN', 'открита процедура', 'closed', '2022-06-01', '2022-07-01'); + +-- eik_valid=1 so these behave like resolved winners; settlement feeds the seat-confirmation rung. +INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, legal_form, settlement) VALUES + ('eik:201122335', 'АЛФА СТРОЙ ООД', '201122335', '201122335', 1, 'ООД', 'Пловдив'), + ('eik:203445566', 'БЕТА КОНСУЛТ ЕООД', '203445566', '203445566', 1, 'ЕООД', 'София'), + ('eik:204556676', 'ГАМА ИНВЕСТ АД', '204556676', '204556676', 1, 'АД', 'Варна'); + +INSERT OR IGNORE INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES + ('c-conflict-01', 't-conflict-01', 'eik:201122335', 840000, 'BGN', '2022-03-14', 'Д-2022-101', 429000), + ('c-conflict-02', 't-conflict-02', 'eik:203445566', 372000, 'BGN', '2022-06-20', 'Д-2022-102', 190000), + ('c-conflict-03', 't-conflict-03', 'eik:204556676', 236000, 'BGN', '2022-08-02', 'Д-2022-103', 120000); + +-- person id = 'person:' || key(name) || '|' || key(institution) — (name, institution), never a bare +-- name (ADR-0026), so two namesakes at different bodies stay distinct. +INSERT OR IGNORE INTO persons (id, name) VALUES + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'Иван Петров Тестов'), + ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ', 'Мария Георгиева Образцова'); + +INSERT OR IGNORE INTO declarations + (id, person_id, xml_file, control_hash, folder_year, declared_year, template, category, institution, position, source_url) +VALUES + ('decl:2023:demo-ivan.xml', 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'demo-ivan.xml', 'demo-hash-ivan', '2023', '2023', 'assets', 'Местна власт', 'Община Пример', 'Кмет', 'https://register.cacbg.bg/2023/demo-ivan.xml'), + ('decl:2023:demo-maria.xml', 'person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ', 'demo-maria.xml', 'demo-hash-maria', '2023', '2023', 'assets', 'Местна власт', 'Община София', 'Директор дирекция', 'https://register.cacbg.bg/2023/demo-maria.xml'); + +INSERT OR IGNORE INTO declared_interests (id, declaration_id, entity_raw, entity_key, kind, detail, timing, seat) VALUES + ('di:decl:2023:demo-ivan.xml:1', 'decl:2023:demo-ivan.xml', 'АЛФА СТРОЙ ООД', 'АЛФА СТРОЙ ООД', 'shares', '50%', 'annual', 'Пловдив'), + ('di:decl:2023:demo-ivan.xml:2', 'decl:2023:demo-ivan.xml', 'ГАМА ИНВЕСТ АД', 'ГАМА ИНВЕСТ АД', 'shares', '0,5%', 'annual', 'Варна'), + ('di:decl:2023:demo-maria.xml:1', 'decl:2023:demo-maria.xml', 'БЕТА КОНСУЛТ ЕООД', 'БЕТА КОНСУЛТ ЕООД', 'shares', '100%', 'annual', 'София'); + +-- Three links, three outcomes: +-- published private_ownership — the official's own stake, own_institution='exact' (the strongest signal) +-- published family_ownership — a relative's declared stake; the relative is never named (ADR-0032) +-- held (АД, C_hold) — a joint-stock parcel: collected, never surfaced (ADR-0022 materiality) +INSERT OR IGNORE INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, + relation, interest_class, contemporaneous, own_institution, evidence_count, + first_declared_year, last_declared_year, contract_count, contract_value_eur, + first_contract_year, last_contract_year, status) +VALUES + ('il:person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', + 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'eik:201122335', '201122335', 'АЛФА СТРОЙ ООД', + 'exact_name_key', 'seed-demo', 'B_distinctive', 'owns', 'private_ownership', 1, 'exact', 1, + '2021', '2023', 1, 429000, '2022', '2022', 'published'), + ('il:person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', + 'person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ', 'eik:203445566', '203445566', 'БЕТА КОНСУЛТ ЕООД', + 'exact_name_key', 'seed-demo', 'A_seat', 'related', 'family_ownership', 1, 'exact', 1, + '2021', '2023', 1, 190000, '2022', '2022', 'published'), + ('il:person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|204556676', 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|204556676', + 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'eik:204556676', '204556676', 'ГАМА ИНВЕСТ АД', + 'exact_name_key', 'seed-demo', 'C_hold', 'owns', 'management_role', 0, 'none', 1, + '2021', '2023', 1, 120000, '2022', '2022', 'held'); + +INSERT OR IGNORE INTO interest_link_authorities (link_key, authority_id, authority_name, contract_count, value_eur, own) VALUES + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'auth-plovdiv', 'Община Пример', 1, 429000, 'exact'), + ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'auth-sofia', 'Община София', 1, 190000, 'exact'); From b36403177c6542af9286cbf888dae998283f76fc Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 17:21:09 +0300 Subject: [PATCH 04/50] =?UTF-8?q?feat(tr):=20registry=20client,=20deed=20c?= =?UTF-8?q?ache=20and=20the=20=D0=95=D0=98=D0=9A=20checksum=20as=20a=20Nod?= =?UTF-8?q?e=20capability?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundations for the Trade Register evidence leg (#279, ADR-0033). No crawling and no publishing decision yet — this is the transport, the cache and the identifier rule, each with the refusals that make the later steps safe. scripts/tr/eik.mjs — the ЕИК checksum, which until now was callable only from SQL (normalize-raw.sql's eik_valid). The crawler has to decide in Node whether a code is worth a lookup. A test lifts the CASE expression straight out of the .sql file and runs both implementations over ~120 values through node:sqlite, so the twin cannot drift from the rule it mirrors. scripts/tr/client.mjs — 429 is never retried. The register's block is sustained (an earlier spike saw 429 at ~50 requests and then 429 to everything, with no Retry-After and no quota header), so it is an instruction to stop, not a transient. 5xx and network faults retry with growing backoff; a 429 arriving mid-retry aborts rather than letting a later 200 mask the block. A foreign host is refused before any packet is sent. Two transport decisions worth stating: this leg does NOT reuse cacbg/tls.mjs — that module pins register.cacbg.bg's leaf because that host serves a broken chain (ADR-0011), whereas this host verifies against system roots, which is stronger. And it uses node:https, not fetch: measured live, the identical request returns 500/empty via undici and 200 with the full 34,398-byte deed via node:https. scripts/tr/cache.mjs — resumability plus the PII rail. The index holds no name at all: ЕИК, dates, codes, verdicts and a body hash, never an excerpt. Any value carrying a ten-digit run is refused outright — the ЕГН shape, and sound as a check precisely because an ЕИК is 9 or 13 digits, never 10. scripts/tr/*.test.mjs matched NO CI job before this commit; the workflow globbed only scripts/ and scripts/cacbg/. Fixed here rather than later, or every test above would have been decorative. cacbg/guard.mjs's assertScratchIgnored takes the subdirectory instead of being copied — a duplicated safety rail drifts from its original. Existing callers are unchanged. --- .github/workflows/scripts-test.yml | 13 +- scripts/cacbg/guard.mjs | 11 +- scripts/tr/cache.mjs | 187 +++++++++++++++++++++++++++++ scripts/tr/cache.test.mjs | 129 ++++++++++++++++++++ scripts/tr/client.mjs | 129 ++++++++++++++++++++ scripts/tr/client.test.mjs | 152 +++++++++++++++++++++++ scripts/tr/eik.mjs | 61 ++++++++++ scripts/tr/eik.test.mjs | 135 +++++++++++++++++++++ scripts/tr/paths.mjs | 54 +++++++++ scripts/tr/paths.test.mjs | 64 ++++++++++ 10 files changed, 928 insertions(+), 7 deletions(-) create mode 100644 scripts/tr/cache.mjs create mode 100644 scripts/tr/cache.test.mjs create mode 100644 scripts/tr/client.mjs create mode 100644 scripts/tr/client.test.mjs create mode 100644 scripts/tr/eik.mjs create mode 100644 scripts/tr/eik.test.mjs create mode 100644 scripts/tr/paths.mjs create mode 100644 scripts/tr/paths.test.mjs diff --git a/.github/workflows/scripts-test.yml b/.github/workflows/scripts-test.yml index 7ee3ef4c1..d3112c48f 100644 --- a/.github/workflows/scripts-test.yml +++ b/.github/workflows/scripts-test.yml @@ -28,10 +28,11 @@ jobs: # so no `pnpm install` is required. - run: node --test scripts/*.test.mjs - # свързани-лица CACBG pipeline tests (parse/classify/load/audit/tr-census/extract). These use - # node:sqlite (DatabaseSync) and import the shared companyNameKey .ts via the register-ts resolve - # hook, so they need Node 24 (node:sqlite + native TS type-stripping, no experimental flag) rather - # than the Node 22 above. The libel-critical resolution logic gates merges here. + # свързани-лица pipeline tests — the CACBG leg (parse/classify/load/audit/tr-census/extract) and the + # Търговски регистър leg (scripts/tr: ЕИК checksum, HTTP client, deed cache). These use node:sqlite + # (DatabaseSync) and import the shared companyNameKey .ts via the register-ts resolve hook, so they + # need Node 24 (node:sqlite + native TS type-stripping, no experimental flag) rather than the Node 22 + # above. The libel-critical resolution logic gates merges here. cacbg: runs-on: ubuntu-latest timeout-minutes: 10 @@ -47,4 +48,6 @@ jobs: # parse.mjs imports fast-xml-parser (a workspace dep), so this job needs node_modules — unlike # the plain-node scripts above. Install before the scraper tests run. - run: pnpm install --frozen-lockfile - - run: node --import ./scripts/cacbg/register-ts.mjs --test scripts/cacbg/*.test.mjs + # Both legs, or the glob silently excludes a whole module: scripts/tr/*.test.mjs matched NO job + # until this line existed, so every test in it would have been decorative. + - run: node --import ./scripts/cacbg/register-ts.mjs --test scripts/cacbg/*.test.mjs scripts/tr/*.test.mjs diff --git a/scripts/cacbg/guard.mjs b/scripts/cacbg/guard.mjs index 97a326bda..9502ac963 100644 --- a/scripts/cacbg/guard.mjs +++ b/scripts/cacbg/guard.mjs @@ -9,8 +9,15 @@ import { fileURLToPath } from 'node:url'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); export const SCRATCH = path.join(ROOT, 'scratch', 'cacbg'); -export function assertScratchIgnored() { - const probe = path.join('scratch', 'cacbg', '.probe'); +/** + * Assert that a `scratch/` tree is git-ignored, before anything writes PII into it. + * Parameterised rather than copied: the Trade Register leg needs the identical rail for its deed + * cache (owner names, company addresses — ADR-0033 decision 5), and a second copy of a safety rail + * drifts from the original. Existing no-argument callers are unaffected. + * @param {string} [subdir] directory under scratch/ to probe + */ +export function assertScratchIgnored(subdir = 'cacbg') { + const probe = path.join('scratch', subdir, '.probe'); try { execFileSync('git', ['check-ignore', '-q', probe], { cwd: ROOT }); } catch { diff --git a/scripts/tr/cache.mjs b/scripts/tr/cache.mjs new file mode 100644 index 000000000..673c6b477 --- /dev/null +++ b/scripts/tr/cache.mjs @@ -0,0 +1,187 @@ +// The deed cache — resumability, and the PII rail (issue #279, ADR-0033 decision 5). +// +// A registry deed contains third-party personal data: the names of owners and managers who hold no +// public office, and the company's street address. Two rails follow from that, and both live here: +// +// 1. The INDEX stores no name at all — ЕИК, dates, codes, verdicts, and a hash of the body. Names +// exist only in the raw JSON under git-ignored scratch/, are read only to produce a boolean, and +// never enter a public table, a response or a log. A hash rather than an excerpt, because an +// excerpt of a deed IS third-party personal data. +// 2. Nothing may carry a ten-digit run. That is the ЕГН shape, and the check is sound precisely +// because an ЕИК is 9 or 13 digits — never 10 — so it cannot reject a legitimate identifier. +// ЕГН was absent from every payload examined; this is the rail for the day one leaks. +// +// Resumability is the other job: the register throttles hard and a 429 ends the run (client.mjs), so a +// crawl must be able to pick up exactly where it stopped without re-requesting what it already has. + +import { DatabaseSync } from 'node:sqlite'; +import fs from 'node:fs'; +import path from 'node:path'; +import { safeEik } from './paths.mjs'; + +const SCHEMA = ` +CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, + status TEXT NOT NULL, -- fetched | outside_tr + http_status INTEGER, + fetched_at TEXT NOT NULL, + raw_path TEXT, -- relative to the raw dir; the ONLY place names live + body_sha256 TEXT, -- integrity + change detection, never an excerpt + legal_form_code INTEGER, + legal_form_verdict TEXT, -- closely_held | joint_stock | unknown (unknown WITHHOLDS) + seat_normalized TEXT, -- settlement only; never a street address (ADR-0010 item 3) + seat_entry_date TEXT, + latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, + outside_reason TEXT +); +CREATE INDEX IF NOT EXISTS idx_deeds_status ON deeds(status); +CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); +`; + +/** Open (creating if absent) the cache at `file`. Idempotent — never wipes an existing cache. */ +export function openCache(file) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const db = new DatabaseSync(file); + db.exec('PRAGMA journal_mode = WAL'); + db.exec(SCHEMA); + return db; +} + +// ── the ЕГН rail ────────────────────────────────────────────────────────────── +const TEN_DIGITS = /\d{10}/; +/** + * Refuse any text destined for the index that carries a ten-digit run. + * An ЕИК is 9 or 13 digits, so a 10-digit run is never a legitimate identifier here — it is an ЕГН + * shape, and storing one would breach ADR-0010 decision 2 outright. + */ +function assertNoEgnShape(value, field) { + if (value != null && TEN_DIGITS.test(String(value))) { + throw new Error( + `REFUSE TO STORE: ${field} carries a ten-digit run (ЕГН shape) — the index holds no personal data`, + ); + } +} + +const TEXT_FIELDS = ['rawPath', 'legalFormVerdict', 'seatNormalized', 'outsideReason']; + +/** + * Record a fetched deed. Replaces on re-fetch so a refresh never duplicates a row. + * @param {import('node:sqlite').DatabaseSync} db + * @param {object} d + */ +export function upsertDeed(db, d) { + const eik = safeEik(d.eik); + for (const f of TEXT_FIELDS) assertNoEgnShape(d[f], f); + db.prepare( + `INSERT INTO deeds (eik, status, http_status, fetched_at, raw_path, body_sha256, + legal_form_code, legal_form_verdict, seat_normalized, seat_entry_date, + latest_own_entry_date, attempts, outside_reason) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(eik) DO UPDATE SET + status=excluded.status, http_status=excluded.http_status, fetched_at=excluded.fetched_at, + raw_path=excluded.raw_path, body_sha256=excluded.body_sha256, + legal_form_code=excluded.legal_form_code, legal_form_verdict=excluded.legal_form_verdict, + seat_normalized=excluded.seat_normalized, seat_entry_date=excluded.seat_entry_date, + latest_own_entry_date=excluded.latest_own_entry_date, + attempts=deeds.attempts + 1, outside_reason=excluded.outside_reason`, + ).run( + eik, + d.status ?? 'fetched', + d.httpStatus ?? null, + d.fetchedAt, + d.rawPath ?? null, + d.bodySha256 ?? null, + d.legalFormCode ?? null, + d.legalFormVerdict ?? null, + d.seatNormalized ?? null, + d.seatEntryDate ?? null, + d.latestOwnEntryDate ?? null, + d.attempts ?? 1, + d.outsideReason ?? null, + ); +} + +/** + * Mark an ЕИК as not present in the Trade Register (ДЗЗД, БУЛСТАT associations). + * + * PERMANENT BY INTENT, so the caller must only reach here on a DOCUMENTED positive response. A 429, + * a 5xx or a timeout is transient and must never be cached as a negative — that is how a temporary + * wall becomes permanent data (R6). + */ +export function markOutsideTr(db, eik, reason, now = new Date()) { + assertNoEgnShape(reason, 'outsideReason'); + upsertDeed(db, { + eik, + status: 'outside_tr', + fetchedAt: now.toISOString(), + outsideReason: reason, + }); +} + +/** One row by ЕИК, camelCased, or null. */ +export function readDeed(db, eik) { + const r = db.prepare('SELECT * FROM deeds WHERE eik = ?').get(safeEik(eik)); + if (!r) return null; + return { + eik: r.eik, + status: r.status, + httpStatus: r.http_status, + fetchedAt: r.fetched_at, + rawPath: r.raw_path, + bodySha256: r.body_sha256, + legalFormCode: r.legal_form_code, + legalFormVerdict: r.legal_form_verdict, + seatNormalized: r.seat_normalized, + seatEntryDate: r.seat_entry_date, + latestOwnEntryDate: r.latest_own_entry_date, + attempts: r.attempts, + outsideReason: r.outside_reason, + }; +} + +/** + * Which of `wanted` still need a request — the resumability primitive. + * A cached row past `maxAgeDays` becomes pending again (the 35-day retention of ADR-0033 decision 5); + * pass no `maxAgeDays` to treat any cached row as current. + */ +export function pendingEiks(db, wanted, { maxAgeDays = null, now = new Date() } = {}) { + const cutoff = maxAgeDays == null ? null : now.getTime() - maxAgeDays * 86_400_000; + const out = []; + const stmt = db.prepare('SELECT fetched_at FROM deeds WHERE eik = ?'); + for (const raw of wanted) { + const row = stmt.get(safeEik(raw)); + if (!row) { + out.push(String(raw)); + continue; + } + if (cutoff != null && Date.parse(row.fetched_at) < cutoff) out.push(String(raw)); + } + return out; +} + +/** + * How much of `wanted` the cache actually covers — the input to the fail-closed load gate. + * `outside_tr` counts as COVERED: it is a known, resolved outcome, not a gap. A partial cache must + * make the loader throw rather than publish a decimated surface (ADR-0033 decision 7). + */ +export function coverage(db, wanted) { + let fetched = 0, + outsideTr = 0; + const stmt = db.prepare('SELECT status FROM deeds WHERE eik = ?'); + for (const raw of wanted) { + const row = stmt.get(safeEik(raw)); + if (!row) continue; + if (row.status === 'outside_tr') outsideTr++; + else fetched++; + } + const covered = fetched + outsideTr; + return { + wanted: wanted.length, + fetched, + outsideTr, + covered, + missing: wanted.length - covered, + ratio: wanted.length === 0 ? 1 : covered / wanted.length, + }; +} diff --git a/scripts/tr/cache.test.mjs b/scripts/tr/cache.test.mjs new file mode 100644 index 000000000..e757593d5 --- /dev/null +++ b/scripts/tr/cache.test.mjs @@ -0,0 +1,129 @@ +// node:test — the deed cache. Its job is to make the crawl resumable and to hold the PII rail. +// +// The rail (ADR-0033 decision 5): the INDEX stores no name at all — only ЕИК, dates, codes, verdicts +// and a body hash. Names exist solely in the raw JSON under git-ignored scratch/, are read only to +// produce a boolean, and never enter a public table, a response or a log. The ten-digit refusal below +// is the ЕГН guard, and it is sound precisely because an ЕИК is 9 or 13 digits — never 10. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { openCache, upsertDeed, markOutsideTr, pendingEiks, readDeed, coverage } from './cache.mjs'; + +function tmpDb() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-cache-')); + return { dir, file: path.join(dir, 'tr-cache.sqlite') }; +} +const withCache = (fn) => { + const { dir, file } = tmpDb(); + const db = openCache(file); + try { + return fn(db); + } finally { + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } +}; + +const deed = (over = {}) => ({ + eik: '115536179', + httpStatus: 200, + fetchedAt: '2026-08-05T10:00:00Z', + rawPath: 'deeds/115536179.json', + bodySha256: 'a'.repeat(64), + legalFormCode: 4, + legalFormVerdict: 'closely_held', + seatNormalized: 'ПЛОВДИВ', + seatEntryDate: '2014-01-23', + latestOwnEntryDate: '2013-07-16', + ...over, +}); + +test('openCache is idempotent — re-opening an existing cache preserves rows', () => { + const { dir, file } = tmpDb(); + let db = openCache(file); + upsertDeed(db, deed()); + db.close(); + db = openCache(file); // must not wipe + assert.equal(readDeed(db, '115536179')?.eik, '115536179'); + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); +}); + +test('upsertDeed replaces on re-fetch rather than duplicating', () => + withCache((db) => { + upsertDeed(db, deed()); + upsertDeed(db, deed({ seatNormalized: 'СОФИЯ', fetchedAt: '2026-09-01T10:00:00Z' })); + assert.equal(coverage(db, ['115536179']).fetched, 1); + assert.equal(readDeed(db, '115536179').seatNormalized, 'СОФИЯ'); + })); + +test('pendingEiks returns only what is not yet cached — this is what makes a run resumable', () => + withCache((db) => { + upsertDeed(db, deed({ eik: '115536179' })); + markOutsideTr(db, '204556676', 'BULSTAT association'); + const want = ['115536179', '204556676', '201122335', '203445566']; + assert.deepEqual(pendingEiks(db, want).sort(), ['201122335', '203445566']); + })); + +test('a stale deed becomes pending again past the TTL, a fresh one does not', () => + withCache((db) => { + upsertDeed(db, deed({ fetchedAt: '2026-01-01T00:00:00Z' })); // long past + upsertDeed(db, deed({ eik: '201122335', fetchedAt: '2026-08-05T00:00:00Z' })); + const now = new Date('2026-08-05T12:00:00Z'); + assert.deepEqual(pendingEiks(db, ['115536179', '201122335'], { maxAgeDays: 35, now }), [ + '115536179', + ]); + })); + +test('coverage reports the fraction cached — the input to the fail-closed load gate', () => + withCache((db) => { + upsertDeed(db, deed({ eik: '115536179' })); + upsertDeed(db, deed({ eik: '201122335' })); + markOutsideTr(db, '204556676', 'ДЗЗД'); + const c = coverage(db, ['115536179', '201122335', '204556676', '203445566']); + assert.equal(c.wanted, 4); + assert.equal(c.fetched, 2); + assert.equal(c.outsideTr, 1); + assert.equal(c.missing, 1); + // „outside ТР" is a RESOLVED outcome, not a gap: it is known and permanent, so it counts as covered. + assert.equal(c.covered, 3); + })); + +// ── the PII rail ────────────────────────────────────────────────────────────── +test('the index REFUSES a value carrying a ten-digit run (the ЕГН guard)', () => + withCache((db) => { + // Sound because an ЕИК is 9 or 13 digits, never 10 — so this can never reject a legitimate code. + assert.throws( + () => upsertDeed(db, deed({ seatNormalized: 'СОФИЯ 8001014567' })), + /ten-digit|ЕГН/i, + ); + assert.throws(() => markOutsideTr(db, '204556676', 'подадено от 8001014567'), /ten-digit|ЕГН/i); + })); + +test('valid 9- and 13-digit codes are NOT caught by the ЕГН guard', () => + withCache((db) => { + assert.doesNotThrow(() => upsertDeed(db, deed({ outsideReason: null }))); + assert.doesNotThrow(() => markOutsideTr(db, '1155361790001', 'клон')); + })); + +test('the schema exposes no column that could hold a person name', () => + withCache((db) => { + const cols = db + .prepare(`SELECT name FROM pragma_table_info('deeds')`) + .all() + .map((r) => r.name); + for (const forbidden of ['name', 'person', 'owner', 'manager', 'holder', 'full_name']) + assert.ok(!cols.includes(forbidden), `deeds.${forbidden} must not exist (PII rail)`); + // A hash, never an excerpt — an excerpt of a deed is third-party personal data. + assert.ok(cols.includes('body_sha256')); + })); + +test('markOutsideTr is permanent-by-intent and records WHY', () => + withCache((db) => { + markOutsideTr(db, '204556676', 'ДЗЗД — BULSTAT, not TR'); + const row = readDeed(db, '204556676'); + assert.equal(row.status, 'outside_tr'); + assert.match(row.outsideReason, /ДЗЗД/); + })); diff --git a/scripts/tr/client.mjs b/scripts/tr/client.mjs new file mode 100644 index 000000000..23d3769ec --- /dev/null +++ b/scripts/tr/client.mjs @@ -0,0 +1,129 @@ +// HTTP client for the Търговски регистър public API (issue #279, ADR-0033). +// +// TLS: PLAIN HTTPS, verified against the system roots at Node's secure default. Do NOT reach for +// scripts/cacbg/tls.mjs here. That module exists because register.cacbg.bg serves an incomplete chain, +// so we pin ITS leaf key (ADR-0011) — `getPinned` hard-refuses every other host by design. This host +// chains correctly, so ordinary verification is available and is strictly stronger than a pin we would +// have to hand-maintain. Copying the pinning across "for consistency" would weaken this leg. +// +// Rate limiting: the register throttles, and when it does the block is SUSTAINED. An earlier spike saw +// HTTP 429 at roughly 50 cumulative requests ending in a burst, and thereafter 429 to every subsequent +// request — including simple /Deeds/{eik} calls that had worked seconds before. There is no +// `Retry-After` and no `X-RateLimit-*` header, so a client cannot pace against a published budget; it +// can only avoid tripping one. That makes a 429 an instruction to STOP, not a transient to retry +// through: `politeTrGet` retries 5xx and network faults with growing backoff, and never retries a 429. +// (#279 §3's "5 retries with growing backoff" and the decision that a 429 ends the run are consistent +// only if the retry set excludes 429.) The limiter is the operator's only way to express a rate +// preference, and tuning around it empirically is what spec §3.3's "NEVER bulk-scrape" forbids. + +import https from 'node:https'; +import { safeEik } from './paths.mjs'; + +export const TR_HOST = 'portal.registryagency.bg'; + +// Identify the crawler honestly. The operator's only lever on us is the rate limiter, so at minimum +// they should be able to see who is calling and where to complain. +export const TR_USER_AGENT = + 'sigma-bot/1.0 (+https://github.com/midt-bg/sigma) contact: via repo issues'; + +/** Thrown on HTTP 429. Distinguishable so the crawler can end the run instead of marking anything. */ +export class RateLimitError extends Error { + constructor(url) { + super(`REFUSE TO CONTINUE: ${TR_HOST} returned 429 for ${url} — the block is sustained; stop`); + this.name = 'RateLimitError'; + this.url = url; + } +} + +/** Refuse any URL that is not an https request to the register. Call before touching the network. */ +export function assertTrHost(url) { + let u; + try { + u = new URL(String(url)); + } catch { + throw new Error(`bad URL: ${url}`); + } + if (u.protocol !== 'https:' || u.hostname !== TR_HOST) { + throw new Error(`refusing non-${TR_HOST} host: ${url}`); + } + return url; +} + +/** The public deed endpoint for one ЕИК. Public JSON, no authentication. */ +export function deedUrl(eik) { + return `https://${TR_HOST}/CR/api/Deeds/${safeEik(eik)}`; +} + +const wait = (ms) => new Promise((r) => setTimeout(r, ms)); + +/** + * Default transport: `node:https`, system roots, secure defaults. + * + * NOT `fetch`. Measured against the live endpoint on 2026-08-05, the identical request returns + * **HTTP 500 with an empty body via undici's fetch and HTTP 200 with the full 34,398-byte deed via + * node:https** — same URL, same accept header, same user-agent. The server rejects something undici + * adds to the wire (encoding negotiation / connection handling); it is not an auth or rate-limit + * problem, and retrying only multiplies the failure. `scripts/cacbg/tls.mjs` is on node:https too, so + * both crawl legs now share one transport primitive. + * + * `rejectUnauthorized` is left at its secure default and named here on purpose: the sibling CACBG + * module deliberately does NOT verify against system roots (it pins a leaf instead, ADR-0011), so a + * reader comparing the two needs to see that this leg takes the ordinary, stronger path. + */ +export function httpsGet(url, { timeoutMs = 20_000 } = {}) { + return new Promise((resolve, reject) => { + const req = https.get( + url, + { headers: { accept: 'application/json', 'user-agent': TR_USER_AGENT } }, + (res) => { + const chunks = []; + res.on('data', (c) => chunks.push(c)); + res.on('end', () => + resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) }), + ); + res.on('error', reject); + }, + ); + req.setTimeout(timeoutMs, () => req.destroy(new Error(`timeout after ${timeoutMs}ms: ${url}`))); + req.on('error', reject); + }); +} + +/** + * GET with polite retries. 5xx and network faults retry with growing backoff; 2xx/4xx return as-is; + * **429 throws `RateLimitError` immediately and is never retried**. + * @param {string} url + * @param {{httpGet?:Function, sleep?:Function, tries?:number, backoffMs?:number}} [opts] + * `httpGet` and `sleep` are the injection seam — the whole retry policy is testable offline. + */ +export async function politeTrGet(url, opts = {}) { + const { httpGet = httpsGet, sleep = wait, tries = 5, backoffMs = 1000 } = opts; + assertTrHost(url); // before any request — a foreign host must cost zero packets + let backoff = backoffMs; + for (let attempt = 1; ; attempt++) { + let res; + try { + res = await httpGet(url); + } catch (err) { + if (attempt >= tries) throw err; + await sleep(backoff); + backoff *= 2; + continue; + } + // Checked before the retry branch: a 429 arriving mid-retry must abort the whole call, not be + // folded into the 5xx budget and not let a later 200 mask the block. + if (res.status === 429) throw new RateLimitError(url); + if (res.status >= 500) { + if (attempt >= tries) return res; // give the caller the last response to record + await sleep(backoff); + backoff *= 2; + continue; + } + return res; + } +} + +/** One deed by ЕИК. Thin wrapper so callers never build the URL themselves. */ +export function trGet(eik, opts) { + return politeTrGet(deedUrl(eik), opts); +} diff --git a/scripts/tr/client.test.mjs b/scripts/tr/client.test.mjs new file mode 100644 index 000000000..6694076fc --- /dev/null +++ b/scripts/tr/client.test.mjs @@ -0,0 +1,152 @@ +// node:test — the Trade Register HTTP client. Offline: every test drives an injected getter. +// +// The load-bearing behaviour here is what the client REFUSES to do. The register rate-limits, and +// when it does the block is sustained (an earlier spike saw HTTP 429 at ~50 cumulative requests, then +// 429 to every subsequent call including ones that had just worked — no Retry-After, no quota header). +// So a 429 is an instruction to stop, not a transient to retry through. ADR-0033 decision 7. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + TR_HOST, + TR_USER_AGENT, + deedUrl, + RateLimitError, + politeTrGet, + assertTrHost, + httpsGet, +} from './client.mjs'; + +const okRes = (body = '{}') => ({ status: 200, headers: {}, body: Buffer.from(body) }); + +test('deedUrl builds the documented endpoint and refuses a non-ЕИК', () => { + assert.equal(deedUrl('115536179'), `https://${TR_HOST}/CR/api/Deeds/115536179`); + assert.equal(deedUrl('000696327'), `https://${TR_HOST}/CR/api/Deeds/000696327`); + for (const bad of ['', null, '../../etc', '115536179?x=1', '11553617x', 'ЕИК 115536179']) + assert.throws(() => deedUrl(bad), /unsafe|ЕИК/i, String(bad)); +}); + +test('assertTrHost refuses every host but the register', () => { + assert.doesNotThrow(() => assertTrHost(`https://${TR_HOST}/CR/api/Deeds/115536179`)); + for (const bad of [ + 'https://evil.example/CR/api/Deeds/115536179', + `http://${TR_HOST}/CR/api/Deeds/115536179`, // plaintext — never + `https://${TR_HOST}.evil.example/x`, // suffix trick + `https://register.cacbg.bg/x`, // the OTHER register: different host, different TLS posture + ]) { + assert.throws(() => assertTrHost(bad), /host/i, bad); + } +}); + +test('a 429 throws RateLimitError and is NEVER retried', async () => { + let calls = 0; + const httpGet = async () => { + calls++; + return { status: 429, headers: {}, body: Buffer.from('') }; + }; + await assert.rejects( + () => politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {}, tries: 5 }), + RateLimitError, + ); + // The whole point: „5 retries with growing backoff" and „429 stops the run" are only consistent + // if the retry set EXCLUDES 429. One call, not five. + assert.equal(calls, 1, 'a 429 must not be retried'); +}); + +test('5xx IS retried with growing backoff, up to the try budget', async () => { + let calls = 0; + const waits = []; + const httpGet = async () => { + calls++; + return { status: 503, headers: {}, body: Buffer.from('') }; + }; + const res = await politeTrGet(deedUrl('115536179'), { + httpGet, + sleep: async (ms) => void waits.push(ms), + tries: 4, + }); + assert.equal(calls, 4); + assert.equal(res.status, 503, 'the last response is returned, not thrown'); + assert.deepEqual( + waits.map((w, i) => (i === 0 ? true : w > waits[i - 1])), + [true, true, true], + `backoff must grow: ${waits.join(',')}`, + ); +}); + +test('a network throw is retried, then rethrown when the budget is spent', async () => { + let calls = 0; + const httpGet = async () => { + calls++; + throw new Error('ECONNRESET'); + }; + await assert.rejects( + () => politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {}, tries: 3 }), + /ECONNRESET/, + ); + assert.equal(calls, 3); +}); + +test('a 429 arriving mid-retry aborts immediately instead of finishing the budget', async () => { + const seq = [503, 503, 429, 200]; + let calls = 0; + const httpGet = async () => { + const status = seq[calls++]; + return { status, headers: {}, body: Buffer.from('') }; + }; + await assert.rejects( + () => politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {}, tries: 5 }), + RateLimitError, + ); + assert.equal(calls, 3, 'must stop AT the 429, not carry on to the 200 behind it'); +}); + +test('200 and 404 return without any retry', async () => { + for (const status of [200, 404]) { + let calls = 0; + const httpGet = async () => { + calls++; + return status === 200 ? okRes() : { status, headers: {}, body: Buffer.from('') }; + }; + const res = await politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {} }); + assert.equal(res.status, status); + assert.equal(calls, 1); + } +}); + +test('politeTrGet refuses a foreign host before making any request', async () => { + let calls = 0; + const httpGet = async () => { + calls++; + return okRes(); + }; + await assert.rejects( + () => politeTrGet('https://evil.example/CR/api/Deeds/1', { httpGet }), + /host/i, + ); + assert.equal(calls, 0, 'the refusal must happen before the network call'); +}); + +test('the crawler identifies itself and points at somewhere to complain', () => { + // The rate limiter is the operator's only lever on us. Crawling anonymously takes away their + // ability to distinguish us from an abusive client, or to ask us to stop. + assert.match(TR_USER_AGENT, /sigma/i); + assert.match(TR_USER_AGENT, /https:\/\/github\.com\//); +}); + +test('the default transport is node:https, not fetch', () => { + // Measured 2026-08-05: the identical live request returns 500/empty via undici fetch and 200 with + // the full deed via node:https. Pinning the choice so a future "modernise to fetch" tidy-up has to + // confront it rather than silently break every lookup. + assert.equal(typeof httpsGet, 'function'); + assert.match(httpsGet.toString(), /https\.get/); +}); + +test('RateLimitError carries the url and is distinguishable from a generic failure', async () => { + const httpGet = async () => ({ status: 429, headers: {}, body: Buffer.from('') }); + const err = await politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {} }).catch( + (e) => e, + ); + assert.ok(err instanceof RateLimitError); + assert.ok(err instanceof Error); + assert.match(err.message, /115536179/); +}); diff --git a/scripts/tr/eik.mjs b/scripts/tr/eik.mjs new file mode 100644 index 000000000..5a4d5b125 --- /dev/null +++ b/scripts/tr/eik.mjs @@ -0,0 +1,61 @@ +// ЕИК validity — the Node twin of the rule that already lives in SQL. +// +// `eik_valid` in scripts/normalize-raw.sql decides which bidders get an ЕИК-keyed identity at all, so +// it defines the ЕИК space the whole matcher works in. Until now that rule was callable ONLY from +// SQL, which is why the registry leg needs this: the crawler must decide, in Node, whether a code is +// worth a lookup. The two implementations are pinned against each other by a test that lifts the CASE +// expression straight out of the .sql file and runs both over the same values (eik.test.mjs) — a copy +// of the rule would drift from the thing it is copying. +// +// The rule (ЗТРРЮЛНЦ / БУЛСТАТ): +// 9-digit — weight digits 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by 3..10; +// a second 10 becomes 0. Digit 9 must equal the control. +// 13-digit — the leading 9 must themselves be a valid 9-digit ЕИК, then weight digits 9..12 by +// 2,7,3,5 (fallback 4,9,5,7; a second 10 becomes 0). Digit 13 must equal that control. +// +// Everything here is string-in, string-out. An ЕИК is an identifier, not a number: public bodies carry +// codes of exactly the `000…` shape, and a numeric round-trip drops the leading zeros and silently +// turns one company's identifier into another's. + +/** Weighted control digit over `digits`, with the standard second-pass fallback. @returns {number} */ +function control(digits, primary, fallback) { + const sum = (ws) => ws.reduce((acc, w, i) => acc + w * digits[i], 0) % 11; + const first = sum(primary); + if (first < 10) return first; + const second = sum(fallback); + return second < 10 ? second : 0; +} + +/** + * Is this a structurally valid ЕИК (9 or 13 digits, correct control digit)? + * Service codes (`000000000`, `0000000000000`) are rejected outright, matching the SQL: they pass the + * arithmetic but are placeholders, and letting them through collapsed unrelated foreign suppliers onto + * one node (#195). + * @param {unknown} eik @returns {boolean} + */ +export function eikChecksumValid(eik) { + const s = String(eik ?? ''); + if (s === '000000000' || s === '0000000000000') return false; + if (!/^\d+$/.test(s)) return false; + if (s.length !== 9 && s.length !== 13) return false; + + const d = [...s].map(Number); + const c9 = control(d.slice(0, 8), [1, 2, 3, 4, 5, 6, 7, 8], [3, 4, 5, 6, 7, 8, 9, 10]); + if (c9 !== d[8]) return false; + if (s.length === 9) return true; + + const c13 = control(d.slice(8, 12), [2, 7, 3, 5], [4, 9, 5, 7]); + return c13 === d[12]; +} + +/** + * Normalise a raw ЕИК string to digits only, or null when it is not one. + * Mirrors the SQL's `eik_clean`: strips a leading „ЕИК " label and surrounding whitespace, and does + * NOT otherwise repair the value. Validity is a separate question — `eikChecksumValid`. + * @param {unknown} raw @returns {string|null} + */ +export function normalizeEik(raw) { + const s = String(raw ?? '').trim(); + const stripped = (s.startsWith('ЕИК ') ? s.slice(4) : s).trim(); + return /^\d{9}$|^\d{13}$/.test(stripped) ? stripped : null; +} diff --git a/scripts/tr/eik.test.mjs b/scripts/tr/eik.test.mjs new file mode 100644 index 000000000..7a80a191e --- /dev/null +++ b/scripts/tr/eik.test.mjs @@ -0,0 +1,135 @@ +// node:test — the ЕИК checksum, and its PARITY with the SQL that already owns this rule. +// +// Why parity matters: `eik_valid` in scripts/normalize-raw.sql decides which bidders get an ЕИК-keyed +// identity at all, so it defines the ЕИК space the whole matcher works in. A JS twin that disagrees +// would silently accept a code the pipeline rejects (or vice versa) and the registry lookup would be +// made against an entity the rest of the system does not believe exists. The parity test below runs +// both implementations over the same values through node:sqlite, so they cannot drift apart. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { DatabaseSync } from 'node:sqlite'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { eikChecksumValid, normalizeEik } from './eik.mjs'; + +// Real ЕИК, verifiable against the public registers — the only external anchor this rule has. +const REAL = [ + '000696327', // Община София + '831661388', // Министерство на регионалното развитие + '115536179', // „ПИМК" ООД + '207695026', // „Профит Екстра" ЕООД +]; + +test('accepts real 9-digit ЕИК', () => { + for (const e of REAL) assert.equal(eikChecksumValid(e), true, e); +}); + +test('rejects a wrong-digit twin of every real ЕИК', () => { + // The failure this rule exists to stop (#195): a typo twin passing as a distinct real company and + // collapsing unrelated suppliers onto one node. Mutating the LAST digit must always break it. + for (const e of REAL) { + const bad = e.slice(0, 8) + ((Number(e[8]) + 1) % 10); + assert.equal(eikChecksumValid(bad), false, `${bad} (twin of ${e})`); + } +}); + +test('rejects service codes, wrong lengths and non-digits', () => { + for (const bad of [ + '000000000', + '0000000000000', + '', + null, + undefined, + '12345678', // 8 + '1234567890', // 10 — never a valid ЕИК length, which is what makes the ЕГН guard sound + '11553617x', + '115 536 179', + ]) { + assert.equal(eikChecksumValid(bad), false, String(bad)); + } +}); + +test('13-digit branch requires BOTH the 9-digit prefix and the 13th control digit', () => { + // A клон/поделение code: the leading 9 must themselves be a valid ЕИК, then weights 2,7,3,5. + const base = '115536179'; + const valid13 = thirteen(base, '001'); + assert.equal(valid13.length, 13, valid13); + assert.equal(eikChecksumValid(valid13), true, valid13); + // break the 13th digit + const broken = valid13.slice(0, 12) + ((Number(valid13[12]) + 1) % 10); + assert.equal(eikChecksumValid(broken), false, broken); + // a valid 13th control over an INVALID 9-prefix must still fail — both halves are load-bearing + assert.equal(eikChecksumValid(thirteen('115536170', '001')), false); +}); + +// Build a 13-digit ЕИК with a correct final control digit: 9-digit prefix + 3 free digits + control. +// The 2,7,3,5 weights run over positions 9..12 — that is the prefix's OWN control digit plus the three +// free ones — and position 13 is the result. +function thirteen(prefix9, three) { + const s = prefix9 + three; + const d = [...s].map(Number); + const w = (ws) => ws.reduce((a, x, i) => a + x * d[8 + i], 0) % 11; + let c = w([2, 7, 3, 5]); + if (c === 10) { + c = w([4, 9, 5, 7]); + if (c === 10) c = 0; + } + return s + c; +} + +test('normalizeEik strips the „ЕИК " prefix and surrounding whitespace, like the SQL does', () => { + assert.equal(normalizeEik('ЕИК 115536179'), '115536179'); + assert.equal(normalizeEik(' 115536179 '), '115536179'); + assert.equal(normalizeEik('115536179'), '115536179'); + assert.equal(normalizeEik('не е ЕИК'), null); + assert.equal(normalizeEik(null), null); +}); + +test('normalizeEik preserves leading zeros (public bodies are exactly 000…)', () => { + // Lose these and the crawler fetches a DIFFERENT company's deed. String in, string out — never a + // numeric round-trip. + assert.equal(normalizeEik('000696327'), '000696327'); + assert.equal(typeof normalizeEik('000696327'), 'string'); +}); + +// ── the anti-drift pin ──────────────────────────────────────────────────────── +test('JS twin agrees with normalize-raw.sql eik_valid on every value', () => { + const sql = readFileSync(fileURLToPath(new URL('../normalize-raw.sql', import.meta.url)), 'utf8'); + const expr = extractEikValidExpression(sql); + + const db = new DatabaseSync(':memory:'); + db.exec('CREATE TABLE probe (eik_clean TEXT)'); + const ins = db.prepare('INSERT INTO probe VALUES (?)'); + + const cases = [...REAL, ...REAL.map((e) => e.slice(0, 8) + ((Number(e[8]) + 1) % 10))]; + for (const e of REAL) cases.push(thirteen(e, '001'), thirteen(e, '002')); + // Every 9-digit code over a fixed prefix — exercises BOTH weight passes incl. the second-10 → 0 fallback. + for (let i = 0; i < 100; i++) cases.push('2011223' + String(i).padStart(2, '0')); + for (const bad of ['000000000', '0000000000000', '12345678', '1234567890', '11553617x']) + cases.push(bad); + + for (const c of cases) ins.run(c); + const rows = db.prepare(`SELECT eik_clean AS e, ${expr} AS v FROM probe`).all(); + db.close(); + + assert.ok(rows.length >= 120, `expected a broad probe set, got ${rows.length}`); + for (const { e, v } of rows) { + assert.equal( + eikChecksumValid(e), + v === 1, + `disagreement on ${JSON.stringify(e)} (SQL said ${v})`, + ); + } +}); + +/** + * Lift the `CASE … END AS eik_valid` expression straight out of normalize-raw.sql, so the test + * compares against the FILE and not a copy of it. A copy would drift with the thing it is pinning. + */ +function extractEikValidExpression(sql) { + const start = sql.indexOf(' CASE\n WHEN eik_clean IS NULL'); + assert.ok(start > 0, 'eik_valid CASE not found in normalize-raw.sql — the pin lost its anchor'); + const end = sql.indexOf('AS eik_valid', start); + assert.ok(end > start, 'eik_valid terminator not found'); + return sql.slice(start, end); +} diff --git a/scripts/tr/paths.mjs b/scripts/tr/paths.mjs new file mode 100644 index 000000000..e325d2934 --- /dev/null +++ b/scripts/tr/paths.mjs @@ -0,0 +1,54 @@ +// Paths and path sanitizers for the Търговски регистър leg (issue #279, ADR-0033). +// +// Everything this leg writes lives under scratch/tr/, git-ignored, behind the same refuse-to-run rail +// the CACBG crawl uses — a deed carries third-party personal data (owner and manager names, the +// company's street address), so it is ADR-0010 decision 6 territory, extended by ADR-0033 to a second +// source with a stated retention. +// +// scratch/tr/deeds/.json raw response, atomic write +// scratch/tr/tr-cache.sqlite the index — ЕИК, dates, codes, verdicts. NO names. +// +// The constants below are only DEFAULTS for the CLI. Every function that touches the filesystem takes +// its path explicitly, so tests drive temp directories without mutating process state. + +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { assertScratchIgnored } from '../cacbg/guard.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); + +export const TR_SCRATCH = path.join(ROOT, 'scratch', 'tr'); +export const TR_RAW = path.join(TR_SCRATCH, 'deeds'); +export const TR_DB = path.join(TR_SCRATCH, 'tr-cache.sqlite'); + +/** Refuse to run unless scratch/tr is git-ignored. Call before the first fetch. */ +export function assertTrScratchIgnored() { + assertScratchIgnored('tr'); +} + +// A bare ЕИК: 9 or 13 digits, nothing else. Deliberately NOT `path.basename`-normalised — an ЕИК that +// needed normalising did not come from where we think it did, and silently repairing it is how you end +// up fetching a different company's deed. +const EIK_SHAPE = /^(?:\d{9}|\d{13})$/; + +/** + * Sanitize an ЕИК before it becomes a path segment or a URL segment. + * + * Returns the value VERBATIM — as a string, always. Bulgarian public bodies carry codes of exactly the + * `000…` shape, so a numeric round-trip anywhere on this path silently rewrites the identifier and the + * crawler fetches somebody else's deed (R8). + * + * Shape only. Whether the code's CHECKSUM is valid is a different question with a different remedy, + * answered by eik.mjs — conflating the two would report a real-but-invalid code as a path attack. + * @param {unknown} eik @returns {string} + */ +export function safeEik(eik) { + const s = String(eik ?? ''); + if (!EIK_SHAPE.test(s)) throw new Error(`unsafe ЕИК: ${JSON.stringify(eik)}`); + return s; +} + +/** Absolute path of the cached raw deed for an ЕИК, under `rawDir` (default TR_RAW). */ +export function deedPath(eik, rawDir = TR_RAW) { + return path.join(rawDir, `${safeEik(eik)}.json`); +} diff --git a/scripts/tr/paths.test.mjs b/scripts/tr/paths.test.mjs new file mode 100644 index 000000000..3401260d6 --- /dev/null +++ b/scripts/tr/paths.test.mjs @@ -0,0 +1,64 @@ +// node:test — path sanitizers and the refuse-to-run rail for the Trade Register leg. +// +// The deed cache holds third-party personal data (owner/manager names, company addresses), so the +// same rail the CACBG crawl runs behind applies here: everything is written under scratch/, and +// scratch/ must be git-ignored, asserted BEFORE any fetch. ADR-0010 decision 6 as extended by ADR-0033. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { TR_SCRATCH, TR_RAW, TR_DB, safeEik, assertTrScratchIgnored } from './paths.mjs'; +import { assertScratchIgnored } from '../cacbg/guard.mjs'; + +test('the TR scratch tree sits under scratch/ and is git-ignored in this repo', () => { + assert.ok(TR_SCRATCH.split(path.sep).includes('scratch'), TR_SCRATCH); + assert.ok(TR_RAW.startsWith(TR_SCRATCH)); + assert.ok(TR_DB.startsWith(TR_SCRATCH)); + assert.doesNotThrow(() => assertTrScratchIgnored()); +}); + +test('the guard is the CACBG one generalized, not a second copy', () => { + // A duplicated safety rail drifts from the original. assertScratchIgnored now takes the + // subdirectory, and its existing no-argument callers keep working unchanged. + assert.doesNotThrow(() => assertScratchIgnored()); + assert.doesNotThrow(() => assertScratchIgnored('tr')); + // .gitignore ignores `scratch/` WHOLESALE, so every subdirectory of it passes — an unknown name is + // not a way to reach the failure branch. Escape scratch/ instead (path.join normalises this to + // `docs/.probe`, which is tracked) to prove the guard still refuses when the target is not ignored. + assert.throws(() => assertScratchIgnored(path.join('..', 'docs')), /REFUSE TO RUN/); +}); + +test('safeEik accepts only a bare 9/13-digit code and returns it verbatim', () => { + assert.equal(safeEik('115536179'), '115536179'); + assert.equal(safeEik('1155361790001'), '1155361790001'); + // Leading zeros survive — public bodies are exactly this shape, and losing them fetches a + // DIFFERENT company's deed. + assert.equal(safeEik('000696327'), '000696327'); +}); + +test('safeEik refuses anything that could leave the intended path or URL', () => { + for (const bad of [ + '', + null, + undefined, + '..', + '../115536179', + '/115536179', + '115536179/../x', + '115536179?x=1', + '115536179#f', + '11553617x', + '11553617', + '1155361790', + 'ЕИК 115536179', + ' 115536179', + '115536179 ', + ]) { + assert.throws(() => safeEik(bad), /unsafe/i, JSON.stringify(bad)); + } +}); + +test('safeEik does not validate the CHECKSUM — that is a separate question', () => { + // Path safety and identity validity are different concerns: a shape-valid but checksum-invalid code + // must still be rejectable by the caller with a specific reason, not conflated into „unsafe path". + assert.equal(safeEik('115536170'), '115536170'); +}); From f8b198f32297a5701730a7e428ac834b7947871e Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 17:57:41 +0300 Subject: [PATCH 05/50] feat(tr): deed parser and the six-rung evidence ladder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The libel-critical part of #279. Pure and offline: deed in, verdict out. Nothing is wired into the publishing path yet — that is the next commit. deed.mjs enforces one order of operations, and it is an invariant with tests on it rather than a comment: decode entities → split on record-container/hr--report → drop erased → strip tags within the entity → match tokens within ONE entity. Field CR_F_19_L routinely holds several съдружници in a single string; matching against the whole field lets one person's given name combine with another's surname, and the output is a named public claim about the wrong human being. The test proves both directions — the frankenstein name does NOT match, the correct declarant DOES, and the naive whole-field approach WOULD have fired. Without that last assertion a matcher that always returns false would pass the suite. evidence.mjs implements the ladder: bar_joint_stock · document · confirmed · refuted · unknown · outside_tr, first match wins. What it establishes is company IDENTITY, never that the official owns anything — the ownership claim is the official's own filed declaration. So the failure mode of a wrong match is a real official attached to the wrong company's contracts. Fixtures are real markup, copied from the live deed for ЕИК 115536179. Writing this against imagined markup is how the entity-boundary bug ships, and doing so caught two things the ADR had wrong. ADR-0033 corrected on implementation, both from re-measuring the full deed: • the erasure marker is `erasure-text-inline`, NOT `field-text--erased` — that class occurs zero times, and an erased container has no field-text paragraph at all. Both spellings are now honoured; assuming one costs a wrong publish. • fieldOperation is not the erasure signal (2 on erased fields here, 1 on the erased history records W0 sampled), so it is not relied on. Guards that can only withhold: an unknown legal form never falls through to a lower rung; a two-token declarant can never earn „Документ"; a Latin homoglyph is a non-match and is counted; an empty seat never confirms; a seat registered after the declared period does not confirm; refutation never applies to a family stake and is suppressed inside the 2011-2012 re-registration window; matched_fact is a closed vocabulary that cannot carry a name. --- ...-evidence-replaces-name-distinctiveness.md | 11 +- scripts/tr/deed.mjs | 284 ++++++++++++++++ scripts/tr/deed.test.mjs | 273 +++++++++++++++ scripts/tr/evidence.mjs | 224 ++++++++++++ scripts/tr/evidence.test.mjs | 318 ++++++++++++++++++ 5 files changed, 1108 insertions(+), 2 deletions(-) create mode 100644 scripts/tr/deed.mjs create mode 100644 scripts/tr/deed.test.mjs create mode 100644 scripts/tr/evidence.mjs create mode 100644 scripts/tr/evidence.test.mjs diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md index e4d8ff2d0..2e449127c 100644 --- a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -68,9 +68,16 @@ requests, 2026-08-04 — well inside the pacing above, so no 429 was encountered — `legalForm=4` and `legalForm=10` return a byte-identical catalogue. - The envelope's `fullName` carries the legal form (`"ПИМК" ООД`) while `CR_F_2_L` is bare. The joint-stock bar can therefore be derived from the ЗТРРЮЛНЦ-mandated suffix, at zero extra requests. -- Erasure is **structural, not textual**: an erased entity is a `record-container` block carrying the class - `field-text--erased`. Entities inside one field are separated by +- Erasure is **structural, not textual**. Entities inside one field are separated by `
` blocks delimited by `
`. + An erased entity is such a block carrying an erasure marker. **Correction, on implementation:** this ADR + first named that marker `field-text--erased`. Re-measured against the full live deed for ЕИК 115536179, + that class occurs **zero** times; the register emits `
` (with + ``) and the erased container carries **no `field-text` paragraph at + all**. The parser therefore treats *either* spelling as erasure — honouring both costs nothing, while + assuming one costs a wrong publish. `fieldOperation` is **not** the signal: it reads 2 on both erased + fields in that deed but 1 on the erased history records W0 sampled, so it is an undocumented enum we do + not rely on. - Erased versions are **live in the current deed**: the first company sampled carries a fully-erased `CR_F_23_L` dated 2013-07-16, which read naïvely becomes „latest ownership entry: 2013-07-16" and feeds the refutation rule. diff --git a/scripts/tr/deed.mjs b/scripts/tr/deed.mjs new file mode 100644 index 000000000..7709e31c0 --- /dev/null +++ b/scripts/tr/deed.mjs @@ -0,0 +1,284 @@ +// Pure parser for a Търговски регистър deed (issue #279, ADR-0033). No I/O, no network, no state. +// +// The envelope is clean JSON, but every field's VALUE is an HTML fragment with semi-structured +// Bulgarian inside. That inner parser is where the libel risk lives, so the order of operations below +// is an invariant with tests on it, not a suggestion: +// +// decode HTML entities +// → split into entities on record-container / hr--report +// → drop entities marked erased +// → strip tags WITHIN one entity +// → separate the name from the address/stake WITHIN that entity +// → match tokens ONLY within a single entity +// +// Any other order silently merges blocks. Field CR_F_19_L routinely holds several съдружници in one +// string; matching a declarant against the whole field lets one person's given name combine with +// another's surname, and the output is a named public claim about the wrong human being. + +const OWNERSHIP_FIELDS = ['CR_F_18_L', 'CR_F_19_L', 'CR_F_23_L']; +const MANAGER_FIELD = 'CR_F_7_L'; +export const ROLE_FIELDS = [MANAGER_FIELD, ...OWNERSHIP_FIELDS]; +export { OWNERSHIP_FIELDS, MANAGER_FIELD }; + +// ── html ────────────────────────────────────────────────────────────────────── +const ENTITIES = { + '"': '"', + ''': "'", + '&': '&', + '<': '<', + '>': '>', + ' ': ' ', +}; +/** Decode the entity set the register actually emits, plus numeric escapes. FIRST step, always. */ +function decodeEntities(s) { + return String(s) + .replace(/&(?:quot|apos|amp|lt|gt|nbsp);/g, (m) => ENTITIES[m]) + .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n))) + .replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCodePoint(parseInt(n, 16))); +} + +/** Strip tags and collapse whitespace, INSIDE one already-isolated entity. */ +function stripTags(html) { + return String(html) + .replace(//gi, ' ') + .replace(/<[^>]*>/g, '') + .replace(/\s+/g, ' ') + .trim(); +} + +// An entity is erased when the register says so. MEASURED on a live deed: the marker is +// `erasure-text-inline` (with `ui-icon-erased`) and the container carries no `field-text` paragraph at +// all. `field-text--erased` is accepted as well — it is the other spelling reported for this register, +// and honouring both costs nothing while assuming one costs a wrong publish. +const ERASED_MARKER = /erasure-text-inline|ui-icon-erased|field-text--erased/; + +/** + * Split one field's htmlData into the separate registered entities it holds. + * + * @param {string} html + * @param {{strict?:boolean}} [opts] `strict` turns the erased-with-content contradiction into a throw. + * @returns {{text:string, erased:boolean}[]} + */ +export function entityBlocks(html, { strict = false } = {}) { + if (html == null || String(html).trim() === '') return []; + const decoded = decodeEntities(html); // decode BEFORE splitting — see the order above + const chunks = decoded + .split(/]*>/i) + .flatMap((part) => part.split(/(?=]*class='[^']*record-container)/i)) + .map((c) => c.trim()) + .filter((c) => c !== ''); + + const out = []; + for (const chunk of chunks) { + const erased = ERASED_MARKER.test(chunk); + // Read the visible text WITHOUT the erasure notice, so „Заличено обстоятелство." never counts as + // content and an erased block reads as empty. + const withoutNotice = chunk.replace( + /]*class='[^']*erasure-text-inline[^']*'[^>]*>.*?<\/div>/gis, + ' ', + ); + const text = stripTags(withoutNotice); + if (erased && text !== '' && strict) { + throw new Error( + `REFUSE: an erased entity carries content (${JSON.stringify(text.slice(0, 60))}) — the ` + + `"erased ⇒ empty" assumption has drifted; stop rather than guess which state is in force`, + ); + } + if (text === '' && !erased) continue; // structural noise, not an entity + out.push({ text, erased }); + } + return out; +} + +/** Every field in the deed, flattened out of sections → subDeeds → groups. */ +function allFields(deed) { + const out = []; + for (const s of deed?.sections ?? []) + for (const sd of s?.subDeeds ?? []) + for (const g of sd?.groups ?? []) for (const f of g?.fields ?? []) out.push(f); + return out; +} + +const isoDay = (v) => (v ? String(v).slice(0, 10) : null); + +/** + * The LIVE entities of the requested field codes — the single entry point to live state. + * Erased entities are dropped here and nowhere else, so the rule is auditable in one place. + * @returns {{nameCode:string, entryDate:string|null, entryNumber:string|null, entities:string[]}[]} + */ +export function liveFields(deed, nameCodes, opts = {}) { + const want = new Set(nameCodes); + const out = []; + for (const f of allFields(deed)) { + if (!want.has(f.nameCode)) continue; + const entities = entityBlocks(f.htmlData, opts) + .filter((b) => !b.erased) + .map((b) => b.text); + if (entities.length === 0) continue; + out.push({ + nameCode: f.nameCode, + // TEXT, never a number: a fieldEntryNumber like 20130716101007 exceeds 2^53 once combined. + entryNumber: f.fieldEntryNumber == null ? null : String(f.fieldEntryNumber), + entryDate: isoDay(f.fieldEntryDate), + entities, + }); + } + return out; +} + +// ── names ───────────────────────────────────────────────────────────────────── +/** + * Name tokens: NFC, upper case, split on non-letters, keep tokens of length ≥2. + * + * Dropping 1-character tokens is what makes „Г. И. Петров" a ONE-token name rather than a three-token + * one — an abbreviated name can then never reach the ≥3 tokens rung 2 requires, instead of passing on + * initials that match half the register. Latin letters are kept (never folded onto Cyrillic + * look-alikes) so a homoglyph is a non-match rather than a false match, matching companyNameKey's + * posture in packages/shared/src/company-name-key.ts. + * + * A near-twin of the module-private `holderTokens` in scripts/cacbg/parse.mjs — deliberately + * re-implemented rather than imported, because that module pulls in fast-xml-parser and would tie + * these pure tests to a workspace install. Keep the two in step. + */ +export function personTokens(name) { + return String(name ?? '') + .normalize('NFC') + .toUpperCase() + .split(/[^\p{L}]+/u) + .filter((t) => [...t].length >= 2); +} + +/** + * Does EVERY token of the declarant's name appear as a whole token of this ONE entity? + * + * Full subset, not a majority: of 301 measured matches, 46 were two-token only, which is precisely + * the homonym risk. Whole-token, not substring: „ПЕТРОВ" inside „ПЕТРОВА" is a different person. + * + * Callers MUST pass a single entity's text (see entityBlocks). Passing a whole field is the + * cross-entity bug, and no signature can prevent it — the test does. + */ +export function fullSubsetMatch(declarantName, entityText) { + const want = personTokens(declarantName); + if (want.length === 0) return false; + const have = new Set(personTokens(entityText)); + return want.every((t) => have.has(t)); +} + +// ── seat ────────────────────────────────────────────────────────────────────── +// Strip a settlement-type prefix only as a WHOLE token: „гр."/„с."/„общ."/„обл."/„ж.к." followed by a +// dot and optional space. R9 — a loose prefix strip turns СОФИЯ into ОФИЯ and ГРАДЕЦ into АДЕЦ. +const SETTLEMENT_PREFIX = /^(?:ГР|С|ОБЩ|ОБЛ|Ж\.К)\.\s*/u; + +/** Normalise a settlement name for comparison. Empty in ⇒ empty out, and empty NEVER confirms. */ +export function normalizeSettlement(raw) { + let s = String(raw ?? '') + .normalize('NFC') + .toUpperCase() + .replace(/\([^)]*\)/g, ' ') // „(столица)" + .trim(); + s = s.split(/[,/]/)[0].trim(); // cut at the first comma or slash — „с. Марково, п.к. 4108" + s = s.replace(SETTLEMENT_PREFIX, ''); + return s + .replace(/[^\p{L}\s-]/gu, '') + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * The company's registered settlement, from CR_F_5_L's „Населено място:" segment ONLY. + * + * ADR-0010 item 3 (addresses are never extracted) is honoured here, and the field makes that a live + * concern rather than a formality: CR_F_5_L also carries street, postcode, phone, fax, e-mail and + * website. Nothing but the settlement and the entry date leaves this function. + * @returns {{settlement:string, entryDate:string|null}} + */ +export function registrySeat(deed) { + for (const f of liveFields(deed, ['CR_F_5_L'])) { + for (const text of f.entities) { + const m = text.match(/Населено място:\s*([^,]+)/u); + if (m) return { settlement: normalizeSettlement(m[1]), entryDate: f.entryDate }; + } + } + return { settlement: '', entryDate: null }; +} + +// ── legal form ──────────────────────────────────────────────────────────────── +// Codes observed empirically (#279 §3 + the spike's catalogue read). DELIBERATELY incomplete: the +// nomenclature endpoint does not settle the enum (legalForm=4 and =10 return a byte-identical +// catalogue), so anything absent here is `unknown` and WITHHOLDS. In particular whether ЕАД carries +// its own code is unresolved — ООД and ЕООД turned out NOT to share one (4 vs 10), so assuming ЕАД +// shares 5 with АД would be exactly the fail-open this bar exists to prevent. +const FORM_CODES = new Map([ + [1, 'closely_held'], // ЕТ + [4, 'closely_held'], // ООД + [5, 'joint_stock'], // АД + [6, 'joint_stock'], // КДА + [10, 'closely_held'], // ЕООД +]); + +// The фирма's legal form is its SUFFIX under ЗТРРЮЛНЦ, and the deed envelope's `fullName` carries it +// („ПИМК" ООД) while CR_F_2_L is bare („ПИМК"). So the bar has a second, independent signal at zero +// extra cost. КДА is included here and is absent from classify.mjs's JOINT_STOCK/FORM_TOKENS today. +const JOINT_SUFFIX = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; +const CLOSELY_SUFFIX = /(?:^|[\s"„“”«»])(ООД|ЕООД|ЕТ|ДЗЗД|КД|СД|КООПЕРАЦИЯ)[\s"„“”«»]*$/u; + +/** + * Legal-form verdict — a union of the numeric code and the mandated фирма suffix. + * Either signal saying joint-stock bars the link. Neither able to say ⇒ `unknown`, which withholds. + * @returns {{code:number|null, codeVerdict:string, suffixVerdict:string, verdict:string}} + */ +export function registryLegalForm(deed) { + const code = typeof deed?.legalForm === 'number' ? deed.legalForm : null; + const codeVerdict = (code != null && FORM_CODES.get(code)) || 'unknown'; + + const name = String(deed?.fullName ?? '') + .normalize('NFC') + .toUpperCase() + .trim(); + const suffixVerdict = JOINT_SUFFIX.test(name) + ? 'joint_stock' + : CLOSELY_SUFFIX.test(name) + ? 'closely_held' + : 'unknown'; + + const verdict = + codeVerdict === 'joint_stock' || suffixVerdict === 'joint_stock' + ? 'joint_stock' + : codeVerdict === 'closely_held' || suffixVerdict === 'closely_held' + ? 'closely_held' + : 'unknown'; + return { code, codeVerdict, suffixVerdict, verdict }; +} + +// ── refutation input ────────────────────────────────────────────────────────── +/** + * Latest entry date across the LIVE ownership fields, or null when none survives. + * + * The trap this avoids was present in the first company sampled: CR_F_23_L sits in the CURRENT deed + * dated 2013-07-16 carrying only „Заличено обстоятелство.". Counted naively it becomes the latest + * ownership entry and can refute a link it says nothing about. liveFields drops it. + */ +export function latestOwnershipEntryDate(deed) { + const dates = liveFields(deed, OWNERSHIP_FIELDS) + .map((f) => f.entryDate) + .filter(Boolean); + return dates.length ? dates.sort().at(-1) : null; +} + +/** + * The deed we got back must be the deed we asked for. + * + * R8: Bulgarian public bodies carry ЕИК of exactly the `000…` shape, so any numeric round-trip on the + * path silently rewrites the identifier. Without this rail, that failure publishes a claim about a + * different company under the right official's name. + */ +export function assertUicEcho(deed, requestedEik) { + const got = deed?.uic == null ? null : String(deed.uic); + if (got !== String(requestedEik)) { + throw new Error( + `REFUSE: deed uic echo mismatch — requested ${JSON.stringify(String(requestedEik))}, ` + + `deed reports ${JSON.stringify(got)}`, + ); + } + return deed; +} diff --git a/scripts/tr/deed.test.mjs b/scripts/tr/deed.test.mjs new file mode 100644 index 000000000..6ec634eff --- /dev/null +++ b/scripts/tr/deed.test.mjs @@ -0,0 +1,273 @@ +// node:test — the deed parser. Pure, offline, and the most dangerous code in this change. +// +// Every fixture below is REAL markup, copied verbatim from a live deed (ЕИК 115536179, fetched +// 2026-08-05) with person names replaced only where a test needs a specific shape. Writing this +// parser against imagined markup is how the entity-boundary bug ships. +// +// The failure this file exists to prevent: field CR_F_19_L holds THREE separate people in one string, +// separated by
. Matching a declarant's tokens against the whole field lets +// the given name of one person combine with the surname of another, and the result is a named public +// claim that a specific official owns a specific company — about the wrong person. ADR-0033 decision 2. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + entityBlocks, + liveFields, + personTokens, + fullSubsetMatch, + normalizeSettlement, + registrySeat, + registryLegalForm, + latestOwnershipEntryDate, + assertUicEcho, +} from './deed.mjs'; + +// ── real markup ─────────────────────────────────────────────────────────────── +const F19_THREE = `

ПИМК ХОЛДИНГ ГРУП АД, ЕИК/ПИК 202294392, Държава: БЪЛГАРИЯ, Размер на дяловото участие: 59980.00 лв.

ПЕНКО НЕСТОРОВ НЕСТОРОВ, Държава: БЪЛГАРИЯ, Размер на дяловото участие: 10.00 лв.

ИЛИЯН КОСТАДИНОВ ФИЛИПОВ, Държава: БЪЛГАРИЯ, Размер на дяловото участие: 10.00 лв.

`; + +const F23_ERASED = `
Заличено обстоятелство.
`; + +const F5_SEAT = `

Държава: БЪЛГАРИЯ
Област: Пловдив, Община: Родопи
Населено място: с. Марково, п.к. 4108
бул./ул. местност ЗАХАРИДЕВО № 043А Телефон: 032/901102 и 032/945149
Адрес на електронна поща: office@pimk-bg.eu

`; + +const F7_MANAGER = `

АНТОН ИЦКОВ ЙОРДАНОВ, Държава: БЪЛГАРИЯ

`; + +const field = (nameCode, htmlData, over = {}) => ({ + nameCode, + htmlData, + fieldEntryNumber: '20130716101007', + fieldEntryDate: '2013-07-16T10:10:07', + fieldOperation: 3, + fieldIdent: '00190', + ...over, +}); +const deedOf = (fields, over = {}) => ({ + uic: '115536179', + fullName: '"ПИМК" ООД', + legalForm: 4, + sections: [{ subDeeds: [{ groups: [{ fields }] }] }], + ...over, +}); + +// ── T1 — the entity boundary ────────────────────────────────────────────────── +test('entityBlocks splits one field into its separate registered entities', () => { + const blocks = entityBlocks(F19_THREE); + assert.equal(blocks.length, 3, 'three съдружници, three blocks'); + assert.match(blocks[0].text, /ПИМК ХОЛДИНГ ГРУП АД/); + assert.match(blocks[1].text, /ПЕНКО НЕСТОРОВ НЕСТОРОВ/); + assert.match(blocks[2].text, /ИЛИЯН КОСТАДИНОВ ФИЛИПОВ/); + // No block may carry another block's name — that is the whole point. + assert.ok(!blocks[1].text.includes('ФИЛИПОВ')); + assert.ok(!blocks[2].text.includes('ПЕНКО')); +}); + +test('T1 — tokens from two DIFFERENT people never combine into a match', () => { + // „ПЕНКО … НЕСТОРОВ" and „ИЛИЯН КОСТАДИНОВ ФИЛИПОВ" are both in field 19. A declarant assembled + // from one person's given name and another's patronymic+surname must NOT match. + const frankenstein = 'ПЕНКО КОСТАДИНОВ ФИЛИПОВ'; + const blocks = entityBlocks(F19_THREE); + assert.equal( + blocks.some((b) => fullSubsetMatch(frankenstein, b.text)), + false, + 'cross-entity match — this is the libel bug', + ); + // Whole-field matching is exactly what must not be done; prove the naive approach WOULD have fired, + // so this test cannot silently pass because the matcher is broken in some other way. + assert.equal(fullSubsetMatch(frankenstein, F19_THREE), true, 'naive whole-field match fires'); +}); + +test('T1 positive control — the CORRECT declarant does match', () => { + // Without this, a matcher that always returns false passes every negative test above (ADR-0027). + const blocks = entityBlocks(F19_THREE); + assert.equal( + blocks.some((b) => fullSubsetMatch('ИЛИЯН КОСТАДИНОВ ФИЛИПОВ', b.text)), + true, + ); + assert.equal( + blocks.some((b) => fullSubsetMatch('Пенко Несторов Несторов', b.text)), + true, + 'case/spacing drift still matches within the right entity', + ); +}); + +test('T1 — HTML entities are decoded BEFORE splitting, not after', () => { + // Decode-after-split leaves "-bearing names mangled; decode-before-split is the tested order. + const encoded = F19_THREE.replace('ПИМК ХОЛДИНГ ГРУП АД', '"ПИМК ХОЛДИНГ ГРУП" АД'); + const blocks = entityBlocks(encoded); + assert.equal(blocks.length, 3); + assert.match(blocks[0].text, /"ПИМК ХОЛДИНГ ГРУП" АД/); + assert.ok(!blocks[0].text.includes('"'), 'entities must be decoded'); +}); + +test('T1 — a corporate съдружник contributes its own tokens, not a person match', () => { + // „ПИМК ХОЛДИНГ ГРУП АД, ЕИК/ПИК 202294392" is a legal entity. A person whose name happens to + // overlap its words must not match it into existence. + const blocks = entityBlocks(F19_THREE); + assert.equal( + fullSubsetMatch('ПИМК ХОЛДИНГ ГРУП', blocks[0].text), + true, + 'literal overlap exists', + ); + // …which is why rung 2 requires ≥3 tokens of a PERSON name; the guard lives in evidence.mjs (T2). +}); + +// ── erasure ─────────────────────────────────────────────────────────────────── +test('an erased entity is detected structurally and dropped from live state', () => { + // MEASURED: the live deed marks erasure with `erasure-text-inline` and the container carries NO + //

at all. An earlier note claimed the marker was `field-text--erased`; that + // class does not occur in the sampled deed. Both are treated as erasure so either shape is safe. + const blocks = entityBlocks(F23_ERASED); + assert.equal(blocks.length, 1); + assert.equal(blocks[0].erased, true); + const live = liveFields(deedOf([field('CR_F_23_L', F23_ERASED, { fieldOperation: 2 })]), [ + 'CR_F_23_L', + ]); + assert.deepEqual(live, [], 'an erased field contributes no live entity'); +}); + +test('the OTHER erasure spelling is honoured too', () => { + const alt = `

СТАР СОБСТВЕНИК

Заличено обстоятелство.
`; + assert.equal(entityBlocks(alt)[0].erased, true); +}); + +test('an erased-looking block carrying real content REFUSES the deed (drift alarm)', () => { + // R7: "erased ⇒ empty" is an empirical observation (93 of 93). If the register ever emits an erased + // marker beside live content, the assumption is broken and we must stop, not silently drop a real + // owner or silently keep a removed one. + const contradictory = `

ЖИВО ИМЕ ТУК

Заличено обстоятелство.
`; + assert.throws(() => entityBlocks(contradictory, { strict: true }), /erased.*content|drift/i); +}); + +test('an empty htmlData yields no entities and does not throw', () => { + assert.deepEqual(entityBlocks(''), []); + assert.deepEqual(entityBlocks(null), []); +}); + +test('liveFields keeps only the requested codes and reports entry date/number', () => { + const d = deedOf([ + field('CR_F_7_L', F7_MANAGER, { nameCode: 'CR_F_7_L', fieldEntryDate: '2017-09-15T00:00:00' }), + field('CR_F_19_L', F19_THREE), + field('CR_F_5_L', F5_SEAT), + ]); + const live = liveFields(d, ['CR_F_7_L', 'CR_F_19_L']); + assert.deepEqual(live.map((f) => f.nameCode).sort(), ['CR_F_19_L', 'CR_F_7_L']); + const f19 = live.find((f) => f.nameCode === 'CR_F_19_L'); + assert.equal(f19.entities.length, 3); + assert.equal(f19.entryDate, '2013-07-16'); + assert.equal(f19.entryNumber, '20130716101007'); + assert.equal(typeof f19.entryNumber, 'string', 'entry numbers exceed 2^53 — never a number'); +}); + +// ── T2 — the token rule ─────────────────────────────────────────────────────── +test('personTokens keeps tokens of length ≥2 and folds case/spacing', () => { + assert.deepEqual(personTokens('Иван Петров Георгиев'), ['ИВАН', 'ПЕТРОВ', 'ГЕОРГИЕВ']); + assert.deepEqual(personTokens(' иван петров '), ['ИВАН', 'ПЕТРОВ']); + // An initial is not a token — „Г. И. Петров" is ONE token, so it can never reach three. + assert.deepEqual(personTokens('Г. И. Петров'), ['ПЕТРОВ']); + // A hyphenated surname is one token, not two. + assert.deepEqual(personTokens('Мария Иванова-Петрова'), ['МАРИЯ', 'ИВАНОВА', 'ПЕТРОВА']); +}); + +test('fullSubsetMatch requires EVERY declarant token, not a majority', () => { + const entity = 'ИЛИЯН КОСТАДИНОВ ФИЛИПОВ, Държава: БЪЛГАРИЯ'; + assert.equal(fullSubsetMatch('ИЛИЯН КОСТАДИНОВ ФИЛИПОВ', entity), true); + // 2-of-3 must fail: of 301 measured matches, 46 were two-token only — the homonym risk itself. + assert.equal(fullSubsetMatch('ИЛИЯН КОСТАДИНОВ ПЕТРОВ', entity), false); + assert.equal(fullSubsetMatch('ИЛИЯН ПЕТРОВ ФИЛИПОВ', entity), false); +}); + +test('a token must match a WHOLE token, never a substring', () => { + // „ПЕТРОВ" must not be found inside „ПЕТРОВА"; that is a different person. + assert.equal(fullSubsetMatch('ИВАН ПЕТРОВ ГЕОРГИЕВ', 'ИВАН ПЕТРОВА ГЕОРГИЕВА'), false); +}); + +// ── T5 — settlement normalization ───────────────────────────────────────────── +test('T5 — the settlement prefix is stripped only as a whole token', () => { + // R9: a naive prefix strip turns СОФИЯ into ОФИЯ and ГРАДЕЦ into АДЕЦ. + assert.equal(normalizeSettlement('гр. Русе'), 'РУСЕ'); + assert.equal(normalizeSettlement('с. Марково'), 'МАРКОВО'); + assert.equal(normalizeSettlement('София'), 'СОФИЯ'); + assert.equal(normalizeSettlement('СОФИЯ'), 'СОФИЯ'); + assert.equal(normalizeSettlement('Градец'), 'ГРАДЕЦ'); + assert.equal(normalizeSettlement('гр.Пловдив'), 'ПЛОВДИВ'); + assert.equal(normalizeSettlement('София (столица)'), 'СОФИЯ'); +}); + +test('T5 — an empty settlement never equals another empty settlement', () => { + // „both blank ⇒ confirmed" would rubber-stamp every link with no seat data at all. + assert.equal(normalizeSettlement(''), ''); + assert.equal(normalizeSettlement(null), ''); + assert.equal(normalizeSettlement(' '), ''); +}); + +test('registrySeat reads the „Населено място" segment and NOTHING else', () => { + const seat = registrySeat(deedOf([field('CR_F_5_L', F5_SEAT, { nameCode: 'CR_F_5_L' })])); + assert.equal(seat.settlement, 'МАРКОВО'); + assert.equal(seat.entryDate, '2013-07-16'); + // ADR-0010 item 3: the parser never returns an address, phone, e-mail or website — and the deed + // demonstrably carries all four. + const blob = JSON.stringify(seat); + for (const leak of ['ЗАХАРИДЕВО', '032/901102', 'office@pimk-bg.eu', 'п.к.', '4108', 'Родопи']) + assert.ok(!blob.includes(leak), `seat must not carry ${leak}`); +}); + +// ── T3 — the joint-stock bar ────────────────────────────────────────────────── +test('T3 — the legal-form verdict is a UNION of the code and the ЗТРРЮЛНЦ suffix', () => { + const jointByCode = registryLegalForm(deedOf([], { legalForm: 5, fullName: 'НЕЩО СИ' })); + assert.equal(jointByCode.verdict, 'joint_stock'); + + // An UNKNOWN code must not fall through: the suffix decides, and if it cannot, we withhold. + const unknownButEad = registryLegalForm(deedOf([], { legalForm: 99, fullName: '"ГАМА" ЕАД' })); + assert.equal(unknownButEad.verdict, 'joint_stock', 'barred by the mandated suffix'); + + const unknownButEood = registryLegalForm(deedOf([], { legalForm: 99, fullName: '"БЕТА" ЕООД' })); + assert.equal(unknownButEood.verdict, 'closely_held', 'the bar is not blanket'); + + const unknownAndUnreadable = registryLegalForm(deedOf([], { legalForm: 99, fullName: 'НЕЩО' })); + assert.equal(unknownAndUnreadable.verdict, 'unknown', 'unknown withholds — it never publishes'); +}); + +test('T3 — КДА is barred (it is not in the existing closelyHeldForm token list)', () => { + assert.equal( + registryLegalForm(deedOf([], { legalForm: 99, fullName: '"X" КДА' })).verdict, + 'joint_stock', + ); + assert.equal( + registryLegalForm(deedOf([], { legalForm: 6, fullName: 'X' })).verdict, + 'joint_stock', + ); +}); + +test('T3 — a real ООД deed reads as closely held, by code AND by suffix', () => { + const v = registryLegalForm(deedOf([])); + assert.equal(v.code, 4); + assert.equal(v.verdict, 'closely_held'); + assert.equal(v.suffixVerdict, 'closely_held', 'fullName carries the form: "ПИМК" ООД'); +}); + +// ── T7 — the UIC echo ───────────────────────────────────────────────────────── +test('T7 — a deed whose UIC does not echo the request is REFUSED', () => { + // R8: ЕИК leading zeros are significant (public bodies are exactly 000…). If anything on the path + // rewrites the identifier, this is the rail that catches it before a claim is made about the + // wrong company. + assert.doesNotThrow(() => assertUicEcho(deedOf([]), '115536179')); + assert.throws(() => assertUicEcho(deedOf([]), '000696327'), /uic|echo/i); + assert.throws(() => assertUicEcho(deedOf([], { uic: '696327' }), '000696327'), /uic|echo/i); + assert.throws(() => assertUicEcho(deedOf([], { uic: null }), '115536179'), /uic|echo/i); +}); + +// ── the refutation input ────────────────────────────────────────────────────── +test('latestOwnershipEntryDate ignores ERASED ownership fields', () => { + // The trap, present in the first company sampled: CR_F_23_L is live in the current deed, dated + // 2013-07-16, and contains only „Заличено обстоятелство.". Read naively it becomes „latest + // ownership entry: 2013-07-16" and can refute a link it knows nothing about. + const d = deedOf([ + field('CR_F_19_L', F19_THREE, { fieldEntryDate: '2011-05-02T00:00:00' }), + field('CR_F_23_L', F23_ERASED, { fieldEntryDate: '2013-07-16T10:10:07', fieldOperation: 2 }), + ]); + assert.equal(latestOwnershipEntryDate(d), '2011-05-02', 'the erased 2013 entry must not count'); +}); + +test('latestOwnershipEntryDate is null when no live ownership field survives', () => { + const d = deedOf([field('CR_F_23_L', F23_ERASED, { fieldOperation: 2 })]); + assert.equal(latestOwnershipEntryDate(d), null); +}); diff --git a/scripts/tr/evidence.mjs b/scripts/tr/evidence.mjs new file mode 100644 index 000000000..262e31eb0 --- /dev/null +++ b/scripts/tr/evidence.mjs @@ -0,0 +1,224 @@ +// The evidence ladder (issue #279 §5, ADR-0033 decision 1). Pure: deed in, verdict out. Zero network. +// +// Six outcomes, FIRST MATCH WINS: +// 1 bar_joint_stock АД / ЕАД / КДА — never published, whatever follows +// 2 document the declarant's full name is in a live CR_F_7/18/19/23_L entity +// 3 confirmed declared seat == registered seat, or the declarant wrote the ЕИК +// 4 refuted own stake only: absent from live state, and the live ownership record +// predates the declared period — the register covers it and does not name them +// 5 unknown everything else — held +// 6 outside_tr not in the register at all (ДЗЗД, БУЛСТАТ associations) — held +// +// WHAT THIS ESTABLISHES, precisely: the identity of the COMPANY — that the company behind the declared +// name is the same legal entity as the winner we matched. It does NOT establish that the official owns +// it; that claim comes from their own filed declaration and is not a heuristic at all. The failure mode +// of a wrong match is therefore not an invented ownership claim but a real official attached to the +// WRONG company's ЕИК, contracts and money. Still a false public statement about a named person, which +// is why rung 2 requires a full three-token subset match inside a single registry entity, and why the +// filters that can only withhold are kept (ADR-0033 decision 2). + +import { + liveFields, + fullSubsetMatch, + personTokens, + normalizeSettlement, + registrySeat, + registryLegalForm, + latestOwnershipEntryDate, + OWNERSHIP_FIELDS, + MANAGER_FIELD, + ROLE_FIELDS, +} from './deed.mjs'; + +/** + * Version of the RULES, not of the code. §8's monotonicity gate keys on this: a previously published + * link disappearing under an UNCHANGED rules version is a hard finding; under a changed one it is an + * expected diff. Bump it whenever a rung's meaning changes. + */ +export const RULES_VERSION = 'tr-rules-1'; + +/** Rung 2 needs a real three-part Bulgarian name (ЗГР чл. 9). Two tokens is the homonym risk itself. */ +const MIN_NAME_TOKENS = 3; + +// Court-registered companies were re-registered into the Търговски регистър in a single administrative +// push, which flattened their entry dates into this window. „Strictly before the declared period" +// certifies nothing when the date is an artefact of the migration rather than of the ownership, so the +// refutation rung is suppressed inside it (R13). A suppressed refutation falls through to `unknown` — +// held, not published, which is the safe direction. +const REREGISTRATION_START = '2011-01-01'; +const REREGISTRATION_END = '2012-12-31'; + +/** + * Find the declarant inside the live entities of the given field codes. + * Matching happens per ENTITY — never against a whole field — because one field routinely holds + * several people and combining tokens across them is the libel bug. + * @returns {{nameCode:string, entryNumber:string|null, entryDate:string|null}|null} + */ +function findPerson(deed, name, nameCodes) { + for (const f of liveFields(deed, nameCodes)) { + for (const entity of f.entities) { + if (fullSubsetMatch(name, entity)) { + return { nameCode: f.nameCode, entryNumber: f.entryNumber, entryDate: f.entryDate }; + } + } + } + return null; +} + +/** + * Decide the evidence for one link. + * + * @param {object} input + * @param {object|null} input.deed parsed deed JSON; null only when `outsideTr` + * @param {boolean} [input.outsideTr] the ЕИК is not in the register at all + * @param {string} input.declarantName the office-holder's name as filed + * @param {string[]} [input.declaredSeats] seats declared BY THIS PERSON FOR THIS COMPANY only — + * 4.9% of company-name keys carry more than one distinct + * declared seat, so a company-only key would let one + * person's seat confirm another person's link + * @param {boolean} [input.declaredEik] the declarant wrote the ЕИК in the declaration + * @param {number|null} [input.firstDeclaredYear] + * @param {'self'|'family'} [input.scope] + * @param {boolean} [input.nameGloballyUnique] AND-gate on the WEAKEST rung only + * @returns {{kind:string, publishable:boolean, registryRole:string|null, matchedFact:string|null, + * entryNumber:string|null, entryDate:string|null, rulesVersion:string, + * shortName:boolean, latinInName:boolean}} + */ +export function evidenceVerdict(input) { + const { + deed, + outsideTr = false, + declarantName, + declaredSeats = [], + declaredEik = false, + firstDeclaredYear = null, + scope = 'self', + nameGloballyUnique = true, + } = input; + + const tokens = personTokens(declarantName); + const telemetry = { + rulesVersion: RULES_VERSION, + // Counted, not silently dropped: a refusal we cannot see is a recall hole nobody can size. + shortName: tokens.length < MIN_NAME_TOKENS, + latinInName: /[A-Za-z]/.test(String(declarantName ?? '')), + }; + const verdict = (kind, publishable, extra = {}) => ({ + kind, + publishable, + registryRole: null, + matchedFact: null, + entryNumber: null, + entryDate: null, + ...telemetry, + ...extra, + }); + + if (outsideTr) return verdict('outside_tr', false); + if (deed == null) { + // Fail closed and loudly. A missing deed quietly downgraded to „unknown" is indistinguishable + // from a real hold, and hides a cache gap that should stop the run. + throw new Error('evidenceVerdict: deed is required unless outsideTr is set'); + } + + // ── rung 1 ────────────────────────────────────────────────────────────────── + // A union of the numeric code and the ЗТРРЮЛНЦ suffix; either saying joint-stock bars the link, and + // neither able to say means we withhold rather than guess. + const form = registryLegalForm(deed); + if (form.verdict === 'joint_stock') return verdict('bar_joint_stock', false); + if (form.verdict === 'unknown') return verdict('unknown', false); + + // ── rung 2 ────────────────────────────────────────────────────────────────── + // Only a full three-token name may assert. A Latin homoglyph makes the name a non-match rather than + // a false match — company-name-key.ts's posture, applied to people. + const eligibleForDocument = !telemetry.shortName && !telemetry.latinInName; + if (eligibleForDocument) { + const owner = findPerson(deed, declarantName, OWNERSHIP_FIELDS); + if (owner) { + return verdict('document', true, { + registryRole: 'owner', + matchedFact: `role:owner:${owner.nameCode}`, + entryNumber: owner.entryNumber, + entryDate: owner.entryDate, + }); + } + const manager = findPerson(deed, declarantName, [MANAGER_FIELD]); + if (manager) { + return verdict('document', true, { + registryRole: 'manager', + matchedFact: `role:manager:${manager.nameCode}`, + entryNumber: manager.entryNumber, + entryDate: manager.entryDate, + }); + } + } + + // ── rung 3 ────────────────────────────────────────────────────────────────── + // The weakest publishing rung, so it carries the extra AND-gate: a nationally shared company name + // cannot ride it (ADR-0017's holding, carried forward). The stronger „Документ" rung above is not + // gated — the register named this person in THIS company, which makes the name key moot. + if (nameGloballyUnique) { + if (declaredEik) return verdict('confirmed', true, { matchedFact: 'eik' }); + + const seat = registrySeat(deed); + // Empty NEVER confirms — otherwise every link with no seat data on either side rubber-stamps. + if (seat.settlement !== '') { + const declared = declaredSeats.map(normalizeSettlement).filter((s) => s !== ''); + // R10: the registered seat must have been in force for the declared period. Seats move, and a + // company that relocated INTO the declared settlement afterwards would otherwise confirm falsely. + const seatCoversPeriod = + firstDeclaredYear == null || + seat.entryDate == null || + seat.entryDate <= `${firstDeclaredYear}-12-31`; + if (seatCoversPeriod && declared.includes(seat.settlement)) { + return verdict('confirmed', true, { + matchedFact: `seat:${seat.settlement}`, + entryDate: seat.entryDate, + }); + } + } + } + + // ── rung 4 ────────────────────────────────────────────────────────────────── + // OWN stakes only. For a family stake the registered owner is the relative, whose name we neither + // store nor check, so absence of the OFFICIAL from the deed is evidence of nothing. An early branch, + // not a caller convention. + if (scope === 'self' && firstDeclaredYear != null) { + const stillPresent = findPerson(deed, declarantName, ROLE_FIELDS); + const latest = latestOwnershipEntryDate(deed); + const inRereg = + latest != null && latest >= REREGISTRATION_START && latest <= REREGISTRATION_END; + if (!stillPresent && latest != null && !inRereg && latest < `${firstDeclaredYear}-01-01`) { + return verdict('refuted', false, { entryDate: latest }); + } + } + + // ── rung 5 ────────────────────────────────────────────────────────────────── + return verdict('unknown', false); +} + +/** + * Reconcile a DECLARED termination against the live deed (#279 §7). + * + * „Terminated" is an inference from silence — the commonest cause is a finished mandate, not a sale — + * so ADR-0021 E11's withdrawal is checked against the register before it takes effect. + * + * Phase 1 uses `terminated` ONLY. `label` is computed but deliberately not rendered: „и към днешна + * дата" asserts a present tense about a named person on evidence whose freshness is bounded by the + * cache refresh cycle, and it is deferred behind an LIA addendum (ADR-0033 decision 4). + * + * @returns {{terminated:boolean, label:'owner_today'|'manager_today'|null}} + */ +export function reconcileTermination({ deed, declarantName, scope = 'self' }) { + // Family first, structurally: there is nothing to look for, and looking would be an attempt to + // identify the relative. + if (scope !== 'self' || deed == null) return { terminated: true, label: null }; + + if (findPerson(deed, declarantName, OWNERSHIP_FIELDS)) { + return { terminated: false, label: 'owner_today' }; + } + if (findPerson(deed, declarantName, [MANAGER_FIELD])) { + return { terminated: true, label: 'manager_today' }; + } + return { terminated: true, label: null }; +} diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs new file mode 100644 index 000000000..7c43b7971 --- /dev/null +++ b/scripts/tr/evidence.test.mjs @@ -0,0 +1,318 @@ +// node:test — the evidence ladder (ADR-0033 decision 1). Pure: deed in, verdict out. +// +// Six outcomes, first match wins. What each rung is allowed to CONCLUDE is the whole subject: +// the registry proves the identity of the COMPANY, never that the official owns it — the ownership +// claim comes from the official's own filed declaration. So a wrong match here does not invent an +// ownership claim, it attaches a real official to the wrong company's ЕИК, contracts and money. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { RULES_VERSION, evidenceVerdict, reconcileTermination } from './evidence.mjs'; + +const container = (t) => + `

${t}

`; +const joined = (...ts) => ts.map(container).join(`
`); +const ERASED = `
Заличено обстоятелство.
`; + +const fld = (nameCode, htmlData, entryDate = '2011-05-02T00:00:00') => ({ + nameCode, + htmlData, + fieldEntryNumber: '20110502101007', + fieldEntryDate: entryDate, + fieldOperation: 3, +}); +const deed = (fields, over = {}) => ({ + uic: '201122335', + fullName: '"АЛФА СТРОЙ" ООД', + legalForm: 4, + sections: [{ subDeeds: [{ groups: [{ fields }] }] }], + ...over, +}); + +const OWNER_DEED = deed([ + fld('CR_F_19_L', joined('ИВАН ПЕТРОВ ТЕСТОВ, Държава: БЪЛГАРИЯ', 'МАРИЯ СТОЯНОВА ИВАНОВА')), + fld('CR_F_5_L', container('Държава: БЪЛГАРИЯ
Населено място: гр. Пловдив, п.к. 4000')), +]); + +const base = { + deed: OWNER_DEED, + declarantName: 'Иван Петров Тестов', + declaredSeats: [], + declaredEik: false, + firstDeclaredYear: 2021, + scope: 'self', + nameGloballyUnique: true, +}; + +test('RULES_VERSION is a stable, non-empty identifier — §8 hangs off it', () => { + assert.equal(typeof RULES_VERSION, 'string'); + assert.ok(RULES_VERSION.length > 0); +}); + +// ── rung 1: the joint-stock bar wins over everything ────────────────────────── +test('rung 1 — a joint-stock company is barred even when the person IS in the deed', () => { + const ad = deed([fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))], { + legalForm: 5, + fullName: '"ГАМА ИНВЕСТ" АД', + }); + const v = evidenceVerdict({ ...base, deed: ad }); + assert.equal(v.kind, 'bar_joint_stock'); + assert.equal(v.publishable, false); +}); + +test('rung 1 — an UNKNOWN legal form withholds; it never falls through to a lower rung', () => { + const odd = deed([fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))], { + legalForm: 99, + fullName: 'НЕЩО БЕЗ ФОРМА', + }); + const v = evidenceVerdict({ ...base, deed: odd }); + assert.equal(v.kind, 'unknown'); + assert.equal(v.publishable, false); +}); + +// ── rung 2: „Документ" ──────────────────────────────────────────────────────── +test('rung 2 — a full-name match in a live ownership field publishes, with the role kept', () => { + const v = evidenceVerdict(base); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); + assert.equal(v.registryRole, 'owner'); + assert.equal(v.matchedFact, 'role:owner:CR_F_19_L'); + assert.equal(v.entryNumber, '20110502101007'); + assert.equal(v.entryDate, '2011-05-02'); +}); + +test('rung 2 — a manager-only match publishes but records the weaker role', () => { + const mgr = deed([ + fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ, Държава: БЪЛГАРИЯ')), + fld('CR_F_19_L', container('ДРУГО ЛИЦЕ ТУК')), + ]); + const v = evidenceVerdict({ ...base, deed: mgr }); + assert.equal(v.kind, 'document'); + assert.equal(v.registryRole, 'manager'); + assert.equal(v.matchedFact, 'role:manager:CR_F_7_L'); +}); + +test('rung 2 — a TWO-token declarant can never earn „Документ"', () => { + // 46 of 301 measured matches were two-token only, which is exactly the homonym risk. Falls to a + // lower rung rather than publishing on a name that half a register could satisfy. + const two = deed([fld('CR_F_19_L', container('ИВАН ТЕСТОВ, Държава: БЪЛГАРИЯ'))]); + const v = evidenceVerdict({ ...base, deed: two, declarantName: 'Иван Тестов' }); + assert.notEqual(v.kind, 'document'); + assert.equal(v.shortName, true, 'the refusal is counted, not silently dropped'); +}); + +test('rung 2 — the match must fall inside ONE entity (the libel guard, end to end)', () => { + const two = deed([ + fld('CR_F_19_L', joined('ПЕНКО НЕСТОРОВ НЕСТОРОВ', 'ИЛИЯН КОСТАДИНОВ ФИЛИПОВ')), + ]); + const v = evidenceVerdict({ ...base, deed: two, declarantName: 'ПЕНКО КОСТАДИНОВ ФИЛИПОВ' }); + assert.notEqual(v.kind, 'document'); +}); + +test('rung 2 — an ERASED ownership entry cannot produce a document match', () => { + const gone = deed([fld('CR_F_23_L', ERASED, '2013-07-16T10:10:07')]); + const v = evidenceVerdict({ ...base, deed: gone }); + assert.notEqual(v.kind, 'document'); +}); + +test('rung 2 — a Latin homoglyph in the name is a NON-match, and is counted', () => { + // company-name-key.ts deliberately does not fold Cyrillic↔Latin; person names take the same posture. + const v = evidenceVerdict({ ...base, declarantName: 'ИBAH ПЕТРОВ ТЕСТОВ' }); // Latin B, A, H + assert.notEqual(v.kind, 'document'); + assert.equal(v.latinInName, true); +}); + +// ── rung 3: „Потвърдено" ────────────────────────────────────────────────────── +test('rung 3 — a declared seat matching the registered seat confirms the company', () => { + const other = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив, п.к. 4000'), '2015-01-01T00:00:00'), + ]); + const v = evidenceVerdict({ ...base, deed: other, declaredSeats: ['Пловдив'] }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.publishable, true); + assert.equal(v.matchedFact, 'seat:ПЛОВДИВ'); +}); + +test('rung 3 — a declared ЕИК confirms the company on its own', () => { + const other = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК'))]); + const v = evidenceVerdict({ ...base, deed: other, declaredEik: true }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'eik'); +}); + +test('rung 3 — an EMPTY declared seat never confirms', () => { + const noSeat = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК'))]); + const v = evidenceVerdict({ ...base, deed: noSeat, declaredSeats: ['', ' '] }); + assert.notEqual(v.kind, 'confirmed'); +}); + +test('rung 3 — a seat registered AFTER the declared period does not confirm', () => { + // R10, and W0 measured that seats move: a company that relocated INTO the declared settlement after + // the fact would otherwise produce a false „Потвърдено". + const moved = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив'), '2024-06-01T00:00:00'), + ]); + const v = evidenceVerdict({ + ...base, + deed: moved, + declaredSeats: ['Пловдив'], + firstDeclaredYear: 2021, + }); + assert.notEqual(v.kind, 'confirmed'); +}); + +test('rung 3 — the weakest rung ALSO requires global name uniqueness (ADR-0017 carried forward)', () => { + const other = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: other, + declaredSeats: ['Пловдив'], + nameGloballyUnique: false, + }); + assert.notEqual(v.kind, 'confirmed', 'a nationally shared name cannot ride the weakest rung'); +}); + +test('rung 3 — name uniqueness does NOT gate the stronger „Документ" rung', () => { + const v = evidenceVerdict({ ...base, nameGloballyUnique: false }); + assert.equal( + v.kind, + 'document', + 'the registry named the person in THIS company; the name key is moot', + ); +}); + +// ── rung 4: „Оборена" ───────────────────────────────────────────────────────── +test('rung 4 — absent from a deed whose ownership predates the declaration refutes the link', () => { + const older = deed([ + fld('CR_F_19_L', container('СЪВСЕМ ДРУГ СОБСТВЕНИК'), '2015-03-01T00:00:00'), + ]); + const v = evidenceVerdict({ ...base, deed: older, firstDeclaredYear: 2021 }); + assert.equal(v.kind, 'refuted'); + assert.equal(v.publishable, false); +}); + +test('rung 4 — the comparison is date-to-DATE, not date-to-year', () => { + // R17: „strictly before the first declared year" means before YYYY-01-01. An entry inside the first + // declared year does NOT cover the period and must not refute. + const inYear = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2021-06-15T00:00:00')]); + assert.notEqual( + evidenceVerdict({ ...base, deed: inYear, firstDeclaredYear: 2021 }).kind, + 'refuted', + ); + const justBefore = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2020-12-31T00:00:00')]); + assert.equal( + evidenceVerdict({ ...base, deed: justBefore, firstDeclaredYear: 2021 }).kind, + 'refuted', + ); +}); + +test('rung 4 — NEVER applies to a family stake', () => { + // The owner there is the relative, whose name we neither store nor check (ADR-0010 item 4, + // ADR-0032 decision 2), so „the official is not in the deed" says nothing at all. + const older = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2015-03-01T00:00:00')]); + const v = evidenceVerdict({ ...base, deed: older, scope: 'family', firstDeclaredYear: 2021 }); + assert.notEqual(v.kind, 'refuted'); + assert.equal(v.kind, 'unknown'); +}); + +test('rung 4 — suppressed inside the 2011–2012 re-registration window', () => { + // R13: court-registered companies had every entry date flattened into the re-registration window, + // so „strictly before" certifies nothing there. + const flattened = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2011-11-04T00:00:00')]); + const v = evidenceVerdict({ ...base, deed: flattened, firstDeclaredYear: 2021 }); + assert.notEqual(v.kind, 'refuted'); + assert.equal(v.kind, 'unknown'); +}); + +// ── rungs 5 and 6 ───────────────────────────────────────────────────────────── +test('rung 5 — everything else is „Неизвестна" and stays hidden', () => { + const recent = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2023-01-01T00:00:00')]); + const v = evidenceVerdict({ ...base, deed: recent, firstDeclaredYear: 2021 }); + assert.equal(v.kind, 'unknown'); + assert.equal(v.publishable, false); +}); + +test('rung 6 — outside the register is its own outcome, and is not publishable', () => { + const v = evidenceVerdict({ ...base, deed: null, outsideTr: true }); + assert.equal(v.kind, 'outside_tr'); + assert.equal(v.publishable, false); +}); + +test('a missing deed that is NOT marked outside-ТР is an error, not a silent hold', () => { + // Fail closed: a cache gap must be visible, never quietly downgraded to „unknown". + assert.throws(() => evidenceVerdict({ ...base, deed: null, outsideTr: false }), /deed/i); +}); + +// ── the seal ────────────────────────────────────────────────────────────────── +test('matched_fact stays inside the closed vocabulary — it can never carry a name', () => { + const CLOSED = /^(?:seat:[\p{Lu} -]+|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + for (const v of [ + evidenceVerdict(base), + evidenceVerdict({ ...base, deed: deed([fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))]) }), + evidenceVerdict({ ...base, declaredEik: true }), + evidenceVerdict({ + ...base, + deed: deed([ + fld('CR_F_19_L', container('ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]), + declaredSeats: ['Пловдив'], + }), + ]) { + if (v.matchedFact == null) continue; + assert.match(v.matchedFact, CLOSED, `matched_fact escaped the vocabulary: ${v.matchedFact}`); + assert.ok(!/ИВАН|ПЕТРОВ|ТЕСТОВ/.test(v.matchedFact), 'a NAME reached matched_fact'); + } +}); + +test('every verdict carries the rules version that produced it', () => { + assert.equal(evidenceVerdict(base).rulesVersion, RULES_VERSION); +}); + +// ── §7 reconciliation ───────────────────────────────────────────────────────── +test('reconcileTermination — still a registered owner ⇒ NOT terminated', () => { + const r = reconcileTermination({ + deed: OWNER_DEED, + declarantName: 'Иван Петров Тестов', + scope: 'self', + }); + assert.equal(r.terminated, false); + assert.equal(r.label, 'owner_today'); +}); + +test('reconcileTermination — manager only ⇒ terminated as a stake, but the tie continues', () => { + const mgr = deed([ + fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ')), + fld('CR_F_19_L', container('ДРУГ')), + ]); + const r = reconcileTermination({ deed: mgr, declarantName: 'Иван Петров Тестов', scope: 'self' }); + assert.equal(r.terminated, true); + assert.equal(r.label, 'manager_today'); +}); + +test('reconcileTermination — absent from the live deed ⇒ the declared termination stands', () => { + const none = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ'))]); + const r = reconcileTermination({ + deed: none, + declarantName: 'Иван Петров Тестов', + scope: 'self', + }); + assert.equal(r.terminated, true); + assert.equal(r.label, null); +}); + +test('reconcileTermination — a FAMILY stake is never reconciled, by an early branch', () => { + // Structural, not a caller convention: the relative's name is not stored, so there is nothing to + // look for, and looking would be a de-anonymisation attempt. + const r = reconcileTermination({ + deed: OWNER_DEED, + declarantName: 'Иван Петров Тестов', + scope: 'family', + }); + assert.equal(r.terminated, true); + assert.equal(r.label, null); +}); From 98ac4f5aaa03572645ffe38894d70f54b5f18c20 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 22:23:32 +0300 Subject: [PATCH 06/50] =?UTF-8?q?feat(tr):=20the=20deed=20crawler=20?= =?UTF-8?q?=E2=80=94=20paced,=20resumable,=20and=20stopping=20on=20a=20429?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One request per candidate ЕИК, sequential, 1 per 3 s. What it refuses to do is the substance, and every refusal is driven offline through injected I/O. • The pace flag cannot make it faster, only slower. Tuning around a rate limiter empirically is what spec §3.3's „NEVER bulk-scrape" forbids, so --min-interval-ms below the documented 3 s is rejected outright. • A 429 ends the run with exit 2 and records NOTHING about the ЕИК that hit it — that ЕИК is unknown, not absent, and the throttle is a fact about us, not about the company. A later re-run resumes exactly there. • The candidate set is closed: it never follows a link out of a deed. • A checksum-invalid code is dropped before any request. • The returned deed's UIC must echo the request, or it is refused and not cached — otherwise one company's deed lands under another's ЕИК. Two things measured against the live register during this work: „Outside the Търговски регистър" does NOT look like #279 §3 describes. An ЕИК that is not a търговец answers HTTP 200 with a ZERO-BYTE body — not a 404, not HTML. Verified on Община София (000696327): empty on two consecutive requests while a real company returned its full deed in the same window. The rule that keeps R6 honest is therefore the STATUS, not the emptiness: empty under 200 is the register's answer and may be cached permanently; empty under 5xx is a failure and stays transient. Both directions are tested. Without this the ~4 non-търговец ЕИК would never resolve and the run could never exit 0. The breaker threshold is 5, not 10, because the unit is ЕИК and each unresolved one costs the full 5-attempt budget. At 10 the breaker would spend ~50 requests against an already-failing endpoint — the exact volume at which the register was observed to begin a sustained 429, i.e. the safety mechanism would itself have tripped the block. At 5 the worst case is ~25. The breaker test caught itself being fake: it had been generating made-up ЕИК of which only 1 in 10 passed the checksum, so the crawler dropped them all and the assertion passed on zero requests. It now generates genuinely valid codes and asserts the REQUEST budget, not just the candidate count. --- ...-evidence-replaces-name-distinctiveness.md | 6 + scripts/tr/fetch-deeds.mjs | 258 ++++++++++++ scripts/tr/fetch-deeds.test.mjs | 373 ++++++++++++++++++ 3 files changed, 637 insertions(+) create mode 100644 scripts/tr/fetch-deeds.mjs create mode 100644 scripts/tr/fetch-deeds.test.mjs diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md index 2e449127c..b3315c993 100644 --- a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -82,6 +82,12 @@ requests, 2026-08-04 — well inside the pacing above, so no 429 was encountered `CR_F_23_L` dated 2013-07-16, which read naïvely becomes „latest ownership entry: 2013-07-16" and feeds the refutation rule. - Seats move. The same company's `CR_F_5_L` shows entry dates 2010-11-04 and 2014-01-23. +- **An ЕИК that is not a търговец answers `HTTP 200` with a ZERO-BYTE body** — not the 404, and not the + HTML, that §3 predicts. Measured on Община София (`000696327`): empty on two consecutive requests, + while a real company returned its full 34,398-byte deed in the same window, so it is the register's + answer rather than an outage. This is what the „извън ТР" rung actually looks like on the wire. The + distinction that keeps R6 honest is therefore the **status**, not the empty body: empty under 200 is + a documented negative and may be cached permanently; empty under 5xx is a failure and stays transient. The as-of capability that `?entryDate=` unlocks would fix the two weakest rungs, but it **re-baselines every control number in #279 §10**. It is therefore deliberately out of scope here and becomes a separate change diff --git a/scripts/tr/fetch-deeds.mjs b/scripts/tr/fetch-deeds.mjs new file mode 100644 index 000000000..0e14915d6 --- /dev/null +++ b/scripts/tr/fetch-deeds.mjs @@ -0,0 +1,258 @@ +// The deed crawler (issue #279 §3, ADR-0033). One request per candidate ЕИК, sequential and paced. +// +// This is the only component in the project that touches a public register at volume, so what it +// REFUSES to do is the substance: +// +// • It never goes faster than 1 request / 3 s, and the flag that sets the pace cannot be used to go +// faster — only slower. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the +// limiter is the operator's only way to state a preference, and tuning around it empirically is +// what that rule exists to prevent. +// • It never retries a 429. The block is sustained (see client.mjs), so a 429 ends the run with its +// own exit code and records NOTHING about the ЕИК that hit it — that ЕИК is unknown, not absent. +// • It never follows a link out of a deed. The candidate set is closed: whatever the caller passes +// in, nothing more. This is what keeps a bounded lookup from drifting into a crawl. +// • It only writes „outside the register" on a DOCUMENTED negative — measured to be an HTTP 200 +// with an empty body, not the 404 the issue predicts. A 5xx or a timeout is transient, and caching +// it as permanent would turn an outage into data that §8 never revisits. +// +// Resumable by construction: the cache is consulted first, so an interrupted run picks up exactly +// where it stopped and a complete cache costs zero requests. + +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { assertTrScratchIgnored, TR_DB, TR_RAW, safeEik, deedPath } from './paths.mjs'; +import { eikChecksumValid } from './eik.mjs'; +import { deedUrl, politeTrGet, RateLimitError, httpsGet } from './client.mjs'; +import { openCache, upsertDeed, markOutsideTr, pendingEiks } from './cache.mjs'; +import { + assertUicEcho, + registryLegalForm, + registrySeat, + latestOwnershipEntryDate, +} from './deed.mjs'; + +/** The documented polite pace: 1 request / 3 s (#279 §3). A floor, never a target to tune down. */ +export const MIN_INTERVAL_MS = 3000; + +/** + * Consecutive unresolved ЕИК before the run gives up rather than keep hammering. + * + * Deliberately small, because the unit is ЕИК and not requests: each unresolved candidate costs up to + * 5 attempts (#279 §3's documented retry budget), so the breaker's real cost is `BREAKER_TRIP × 5` + * requests against an endpoint that is already failing. At 10 that would be ~50 — the exact volume at + * which the register was observed to start returning a sustained 429. At 5 the worst case is ~25, + * which the same observation saw pass without a block. + */ +export const BREAKER_TRIP = 5; +/** Attempts per candidate, per #279 §3. Exported so the breaker's request budget is derivable. */ +export const TRIES_PER_EIK = 5; + +export function parseTrOptions(argv) { + const get = (name, def) => { + const i = argv.indexOf(`--${name}`); + return i >= 0 && argv[i + 1] ? argv[i + 1] : def; + }; + const posInt = (raw, name) => { + const n = Number(raw); + if (!Number.isInteger(n) || n < 1) + throw new Error(`--${name} must be a positive integer, got ${JSON.stringify(raw)}`); + return n; + }; + + const eiksFile = get('eiks-file', ''); + if (!eiksFile) throw new Error('--eiks-file is required (the closed candidate set)'); + + const limitRaw = get('limit', ''); + const intervalRaw = get('min-interval-ms', ''); + const minIntervalMs = intervalRaw ? posInt(intervalRaw, 'min-interval-ms') : MIN_INTERVAL_MS; + // Slower is always allowed; faster is not a knob. Making this un-passable is the point. + if (minIntervalMs < MIN_INTERVAL_MS) { + throw new Error( + `--min-interval-ms may not go below the documented pace of ${MIN_INTERVAL_MS}ms — ` + + `the register's rate limit is a stated preference, not an obstacle to tune around`, + ); + } + const maxAgeRaw = get('max-age-days', ''); + return { + eiksFile, + limit: limitRaw ? posInt(limitRaw, 'limit') : Infinity, + minIntervalMs, + maxAgeDays: maxAgeRaw ? posInt(maxAgeRaw, 'max-age-days') : null, + }; +} + +/** Read the closed candidate set: one ЕИК per line, blanks and `#` comments ignored. */ +export function readEiksFile(file) { + return fs + .readFileSync(file, 'utf8') + .split('\n') + .map((l) => l.trim()) + .filter((l) => l !== '' && !l.startsWith('#')); +} + +function atomicWrite(file, buf) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const tmp = `${file}.tmp-${process.pid}`; + fs.writeFileSync(tmp, buf); + fs.renameSync(tmp, file); +} + +/** + * Crawl the candidate ЕИК. Returns the intended process exit code, so the decision is testable + * without a global side effect: + * 0 — every candidate resolved (or the run was deliberately bounded by --limit) + * 1 — at least one candidate is unresolved (transient failure, refused deed, breaker tripped) + * 2 — the register rate-limited us; the run stopped and nothing was marked + * + * Every I/O edge is injectable so the whole policy is exercised offline. + */ +export async function run({ + httpGet = httpsGet, + sleep = (ms) => new Promise((r) => setTimeout(r, ms)), + now = () => new Date(), + guard = assertTrScratchIgnored, + dbFile = TR_DB, + rawDir = TR_RAW, + argv = process.argv, +} = {}) { + guard(); + const { eiksFile, limit, minIntervalMs, maxAgeDays } = parseTrOptions(argv); + + const requested = readEiksFile(eiksFile); + // A shape- or checksum-invalid code is dropped BEFORE any request: it cannot name a real company, + // so asking about it would spend the register's budget to learn nothing. + const candidates = []; + let invalid = 0; + for (const raw of requested) { + try { + const eik = safeEik(raw); + if (!eikChecksumValid(eik)) throw new Error('checksum'); + candidates.push(eik); + } catch { + invalid++; + } + } + + const db = openCache(dbFile); + try { + const pending = pendingEiks(db, candidates, { maxAgeDays, now: now() }); + const todo = Number.isFinite(limit) ? pending.slice(0, limit) : pending; + console.log( + `candidates ${candidates.length} · invalid ${invalid} · cached ${candidates.length - pending.length} · to fetch ${todo.length}`, + ); + + let unresolved = 0; + let consecutive = 0; + let first = true; + + for (const eik of todo) { + if (!first) await sleep(minIntervalMs); // pace BETWEEN requests, not before the first + first = false; + + let res; + try { + res = await politeTrGet(deedUrl(eik), { httpGet, sleep, tries: TRIES_PER_EIK }); + } catch (err) { + if (err instanceof RateLimitError) { + // Stop the whole run. Recording anything here would attribute the register's throttle to + // this ЕИК, which is a fact about us, not about the company. + console.error(`${err.message}\nSTOPPING — re-run later; progress so far is cached.`); + return 2; + } + console.error( + ` ${eik}: ${err instanceof Error ? err.message : err} (transient, not cached)`, + ); + unresolved++; + consecutive++; + if (consecutive >= BREAKER_TRIP) { + console.error(`breaker: ${consecutive} consecutive failures — aborting the run`); + return 1; + } + continue; + } + + // ── the documented negatives ──────────────────────────────────────────── + // MEASURED 2026-08-05: an ЕИК that is not a търговец answers **HTTP 200 with a ZERO-BYTE body**, + // not a 404 and not the HTML #279 §3 predicts. Verified on Община София (000696327): empty on + // two consecutive requests, while a real company returned its full 34,398-byte deed in the same + // window — so it is the register's answer, not an outage. + // + // The distinction that keeps R6 honest is the STATUS, not the empty body: an empty body under + // 200 is the register saying „no deed"; an empty body under 5xx is a failure and stays + // transient. Getting this backwards either caches a false negative forever or leaves ~4 ЕИК + // permanently unresolved so the run can never exit 0. + if (res.status === 200 && res.body.length === 0) { + markOutsideTr(db, eik, 'HTTP 200, empty body — no deed in the Търговски регистър', now()); + consecutive = 0; + continue; + } + if (res.status === 404) { + markOutsideTr(db, eik, 'HTTP 404 — not in the Търговски регистър (BULSTAT/ДЗЗД?)', now()); + consecutive = 0; + continue; + } + if (res.status !== 200) { + console.error(` ${eik}: HTTP ${res.status} after retries (transient, not cached)`); + unresolved++; + consecutive++; + if (consecutive >= BREAKER_TRIP) { + console.error(`breaker: ${consecutive} consecutive failures — aborting the run`); + return 1; + } + continue; + } + + let deed; + try { + deed = JSON.parse(res.body.toString('utf8')); + // The deed we got back must be the deed we asked for, or every claim derived from it names + // the wrong company (R8). + assertUicEcho(deed, eik); + } catch (err) { + console.error(` ${eik}: REFUSED — ${err instanceof Error ? err.message : err}`); + unresolved++; + consecutive++; + continue; + } + + // The raw response is the ONLY place names live; it stays under git-ignored scratch. + atomicWrite(deedPath(eik, rawDir), res.body); + const seat = registrySeat(deed); + upsertDeed(db, { + eik, + status: 'fetched', + httpStatus: 200, + fetchedAt: now().toISOString(), + rawPath: path.relative(rawDir, deedPath(eik, rawDir)), + bodySha256: crypto.createHash('sha256').update(res.body).digest('hex'), + legalFormCode: registryLegalForm(deed).code, + legalFormVerdict: registryLegalForm(deed).verdict, + seatNormalized: seat.settlement || null, + seatEntryDate: seat.entryDate, + latestOwnEntryDate: latestOwnershipEntryDate(deed), + }); + consecutive = 0; + } + + if (unresolved > 0) { + console.error(`${unresolved} candidate(s) unresolved — the cache is incomplete`); + return 1; + } + return 0; + } finally { + db.close(); + } +} + +// CLI entry. Kept off the import path so the module stays testable. +if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) { + run() + .then((code) => { + process.exitCode = code; + }) + .catch((err) => { + console.error(err); + process.exitCode = 1; + }); +} diff --git a/scripts/tr/fetch-deeds.test.mjs b/scripts/tr/fetch-deeds.test.mjs new file mode 100644 index 000000000..8b8296d7b --- /dev/null +++ b/scripts/tr/fetch-deeds.test.mjs @@ -0,0 +1,373 @@ +// node:test — the deed crawler, driven entirely through injected I/O. No network, no real scratch. +// +// This is the only component that touches a public register at volume, so what it must NOT do is the +// substance: never exceed the pace, never retry a 429, never turn a transient wall into permanent +// data, and never re-request what it already holds. Spec §3.3 permits a bounded per-ЕИК lookup and +// forbids bulk scraping; the difference between the two is enforced here. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { + parseTrOptions, + run, + MIN_INTERVAL_MS, + BREAKER_TRIP, + TRIES_PER_EIK, +} from './fetch-deeds.mjs'; + +const DEED = (uic) => ({ + uic, + fullName: '"АЛФА СТРОЙ" ООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: `

ИВАН ПЕТРОВ ТЕСТОВ

`, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + { + nameCode: 'CR_F_5_L', + htmlData: `

Населено място: гр. Пловдив

`, + fieldEntryNumber: '20110502101008', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], +}); + +const ok = (uic) => ({ + status: 200, + headers: {}, + body: Buffer.from(JSON.stringify(DEED(uic)), 'utf8'), +}); +const status = (s) => ({ status: s, headers: {}, body: Buffer.from('') }); + +// Two valid ЕИК (real checksums) plus one that is shape-valid but checksum-invalid. +const A = '201122335'; +const B = '203445566'; +const C = '204556676'; +const BAD_CHECKSUM = '201122336'; + +function ctx() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-crawl-')); + return { + dir, + dbFile: path.join(dir, 'tr-cache.sqlite'), + rawDir: path.join(dir, 'deeds'), + cleanup: () => fs.rmSync(dir, { recursive: true, force: true }), + }; +} +const eiksFile = (dir, eiks) => { + const f = path.join(dir, 'eiks.txt'); + fs.writeFileSync(f, eiks.join('\n') + '\n'); + return f; +}; + +/** Drive run() with a recording transport. `routes` maps ЕИК → response (or a function of attempt). */ +function harness(c, eiks, routes, extraArgv = []) { + const calls = []; + const waits = []; + const httpGet = async (url) => { + const eik = url.split('/').pop(); + calls.push(eik); + const r = routes[eik]; + return typeof r === 'function' ? r(calls.filter((e) => e === eik).length) : (r ?? status(404)); + }; + return { + calls, + waits, + promise: run({ + httpGet, + sleep: async (ms) => void waits.push(ms), + now: () => new Date('2026-08-05T12:00:00Z'), + guard: () => {}, + dbFile: c.dbFile, + rawDir: c.rawDir, + argv: ['node', 'fetch-deeds.mjs', '--eiks-file', eiksFile(c.dir, eiks), ...extraArgv], + }), + }; +} + +const openCacheRO = (dbFile) => new DatabaseSync(dbFile); +const rows = (dbFile) => { + const db = openCacheRO(dbFile); + const r = db.prepare('SELECT eik, status, outside_reason FROM deeds ORDER BY eik').all(); + db.close(); + return r; +}; + +// ── options ─────────────────────────────────────────────────────────────────── +test('parseTrOptions: defaults are the documented polite pace', () => { + const o = parseTrOptions(['node', 'x', '--eiks-file', '/tmp/e.txt']); + assert.equal(o.eiksFile, '/tmp/e.txt'); + assert.equal(o.limit, Infinity); + assert.equal(o.minIntervalMs, MIN_INTERVAL_MS); + assert.ok(MIN_INTERVAL_MS >= 3000, 'the documented pace is 1 request / 3 s'); +}); + +test('parseTrOptions: rejects a pace FASTER than the documented one', () => { + // Tuning around a limiter empirically is precisely what spec §3.3 forbids; make it un-passable. + assert.throws( + () => parseTrOptions(['node', 'x', '--eiks-file', '/e', '--min-interval-ms', '100']), + /min-interval/i, + ); +}); + +test('parseTrOptions: --eiks-file is required, and numeric flags are validated', () => { + assert.throws(() => parseTrOptions(['node', 'x']), /eiks-file/i); + for (const bad of ['0', '-1', 'abc', '1.5']) + assert.throws( + () => parseTrOptions(['node', 'x', '--eiks-file', '/e', '--limit', bad]), + /limit/i, + bad, + ); +}); + +// ── pacing ──────────────────────────────────────────────────────────────────── +test('requests are sequential and spaced by at least the documented interval', async () => { + const c = ctx(); + try { + const h = harness(c, [A, B, C], { [A]: ok(A), [B]: ok(B), [C]: ok(C) }); + assert.equal(await h.promise, 0); + assert.deepEqual(h.calls, [A, B, C], 'sequential, in order — never concurrent'); + const paces = h.waits.filter((w) => w >= MIN_INTERVAL_MS); + assert.ok(paces.length >= 2, `expected a pace wait between requests, got ${h.waits.join(',')}`); + } finally { + c.cleanup(); + } +}); + +// ── 429 ─────────────────────────────────────────────────────────────────────── +test('a 429 ends the run with exit 2 and marks NOTHING', async () => { + const c = ctx(); + try { + const h = harness(c, [A, B, C], { [A]: ok(A), [B]: status(429), [C]: ok(C) }); + assert.equal(await h.promise, 2, 'a rate-limit block is its own exit code'); + assert.deepEqual(h.calls, [A, B], 'stops AT the 429 — C is never requested'); + // B must not be recorded at all: it is unknown, not absent, and certainly not outside the register. + assert.deepEqual( + rows(c.dbFile).map((r) => r.eik), + [A], + ); + } finally { + c.cleanup(); + } +}); + +test('after a 429 the run resumes exactly where it stopped', async () => { + const c = ctx(); + try { + const first = harness(c, [A, B, C], { [A]: ok(A), [B]: status(429), [C]: ok(C) }); + assert.equal(await first.promise, 2); + const second = harness(c, [A, B, C], { [A]: ok(A), [B]: ok(B), [C]: ok(C) }); + assert.equal(await second.promise, 0); + assert.deepEqual(second.calls, [B, C], 'A is already cached — not re-requested'); + } finally { + c.cleanup(); + } +}); + +// ── resumability ────────────────────────────────────────────────────────────── +test('a re-run over a complete cache makes ZERO requests', async () => { + const c = ctx(); + try { + assert.equal(await harness(c, [A, B], { [A]: ok(A), [B]: ok(B) }).promise, 0); + const again = harness(c, [A, B], { [A]: ok(A), [B]: ok(B) }); + assert.equal(await again.promise, 0); + assert.deepEqual(again.calls, [], 'nothing to fetch'); + } finally { + c.cleanup(); + } +}); + +// ── permanence ──────────────────────────────────────────────────────────────── +test('an empty 200 is the register saying „no deed" and is cached as outside-ТР', async () => { + // MEASURED against the live API: an ЕИК that is not a търговец (Община София, 000696327) answers + // HTTP 200 with a ZERO-BYTE body — not the 404 or HTML #279 §3 predicts. Reproduced twice, with a + // real company returning its full deed in the same window, so it is an answer and not an outage. + const c = ctx(); + try { + const empty = { status: 200, headers: {}, body: Buffer.alloc(0) }; + assert.equal(await harness(c, [A], { [A]: empty }).promise, 0); + const [row] = rows(c.dbFile); + assert.equal(row.status, 'outside_tr'); + assert.match(row.outside_reason, /empty body/i); + } finally { + c.cleanup(); + } +}); + +test('an empty body under a 5xx stays TRANSIENT — the status decides, not the emptiness', async () => { + // The pair that keeps R6 honest. Both responses have a zero-byte body; only the 200 is an answer. + const c = ctx(); + try { + assert.equal(await harness(c, [A], { [A]: status(503) }).promise, 1); + assert.deepEqual(rows(c.dbFile), [], 'a 5xx must never become permanent „outside ТР"'); + } finally { + c.cleanup(); + } +}); + +test('a 404 is a DOCUMENTED negative and is cached as outside-ТР', async () => { + const c = ctx(); + try { + assert.equal(await harness(c, [A], { [A]: status(404) }).promise, 0); + const [row] = rows(c.dbFile); + assert.equal(row.status, 'outside_tr'); + assert.match(row.outside_reason, /404/); + } finally { + c.cleanup(); + } +}); + +test('a persistent 5xx is TRANSIENT and is never cached as outside-ТР', async () => { + // R6: „outside the register" is permanent by intent. Writing it after a transient wall turns a + // temporary outage into permanent data that §8 never re-examines. + const c = ctx(); + try { + const code = await harness(c, [A], { [A]: status(503) }).promise; + assert.equal(code, 1, 'an unresolved ЕИК makes the run incomplete'); + assert.deepEqual(rows(c.dbFile), [], 'nothing may be recorded from a 5xx'); + } finally { + c.cleanup(); + } +}); + +test('a deed whose UIC does not echo the request is REFUSED, not cached', async () => { + // R8, at the crawl boundary: if anything rewrote the identifier we would be caching one company's + // deed under another company's ЕИК, and every downstream claim about it would name the wrong firm. + const c = ctx(); + try { + const code = await harness(c, [A], { [A]: ok('999999999') }).promise; + assert.equal(code, 1); + assert.deepEqual(rows(c.dbFile), []); + assert.deepEqual(fs.existsSync(c.rawDir) ? fs.readdirSync(c.rawDir) : [], []); + } finally { + c.cleanup(); + } +}); + +test('a checksum-invalid ЕИК is skipped without ever being requested', async () => { + const c = ctx(); + try { + const h = harness(c, [BAD_CHECKSUM, A], { [A]: ok(A) }); + assert.equal(await h.promise, 0); + assert.deepEqual(h.calls, [A], 'the invalid code costs the register nothing'); + } finally { + c.cleanup(); + } +}); + +// ── output ──────────────────────────────────────────────────────────────────── +test('the raw deed is written under the raw dir and the index records only non-PII', async () => { + const c = ctx(); + try { + assert.equal(await harness(c, [A], { [A]: ok(A) }).promise, 0); + assert.deepEqual(fs.readdirSync(c.rawDir), [`${A}.json`]); + const db = openCacheRO(c.dbFile); + const row = db.prepare('SELECT * FROM deeds WHERE eik = ?').get(A); + db.close(); + assert.equal(row.status, 'fetched'); + assert.equal(row.legal_form_verdict, 'closely_held'); + assert.equal(row.seat_normalized, 'ПЛОВДИВ'); + assert.equal(row.body_sha256.length, 64); + // The raw file holds the names; the index must not. + assert.ok(!JSON.stringify(row).includes('ТЕСТОВ')); + assert.ok(fs.readFileSync(path.join(c.rawDir, `${A}.json`), 'utf8').includes('ТЕСТОВ')); + } finally { + c.cleanup(); + } +}); + +test('--limit bounds a run without marking the remainder as anything', async () => { + const c = ctx(); + try { + const h = harness(c, [A, B, C], { [A]: ok(A), [B]: ok(B), [C]: ok(C) }, ['--limit', '2']); + assert.equal(await h.promise, 0, 'a deliberately bounded run is not an incomplete one'); + assert.deepEqual(h.calls, [A, B]); + assert.deepEqual( + rows(c.dbFile).map((r) => r.eik), + [A, B], + ); + } finally { + c.cleanup(); + } +}); + +/** + * N genuinely checksum-valid 9-digit ЕИК. Generated, not hand-written: an earlier version of the + * breaker test below used made-up codes, of which only 1 in 10 was valid — so the crawler dropped + * them all before requesting anything and the assertion passed on zero calls. A test that exercises + * nothing is worse than no test (ADR-0027). + */ +function validEiks(n) { + const control = (p8) => { + const d = [...p8].map(Number); + let s = d.reduce((a, x, i) => a + x * (i + 1), 0) % 11; + if (s === 10) { + s = d.reduce((a, x, i) => a + x * (i + 3), 0) % 11; + if (s === 10) s = 0; + } + return s; + }; + const out = []; + for (let i = 0; out.length < n; i++) { + const p8 = String(20000000 + i); + out.push(p8 + control(p8)); + } + return out; +} + +test('the ЕИК generator used by the breaker test really produces valid codes', () => { + const eiks = validEiks(15); + assert.equal(eiks.length, 15); + assert.equal(new Set(eiks).size, 15); + // Proves the breaker test below actually reaches the network path rather than being filtered out. + assert.deepEqual( + eiks.filter((e) => e.length !== 9), + [], + ); +}); + +test('the circuit breaker aborts a sustained wall of failures', async () => { + // A long run against a broken endpoint must stop hammering, even though each individual 5xx is + // „transient". Distinct from the 429 path: that stops instantly and deliberately. + const c = ctx(); + try { + const many = validEiks(BREAKER_TRIP + 10); + const routes = Object.fromEntries(many.map((e) => [e, status(503)])); + const h = harness(c, many, routes); + assert.equal(await h.promise, 1); + + const attempted = [...new Set(h.calls)]; + assert.equal(attempted.length, BREAKER_TRIP, 'the breaker cuts the run at its threshold'); + assert.ok(attempted.length < many.length, 'and therefore short of the full candidate set'); + // The number that actually matters is REQUESTS, not candidates: each unresolved ЕИК costs the + // full retry budget, so the breaker's real cost is BREAKER_TRIP × TRIES_PER_EIK. Keep that under + // the ~50 at which the register was observed to start returning a sustained 429 — otherwise the + // safety mechanism is itself what trips the block. + assert.equal(h.calls.length, BREAKER_TRIP * TRIES_PER_EIK); + assert.ok( + h.calls.length <= 25, + `a failing run must not spend ${h.calls.length} requests before giving up`, + ); + assert.deepEqual(rows(c.dbFile), [], 'a wall of 5xx records nothing'); + } finally { + c.cleanup(); + } +}); From 800b2c1545edcdfe4a566108ddf6f7aa478f18a8 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 5 Aug 2026 23:23:21 +0300 Subject: [PATCH 07/50] =?UTF-8?q?feat(db):=20migration=200006=20=E2=80=94?= =?UTF-8?q?=20the=20Trade=20Register=20evidence=20seal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A side table, not columns on interest_links, for three reasons that are easy to forget once the code is written: SQLite's ADD COLUMN has no IF NOT EXISTS and these migrations are applied by a bare `d1 execute --file` with no ledger, so a re-apply must be a no-op; load.mjs rebuilds the CACBG tables from 0003 alone, so any column added here would have to be duplicated into 0003 and kept in step forever; and §8 describes the seal as an attached artefact of a link. A seal is written for EVERY link, not only published ones — the seals on held and withdrawn links are what make the review queue reviewable. matched_fact is a closed vocabulary ('seat:' | 'role:owner:' | 'role:manager:' | 'eik') and must never carry the matched name; the deed's names are read only to produce a boolean and never leave git-ignored scratch. A schema cannot enforce that, so the audit will. Wired everywhere the 0003 table list already is, and the two FK orderings are opposite by necessity: the seal ships AFTER interest_links and is wiped BEFORE it. D1 enforces foreign keys, so getting either backwards breaks the re-seed. • ship-related-persons.mjs — TABLES and WIPE_ORDER, with a test asserting both orderings and a third asserting every shipped table is also wiped (a table that ships but is never wiped accumulates seals for links that no longer exist). • related-persons.ts CONFLICT_TABLES — so an environment without 0006 degrades to an empty surface rather than a 500. • load.mjs — applies 0003 AND 0006, and drops the seal first. • BOTH related-persons-data.yml and deploy.yml hardcode migration filenames; each needed its own step. Missing either leaves a D1 without the table. ship-reseed.test.mjs builds its fixture from the migrations, so 0006 joins it — the new table is part of the FK graph that test exists to police. --- .github/workflows/deploy.yml | 6 +++ .github/workflows/related-persons-data.yml | 6 +++ .../0006_interest_link_evidence.sql | 41 ++++++++++++++ packages/db/src/migrations.test.ts | 54 +++++++++++++++++++ packages/db/src/queries/related-persons.ts | 3 ++ scripts/cacbg/load.mjs | 10 ++++ scripts/ship-related-persons.mjs | 4 ++ scripts/ship-related-persons.test.mjs | 21 ++++++++ scripts/ship-reseed.test.mjs | 5 ++ 9 files changed, 150 insertions(+) create mode 100644 packages/db/migrations/0006_interest_link_evidence.sql diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2c3f42ae2..f52927c9d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -145,6 +145,12 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0003_related_persons_foundation.sql + # 0006 attaches the Trade Register evidence seal (#279, ADR-0033). Both migrations are + # applied by name here because wrangler's migration ledger is empty on this D1 (the base + # schema was created out-of-band), so `d1 migrations apply` would collide on 0000. + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0006_interest_link_evidence.sql # The base schema was created out-of-band with `d1 execute --file`, so wrangler's migration # ledger is empty and `d1 migrations apply` would collide on 0000. Probe the actual table # instead. SQLite has no `ADD COLUMN IF NOT EXISTS`; a completion-marker table is created only diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index deb24884b..fa84f21ef 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -185,6 +185,12 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0003_related_persons_foundation.sql + # 0006 attaches the Trade Register evidence seal (#279, ADR-0033). Both migrations are + # applied by name here because wrangler's migration ledger is empty on this D1 (the base + # schema was created out-of-band), so `d1 migrations apply` would collide on 0000. + pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0006_interest_link_evidence.sql - name: Ship свързани-лица tables → D1 # ship-related-persons.mjs shells out to `wrangler d1 execute` with cwd=apps/web. Two things it diff --git a/packages/db/migrations/0006_interest_link_evidence.sql b/packages/db/migrations/0006_interest_link_evidence.sql new file mode 100644 index 000000000..a7b83033c --- /dev/null +++ b/packages/db/migrations/0006_interest_link_evidence.sql @@ -0,0 +1,41 @@ +-- Trade Register evidence seal for a свързани-лица link (#279 §8, ADR-0033). +-- +-- WHY A SIDE TABLE and not columns on interest_links, since both were on the table: +-- 1. SQLite's ALTER TABLE ... ADD COLUMN has no IF NOT EXISTS, and migrations here are applied by a +-- bare `wrangler d1 execute --file` with no applied-migrations tracking (see the workflow steps), +-- so a re-apply MUST be a no-op. `CREATE TABLE IF NOT EXISTS` is; ADD COLUMN is not. +-- 2. scripts/cacbg/load.mjs rebuilds the CACBG tables from 0003 ALONE. A column added here would have +-- to be duplicated into 0003 and kept in step with it forever, and the day the two diverge the +-- loader silently drops the evidence for every link it writes. +-- 3. §8 describes the seal as an attached artefact of a link, which is what this is. +-- +-- A seal is written for EVERY link, not only published ones. The seals on held and withdrawn links are +-- what make the review queue reviewable — without them „why is this one hidden?" has no answer. +-- +-- PII rail: `matched_fact` is a CLOSED VOCABULARY — 'seat:' | 'role:owner:' | +-- 'role:manager:' | 'eik'. It must NEVER carry the matched name. The registry deed's names are +-- read only to produce a boolean and never leave git-ignored scratch (ADR-0033 decision 5); storing one +-- here would put third-party personal data on the served surface, which #279 §9 forbids outright. The +-- audit enforces the vocabulary with a pattern check, because a schema cannot. + +CREATE TABLE IF NOT EXISTS interest_link_evidence ( + link_key TEXT PRIMARY KEY REFERENCES interest_links(link_key), + -- document | confirmed | refuted | bar_joint_stock | unknown | outside_tr (ADR-0033 decision 1) + evidence_kind TEXT NOT NULL, + registry_role TEXT, -- owner | manager | NULL — only meaningful for evidence_kind='document' + matched_fact TEXT, -- the closed vocabulary above. NEVER a name. + -- TEXT, not INTEGER: a fieldEntryNumber like 20130716101007 is already 14 digits and exceeds the + -- exact-integer range once it round-trips through JSON/JS. + entry_number TEXT, + entry_date TEXT, -- ISO date of the registry entry the evidence rests on + lookup_date TEXT NOT NULL, -- when the deed was fetched — the freshness bound on the claim + rules_version TEXT NOT NULL, -- evidence.mjs RULES_VERSION; §8's monotonicity gate keys on this + -- live | terminated_owner_still | terminated_manager_still | terminated. + -- RE-DERIVED on every run and deliberately NOT part of the seal's permanence: it asserts a present + -- tense about a named person, and its freshness is bounded by the cache refresh cycle (ADR-0033 R3). + live_status TEXT NOT NULL, + sealed_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- The surface filters on evidence_kind ('document' | 'confirmed' publish); the audit scans by kind too. +CREATE INDEX IF NOT EXISTS idx_ile_kind ON interest_link_evidence(evidence_kind); diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index e5296aaa9..b92c174b4 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -11,6 +11,7 @@ const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); const migration2 = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); const backfill = resolve(root, 'scripts/backfill-current-value-currency.sql'); const precompute = resolve(root, 'scripts/precompute.sql'); @@ -150,4 +151,57 @@ describe('served migrations', () => { rmSync(dir, { recursive: true, force: true }); } }); + + // 0006 attaches the Trade Register evidence seal to a link (#279, ADR-0033). A SIDE TABLE rather than + // columns on interest_links, for two reasons that are easy to forget: SQLite's ADD COLUMN has no + // IF NOT EXISTS and migrations are applied by a bare `d1 execute --file` with no tracking, so a + // re-apply must be a no-op; and load.mjs rebuilds the CACBG tables from 0003 alone, so any column + // added here would have to be duplicated into 0003 and kept in step forever. + it('0006 adds the evidence seal and re-applying it is a no-op', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-0006-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + for (const m of [migration0, migration1, migration2, migration3, migration6]) + readScript(dbPath, m); + + expect( + sqlite( + dbPath, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='interest_link_evidence';", + ).trim(), + ).toBe('1'); + + // A seal for a real link survives a second apply — the migration must not drop and recreate. + sqlite( + dbPath, + `INSERT INTO persons (id, name) VALUES ('person:a', 'Иван Петров Тестов'); + INSERT INTO bidders (id, name, eik_normalized, eik_valid) + VALUES ('eik:201122335', 'АЛФА СТРОЙ ООД', '201122335', 1); + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:k', 'k', 'person:a', 'eik:201122335', '201122335', 'АЛФА СТРОЙ ООД', + 'test', 'document', 'owns', 'private_ownership', 'published'); + INSERT INTO interest_link_evidence + (link_key, evidence_kind, matched_fact, lookup_date, rules_version, live_status) + VALUES ('k', 'document', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live');`, + ); + readScript(dbPath, migration6); // idempotent re-apply + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_link_evidence;').trim()).toBe('1'); + + // The FK is real: a seal for a link that does not exist is rejected. D1 enforces foreign keys, + // so this is what stops the ship path inserting seals before (or after wiping) their links. + expect(() => + sqlite( + dbPath, + `PRAGMA foreign_keys=ON; + INSERT INTO interest_link_evidence + (link_key, evidence_kind, matched_fact, lookup_date, rules_version, live_status) + VALUES ('nope', 'document', 'eik', '2026-08-05', 'tr-rules-1', 'live');`, + ), + ).toThrow(/FOREIGN KEY/i); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/packages/db/src/queries/related-persons.ts b/packages/db/src/queries/related-persons.ts index 05c3f228f..ce15ff2b6 100644 --- a/packages/db/src/queries/related-persons.ts +++ b/packages/db/src/queries/related-persons.ts @@ -19,6 +19,9 @@ const CONFLICT_TABLES = [ 'declared_interests', 'interest_link_authorities', 'related_persons_internal', + // 0006 (#279, ADR-0033). Listed here for the same reason as the rest: on an environment where 0006 + // has not been applied yet, the evidence join must degrade to an empty surface rather than a 500. + 'interest_link_evidence', ]; // „D1_ERROR: no such table: interest_links: SQLITE_ERROR" → capture the table name and test membership. const MISSING_TABLE = /no such table:\s*(?:main\.)?"?([a-z_]+)"?/i; diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 4dc360b28..984ead206 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -30,6 +30,13 @@ const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '. const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); const STAGING = process.env.CACBG_STAGING || path.join(ROOT, 'scratch/cacbg/staging'); const MIGRATION = path.join(ROOT, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// 0006 attaches the Trade Register evidence seal (#279, ADR-0033). Applied here as well as 0003 +// because this loader rebuilds the CACBG tables from the migrations on every run — a seal table +// missing from the work DB would make every evidence write fail at ship time instead of at load. +const MIGRATION_EVIDENCE = path.join( + ROOT, + 'packages/db/migrations/0006_interest_link_evidence.sql', +); const REPORT = path.join(STAGING, 'findings.md'); const MATCHER_VERSION = 'cnk-1+classify-1'; // bump when the normalizer or classify logic changes const { companyNameKey, isMatchableKey } = @@ -71,6 +78,8 @@ const usedSuppressions = new Set(); // Full idempotent rebuild that also picks up schema changes: drop the CACBG tables (children first — // FK-safe) and re-apply the migration. Nothing to preserve — suppressions are external now. for (const t of [ + // FIRST: interest_link_evidence references interest_links, so it must go before its parent. + 'interest_link_evidence', 'interest_link_authorities', 'interest_links', 'declared_interests', @@ -80,6 +89,7 @@ for (const t of [ ]) db.exec(`DROP TABLE IF EXISTS ${t}`); db.exec(fs.readFileSync(MIGRATION, 'utf8')); +db.exec(fs.readFileSync(MIGRATION_EVIDENCE, 'utf8')); // A link is suppressed when its fingerprint is in the list. Only compute the HMAC when the list is // non-empty (size>0 ⇒ salt present, else the loader above threw), so the empty common path skips crypto. const isSuppressed = (linkKey) => { diff --git a/scripts/ship-related-persons.mjs b/scripts/ship-related-persons.mjs index 54da33deb..6ecd1e59a 100644 --- a/scripts/ship-related-persons.mjs +++ b/scripts/ship-related-persons.mjs @@ -28,6 +28,8 @@ export const TABLES = [ 'declarations', 'declared_interests', 'interest_links', + // AFTER interest_links: a seal references its link, so inserting it first fails the FK (#279). + 'interest_link_evidence', 'interest_link_authorities', ]; // DELETE order for the pre-insert wipe — children before parents. related_persons_internal (PII, never @@ -35,6 +37,8 @@ export const TABLES = [ // carrying internal rows would block DELETE FROM declarations. export const WIPE_ORDER = [ 'interest_link_authorities', + // BEFORE interest_links, for the mirror reason: deleting a link whose seal survives fails the FK. + 'interest_link_evidence', 'related_persons_internal', 'interest_links', 'declared_interests', diff --git a/scripts/ship-related-persons.test.mjs b/scripts/ship-related-persons.test.mjs index b15612d13..76e8c18ad 100644 --- a/scripts/ship-related-persons.test.mjs +++ b/scripts/ship-related-persons.test.mjs @@ -10,6 +10,7 @@ import { sqlLiteral, sqlIdent, TABLES, + WIPE_ORDER, } from './ship-related-persons.mjs'; test('sqlLiteral escapes quotes, strips NUL, and NULLs non-finite/absent', () => { @@ -70,11 +71,31 @@ test('TABLES ships parents before children and covers the served related-persons 'declared_interests', 'interest_links', 'interest_link_authorities', + 'interest_link_evidence', ]) { assert.ok(TABLES.includes(t), `missing ${t}`); } }); +test('the evidence seal ships AFTER the links it references, and is wiped BEFORE them', () => { + // D1 enforces foreign keys, so ordering is not cosmetic: inserting a seal before its link fails, + // and deleting a link while a seal still references it fails the re-seed at interest_links. + assert.ok( + TABLES.indexOf('interest_link_evidence') > TABLES.indexOf('interest_links'), + 'a seal inserted before its link violates the FK', + ); + assert.ok( + WIPE_ORDER.indexOf('interest_link_evidence') < WIPE_ORDER.indexOf('interest_links'), + 'a link deleted while its seal survives violates the FK', + ); +}); + +test('every shipped table is wiped, and every wiped table is real', () => { + // A table added to TABLES but forgotten in WIPE_ORDER accumulates stale rows on every re-ship — + // the surface would then carry evidence for links that no longer exist. + for (const t of TABLES) assert.ok(WIPE_ORDER.includes(t), `${t} ships but is never wiped`); +}); + test('assertD1TargetAuthorized: declared env + allowlisted name + (name↔id) before a remote wipe (T48)', () => { const ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'; const ok = { diff --git a/scripts/ship-reseed.test.mjs b/scripts/ship-reseed.test.mjs index 4864029dd..9c5b66172 100644 --- a/scripts/ship-reseed.test.mjs +++ b/scripts/ship-reseed.test.mjs @@ -15,6 +15,9 @@ import { wipeSql, TABLES } from './ship-related-persons.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); const MIG = resolve(HERE, '..', 'packages/db/migrations/0003_related_persons_foundation.sql'); +// 0006 adds interest_link_evidence, which references interest_links — so it is part of the FK graph +// this test exists to police, and the wipe must clear it before its parent (#279, ADR-0033). +const MIG_EVIDENCE = resolve(HERE, '..', 'packages/db/migrations/0006_interest_link_evidence.sql'); // Stub only the two 0000 tables 0002's FKs reference (bidders, authorities) — a PK column is all a FK needs — // then apply the real 0002 and seed one row down every FK chain so a parent delete has live children. @@ -25,6 +28,7 @@ function newPopulatedDb() { CREATE TABLE bidders(id TEXT PRIMARY KEY); CREATE TABLE authorities(id TEXT PRIMARY KEY); .read ${MIG} +.read ${MIG_EVIDENCE} INSERT INTO bidders(id) VALUES('eik:1'); INSERT INTO authorities(id) VALUES('auth:1'); INSERT INTO persons(id,name) VALUES('p1','П Тест'); @@ -33,6 +37,7 @@ INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VAL INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('il1','p1|1','p1','eik:1','1','e','v1','B_distinctive','owns','published'); INSERT INTO interest_link_authorities(link_key,authority_id,authority_name) VALUES('p1|1','auth:1','A'); INSERT INTO related_persons_internal(id,declaration_id,related_name,related_kind) VALUES('rp1','d1','X','related_person'); +INSERT INTO interest_link_evidence(link_key,evidence_kind,matched_fact,lookup_date,rules_version,live_status) VALUES('p1|1','document','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'); `; execFileSync('sqlite3', ['-bail', db], { input: setup, stdio: 'pipe' }); return { dir, db }; From 5f64f5cfda40c2a6bf65e2ece7e2e9ad6bf62249 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 07:56:13 +0300 Subject: [PATCH 08/50] =?UTF-8?q?fix(cacbg):=20closelyHeldForm=20did=20not?= =?UTF-8?q?=20exclude=20=D0=9A=D0=94=D0=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit КДА — командитно дружество с акции — issues shares like an АД, so the same two reasons apply: the shareholder book is not public, making a declared parcel unverifiable, and the parcel may be immaterial. It was absent from both JOINT_STOCK and FORM_TOKENS, so a КДА read as CLOSELY HELD and its declared holder could reach the public ownership surface as an owner — the exact class of claim the exclusion exists to prevent. Two independent effects, hence both lists: JOINT_STOCK is the materiality bar, and FORM_TOKENS feeds the content-word count in nameDistinctiveness, where a form token miscounted as a content word inflates distinctiveness toward publishing. Found while implementing #279, whose rung 1 names КДА explicitly alongside АД and ЕАД, but the gap predates it and is a bug on its own — hence a separate commit. The mirror cases stay correct: „КДА-ТРЕЙД ЕООД" and „КДА ГРУП ООД" are ООД. --- scripts/cacbg/classify.mjs | 8 +++++--- scripts/cacbg/classify.test.mjs | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/scripts/cacbg/classify.mjs b/scripts/cacbg/classify.mjs index fe979345f..f8e7dcd72 100644 --- a/scripts/cacbg/classify.mjs +++ b/scripts/cacbg/classify.mjs @@ -16,6 +16,7 @@ const FORM_TOKENS = new Set([ 'КД', 'СД', 'АДСИЦ', + 'КДА', 'КООПЕРАЦИЯ', 'ФОНДАЦИЯ', 'СДРУЖЕНИЕ', @@ -71,15 +72,16 @@ const norm = (s) => .replace(/[\s.\-–—]+/g, ' ') .trim(); -// Joint-stock / listed legal form (АД / ЕАД / АДСИЦ) as the TRAILING form token. In BG company names the +// Joint-stock / share-issuing legal form (АД / ЕАД / АДСИЦ / КДА) as the TRAILING form token. In BG company names the // legal form is always the suffix, so anchor to the end (optionally followed by quotes/whitespace); a whole // token bounded on the left by string edge, whitespace or quotes — NOT hyphens/dots, so „АД-ХОК ЕООД" (a // hyphenated ООД name) is not misread. Anchoring to the suffix is what stops „АД ГРУП ООД" (an ООД whose // NAME begins with the token „АД") being wrongly excluded as joint-stock — the form there is ООД. -const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ)[\s"„“”«»]*$/u; +const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; /** * Materiality by legal form. The public ownership surface is CLOSELY-HELD companies only (ООД/ЕООД/ЕТ/ - * КД/СД/ДЗЗД or a form-unspecified name from the closely-held table). Joint-stock forms (АД/ЕАД/АДСИЦ) are + * КД/СД/ДЗЗД or a form-unspecified name from the closely-held table). Joint-stock forms (АД/ЕАД/АДСИЦ/КДА, + * the last a командитно дружество с акции — it issues shares, so it belongs with them) are * public-float securities — a declared parcel of listed shares is NOT a material ownership conflict, and * presenting it as one defames (the „11 Trace shares → €88M" trap). Excludes only an explicit АД-form token, * so it withholds rather than fabricates. @returns {boolean} true ⇒ material/closely-held. diff --git a/scripts/cacbg/classify.test.mjs b/scripts/cacbg/classify.test.mjs index 7d2bddd15..b62d3c7e9 100644 --- a/scripts/cacbg/classify.test.mjs +++ b/scripts/cacbg/classify.test.mjs @@ -71,6 +71,25 @@ test('closelyHeldForm: ООД/ЕООД/ЕТ material; АД/ЕАД/АДСИЦ (li assert.equal(closelyHeldForm('АД СТИЛ ЕООД'), true); }); +test('closelyHeldForm: КДА (командитно дружество с акции) is joint-stock and must be excluded', () => { + // КДА issues shares like an АД — the shareholder book is not public, so a declared parcel is neither + // verifiable nor necessarily material, which is the whole basis of the exclusion. It was missing from + // both JOINT_STOCK and FORM_TOKENS, so a КДА read as closely-held and its holder could be published + // as an owner. #279 rung 1 names it explicitly alongside АД and ЕАД. + assert.equal(closelyHeldForm('ФИНАНС КДА'), false); + assert.equal(closelyHeldForm('"АЛФА ИНВЕСТ" КДА'), false); + assert.equal(closelyHeldForm('АЛФА КДА, гр. София'), false); // seat suffix must not rescue it + // …and the mirror: „КДА" inside a word or as a leading token is not the form. + assert.equal(closelyHeldForm('КДА-ТРЕЙД ЕООД'), true); + assert.equal(closelyHeldForm('КДА ГРУП ООД'), true); +}); + +test('nameDistinctiveness: КДА counts as a legal form, not a content word', () => { + // FORM_TOKENS feeds the content-word count. A form token counted as content inflates distinctiveness, + // which is the direction that publishes prematurely. + assert.equal(nameDistinctiveness('ФИНАНС КДА'), 'generic'); +}); + test('closelyHeldForm: a trailing седалище after the form does not flip an АД to closely-held (libel)', () => { // The declarant appended the seat to the name cell. Without stripping it, the end-anchored form test // misses the АД and returns closely-held=true → a listed-АД parcel presented as a material conflict. From 4226ae348437d5e0d33e88c3414d43a892b0ad53 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 08:10:06 +0300 Subject: [PATCH 09/50] feat(cacbg): the evidence ladder replaces the publish tiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decision swap (#279, ADR-0033). load.mjs no longer asks whether a declared name looks distinctive; it asks what the Trade Register says about that ЕИК. publishTier() and seatConfirmed() are deleted — one production caller each — and publish_tier now carries the evidence kind. FAIL CLOSED, twice. A missing cache refuses the load outright. A PARTIAL cache refuses it too, and that is the one that matters: it does not fail loudly downstream. An 80%-restored cache yields ~80 published links, which clears the ship floor of 50, ships a decimated surface and wipes the rest of the live links. --allow-partial-tr is the deliberate override, and an uncached ЕИК under it is „unknown" (held), never a reason to publish. The workflow now passes --min-links 250 explicitly rather than relying on the default of 50. tr-census.mjs is deleted. Its promote() moved C_hold → published on pure name uniqueness — identity by name coincidence, which is what this change abolishes. Two design errors the fixtures caught, both about how far ADR-0017's gate reaches: • Gating the declared-ЕИК leg on name uniqueness would discard the strongest identifier we have exactly where the name is useless — a фирма backing two ЕИК is the case ADR-0028 was written for. Now ungated. • Gating the seat leg on nameDistinctiveness would empty the rung of its purpose: rescuing a GENERIC name is what the seat is for. Only national non-uniqueness blocks it. A consequence worth stating plainly: #279 NARROWS the family surface. The registered owner of a family stake is the relative, whose name we deliberately never store, so rung 2 („Документ") can never fire for a family link by construction — its identity can only be confirmed by something the OFFICIAL declared, the seat or the ЕИК. ADR-0032's decision is untouched (family publishes on the named surface exactly like self); it now needs the same evidence as everything else. The fixture's seat-less family case is therefore held, and a seat-carrying one was added as the positive control, without which „family published: 0" would be indistinguishable from a dead path. Fixture declarants gained patronymics. Bulgarian names are three-part by statute (ЗГР чл. 9) and rung 2 requires the full triple, so two-token fixture names were not merely unrealistic — they could never earn „Документ" and were quietly testing the wrong path. A seal is written for every link, held and withdrawn included, so the review queue can explain itself; live_status is re-derived each run and never sealed. load.mjs also emits candidate-eiks.txt — every resolved ЕИК, not just published ones — which is the crawler's closed input set. --- .github/workflows/related-persons-data.yml | 9 +- .github/workflows/scripts-test.yml | 2 +- scripts/cacbg/classify.mjs | 26 +- scripts/cacbg/classify.test.mjs | 15 - scripts/cacbg/load-collision.test.mjs | 75 +++- .../cacbg/load-divestment-nonwinner.test.mjs | 92 +++- scripts/cacbg/load-homonym.test.mjs | 92 +++- scripts/cacbg/load.mjs | 194 ++++++++- scripts/cacbg/load.test.mjs | 392 +++++++++++++++--- scripts/cacbg/tr-census.mjs | 195 --------- scripts/cacbg/tr-census.test.mjs | 121 ------ scripts/tr/evidence.mjs | 11 +- scripts/tr/evidence.test.mjs | 32 ++ 13 files changed, 811 insertions(+), 445 deletions(-) delete mode 100644 scripts/cacbg/tr-census.mjs delete mode 100644 scripts/cacbg/tr-census.test.mjs diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index fa84f21ef..16f35c7e1 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -204,7 +204,14 @@ jobs: node scripts/wrangler-render.mjs apps/web/wrangler.jsonc cp apps/web/wrangler.deploy.jsonc apps/web/wrangler.jsonc export PATH="$GITHUB_WORKSPACE/apps/web/node_modules/.bin:$GITHUB_WORKSPACE/node_modules/.bin:$PATH" - node scripts/ship-related-persons.mjs --work-db data/work/backfill.sqlite --remote --yes + # --min-links is passed EXPLICITLY, not left at its default of 50. Under #279 the surface is + # measured at ~329 links, so 50 no longer protects anything: a partially-restored Trade + # Register cache yields roughly 80 published links, which clears a floor of 50, ships a + # decimated surface and wipes the rest. The loader's coverage gate is the first defence and + # this is the second (ADR-0033 decision 7). Lower it deliberately, in a PR, if a genuinely + # smaller surface is ever expected. + node scripts/ship-related-persons.mjs --work-db data/work/backfill.sqlite --remote --yes \ + --min-links 250 - name: Reindex search projection → officials searchable (same batch the cron runs) # ship writes ONLY the свързани-лица domain tables (interest_links, persons, …), NOT search_index. diff --git a/.github/workflows/scripts-test.yml b/.github/workflows/scripts-test.yml index d3112c48f..81dd4ec45 100644 --- a/.github/workflows/scripts-test.yml +++ b/.github/workflows/scripts-test.yml @@ -28,7 +28,7 @@ jobs: # so no `pnpm install` is required. - run: node --test scripts/*.test.mjs - # свързани-лица pipeline tests — the CACBG leg (parse/classify/load/audit/tr-census/extract) and the + # свързани-лица pipeline tests — the CACBG leg (parse/classify/load/audit/extract) and the # Търговски регистър leg (scripts/tr: ЕИК checksum, HTTP client, deed cache). These use node:sqlite # (DatabaseSync) and import the shared companyNameKey .ts via the register-ts resolve hook, so they # need Node 24 (node:sqlite + native TS type-stripping, no experimental flag) rather than the Node 22 diff --git a/scripts/cacbg/classify.mjs b/scripts/cacbg/classify.mjs index f8e7dcd72..8bc07bceb 100644 --- a/scripts/cacbg/classify.mjs +++ b/scripts/cacbg/classify.mjs @@ -96,26 +96,12 @@ export function closelyHeldForm(name) { ); } -/** Seat proof: declared seat and winner settlement both present and equal ⇒ same entity (deterministic). */ -export function seatConfirmed(declSeat, winnerSettlement) { - const a = norm(declSeat); - const b = norm(winnerSettlement); - return a.length > 0 && b.length > 0 && a === b; -} - -/** - * Publish tier for a single-winner-ЕИК match: - * 'A_eik' — a declarant-provided ЕИК resolved the winner: the national unique identifier, - * deterministic even behind a generic/colliding name. Assigned by the loader for - * declared_eik matches (not this function — it needs the match method); ADR-0016. - * 'A_seat' — seat-confirmed: deterministic, publishable even for generic names. - * 'B_distinctive' — single-ЕИК + structurally distinctive name: publishable (disclosed heuristic). - * 'C_hold' — generic name, no seat/ЕИК proof: withhold pending TR name-census. - */ -export function publishTier({ seatOk, distinctiveness }) { - if (seatOk) return 'A_seat'; - return distinctiveness === 'distinctive' ? 'B_distinctive' : 'C_hold'; -} +// seatConfirmed() and publishTier() lived here until #279. The publish tiers they produced +// (A_seat / B_distinctive / C_hold) are superseded by the Trade Register evidence ladder in +// scripts/tr/evidence.mjs — identity now rests on a checkable registry fact rather than on the shape +// of the declared name (ADR-0033, superseding ADR-0009). The seat comparison moved with it, because a +// declared seat is now matched against the REGISTERED seat rather than against the winner row's +// settlement column. nameDistinctiveness above survives, narrowed to an AND-gate on the weakest rung. /** * Temporal relation of a contract to the years a stake was declared (asset decls are annual snapshots). diff --git a/scripts/cacbg/classify.test.mjs b/scripts/cacbg/classify.test.mjs index b62d3c7e9..e0e8b2fa1 100644 --- a/scripts/cacbg/classify.test.mjs +++ b/scripts/cacbg/classify.test.mjs @@ -2,8 +2,6 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { nameDistinctiveness, - seatConfirmed, - publishTier, temporalStatus, localityToken, closelyHeldForm, @@ -32,19 +30,6 @@ test('nameDistinctiveness: numbers / Latin / ≥3 words are distinctive; bare 1- assert.equal(nameDistinctiveness('„ДОМИНО" ЕООД'), 'generic'); // quoted single word + form }); -test('seatConfirmed: equal non-empty seats confirm; empty or mismatched do not', () => { - assert.equal(seatConfirmed('Шумен', 'ШУМЕН'), true); - assert.equal(seatConfirmed('София', 'Пловдив'), false); - assert.equal(seatConfirmed('', 'София'), false); // sparse winner/declared seat never confirms - assert.equal(seatConfirmed('София', ''), false); -}); - -test('publishTier: seat proof wins; else distinctiveness decides publish vs hold', () => { - assert.equal(publishTier({ seatOk: true, distinctiveness: 'generic' }), 'A_seat'); - assert.equal(publishTier({ seatOk: false, distinctiveness: 'distinctive' }), 'B_distinctive'); - assert.equal(publishTier({ seatOk: false, distinctiveness: 'generic' }), 'C_hold'); -}); - test('temporalStatus: contract within declared-year span is contemporaneous', () => { assert.equal(temporalStatus([2020, 2021, 2022], 2021), 'contemporaneous'); assert.equal(temporalStatus([2020, 2022], 2024), 'after_last_decl'); diff --git a/scripts/cacbg/load-collision.test.mjs b/scripts/cacbg/load-collision.test.mjs index 32991193b..8d709ec28 100644 --- a/scripts/cacbg/load-collision.test.mjs +++ b/scripts/cacbg/load-collision.test.mjs @@ -16,7 +16,7 @@ import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad() { execFileSync( @@ -24,15 +24,81 @@ function runLoad() { ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], { cwd: ROOT, - env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + }, stdio: 'pipe', }, ); } + +/** + * Minimal Trade Register evidence for this fixture (#279, ADR-0033). Publishing now rests on a registry + * fact, so a loader test without a cache would only ever exercise the fail-closed path. Each winner's + * deed names its own declarant as съдружник, which is the „Документ" rung. + */ +function buildTrCache(owners) { + fs.mkdirSync(TR_RAW, { recursive: true }); + const cache = new DatabaseSync(TR_DB); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + for (const [eik, name] of Object.entries(owners)) { + const deed = { + uic: eik, + fullName: '"ФИКС" ЕООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: `

${name}

`, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], + }; + fs.writeFileSync(path.join(TR_RAW, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code,legal_form_verdict,latest_own_entry_date) VALUES(?,?,?,?,?,?,?,?)', + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + 4, + 'closely_held', + '2011-05-02', + ); + } + cache.close(); +} const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-collision-')); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); fs.mkdirSync(STAGING, { recursive: true }); @@ -92,6 +158,11 @@ before(() => { holdings.map((h) => JSON.stringify(h)).join('\n') + '\n', ); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + + buildTrCache({ + 100000001: 'ИВАН ПЪРВИ ТЕСТОВ', + 200000002: 'ПЕТЪР ВТОРИ ПРОБЕН', + }); }); after(() => fs.rmSync(dir, { recursive: true, force: true })); diff --git a/scripts/cacbg/load-divestment-nonwinner.test.mjs b/scripts/cacbg/load-divestment-nonwinner.test.mjs index 226be07c4..fac967b58 100644 --- a/scripts/cacbg/load-divestment-nonwinner.test.mjs +++ b/scripts/cacbg/load-divestment-nonwinner.test.mjs @@ -17,7 +17,7 @@ import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad() { execFileSync( @@ -25,15 +25,86 @@ function runLoad() { ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], { cwd: ROOT, - env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + }, stdio: 'pipe', }, ); } + +/** + * Minimal Trade Register evidence for this fixture (#279, ADR-0033). Publishing now rests on a registry + * fact, so a loader test without a cache would only ever exercise the fail-closed path. Each winner's + * deed names its own declarant as съдружник — the „Документ" rung. + */ +function buildTrCache(owners) { + fs.mkdirSync(TR_RAW, { recursive: true }); + const cache = new DatabaseSync(TR_DB); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + for (const [eik, names] of Object.entries(owners)) { + const html = [] + .concat(names) + .map((n) => `

${n}

`) + .join(`
`); + const deed = { + uic: eik, + fullName: '"ФИКС" ЕООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: html, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], + }; + fs.writeFileSync(path.join(TR_RAW, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code,legal_form_verdict,latest_own_entry_date) VALUES(?,?,?,?,?,?,?,?)', + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + 4, + 'closely_held', + '2011-05-02', + ); + } + cache.close(); +} + const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-divest-')); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); fs.mkdirSync(STAGING, { recursive: true }); @@ -64,7 +135,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Диан Дивестов', + person: 'Диан Иванов Дивестов', position: '', entity: 'ДИВ ТЕХ 5 ЕООД', kind: 'shares', @@ -80,7 +151,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Диан Дивестов', + person: 'Диан Иванов Дивестов', position: '', entity: 'НЕПОБЕДИМ КОМПАНИ ООД', kind: 'shares', @@ -99,7 +170,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Верен Държателев', + person: 'Верен Иванов Държателев', position: '', entity: 'ДРУГ ВИН 6 ЕООД', kind: 'shares', @@ -114,6 +185,13 @@ before(() => { holdings.map((h) => JSON.stringify(h)).join('\n') + '\n', ); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + + buildTrCache({ + // Диан DIVESTED, so the live deed must name somebody else — otherwise §7's reconciliation + // correctly overturns his declared termination and the case stops testing what it is for. + 100000001: 'НОВ ИВАНОВ СОБСТВЕНИК', + 200000002: 'ВЕРЕН ИВАНОВ ДЪРЖАТЕЛЕВ', + }); // filings.jsonl — one record per declaration (as extract.mjs emits it), carrying the declaration type. The // divest horizon is built from this: Диан's 2022 assets declaration (listing only the non-winner) advances // his assets horizon to 2022 → the 2019 ДИВ ТЕХ 5 winner stake is withdrawn. Верен has only a 2019 filing. @@ -143,8 +221,8 @@ test('a later NON-winner ownership filing still withdraws a divested winner stak ) .get(eik, person); - const dian = link('100000001', 'Диан Дивестов'); - const veren = link('200000002', 'Верен Държателев'); + const dian = link('100000001', 'Диан Иванов Дивестов'); + const veren = link('200000002', 'Верен Иванов Държателев'); // The divested winner stake is dated to its last declaration and excluded from the public surface. assert.equal(dian.status, 'withdrawn'); diff --git a/scripts/cacbg/load-homonym.test.mjs b/scripts/cacbg/load-homonym.test.mjs index 395b62edb..7f38b12bf 100644 --- a/scripts/cacbg/load-homonym.test.mjs +++ b/scripts/cacbg/load-homonym.test.mjs @@ -18,19 +18,94 @@ import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad() { execFileSync( 'node', ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], - { cwd: ROOT, env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, stdio: 'pipe' }, + { + cwd: ROOT, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + }, + stdio: 'pipe', + }, ); } + +/** + * Minimal Trade Register evidence for this fixture (#279, ADR-0033). Publishing now rests on a registry + * fact, so a loader test without a cache would only ever exercise the fail-closed path. Each winner's + * deed names its own declarant as съдружник — the „Документ" rung. + */ +function buildTrCache(owners) { + fs.mkdirSync(TR_RAW, { recursive: true }); + const cache = new DatabaseSync(TR_DB); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + for (const [eik, names] of Object.entries(owners)) { + const html = [] + .concat(names) + .map((n) => `

${n}

`) + .join(`
`); + const deed = { + uic: eik, + fullName: '"ФИКС" ЕООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: html, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], + }; + fs.writeFileSync(path.join(TR_RAW, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code,legal_form_verdict,latest_own_entry_date) VALUES(?,?,?,?,?,?,?,?)', + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + 4, + 'closely_held', + '2011-05-02', + ); + } + cache.close(); +} + const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-homonym-')); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); fs.mkdirSync(STAGING, { recursive: true }); @@ -59,7 +134,7 @@ before(() => { template: 'assets', category: '', institution: 'ОБЩИНА СОФИЯ', - person: 'Георги Иванов', + person: 'Георги Иванов Петров', position: 'Кмет', entity: 'ВИН ЕДНО 5 ЕООД', kind: 'shares', @@ -77,7 +152,7 @@ before(() => { template: 'assets', category: '', institution: 'МИНИСТЕРСТВО НА ТЕСТА', - person: 'Георги Иванов', + person: 'Георги Иванов Петров', position: 'Директор', entity: 'ВИН ДВЕ 6 ЕООД', kind: 'shares', @@ -95,7 +170,7 @@ before(() => { template: 'assets', category: '', institution: 'ОБЩИНА СОФИЯ', - person: 'Георги Иванов', + person: 'Георги Иванов Петров', position: 'Кмет', entity: 'ВИН ЕДНО 5 ЕООД', kind: 'shares', @@ -110,6 +185,11 @@ before(() => { holdings.map((h) => JSON.stringify(h)).join('\n') + '\n', ); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + + buildTrCache({ + 100000001: 'ГЕОРГИ ИВАНОВ ПЕТРОВ', + 200000002: 'ГЕОРГИ ИВАНОВ ПЕТРОВ', + }); }); after(() => fs.rmSync(dir, { recursive: true, force: true })); @@ -120,7 +200,7 @@ test('same-named officials at different institutions do NOT merge into one perso // Two DISTINCT persons named „Георги Иванов" — one per institution — never one merged identity. const persons = db - .prepare("SELECT id, name FROM persons WHERE name = 'Георги Иванов' ORDER BY id") + .prepare("SELECT id, name FROM persons WHERE name = 'Георги Иванов Петров' ORDER BY id") .all(); assert.equal(persons.length, 2, 'two distinct namesake officials, not one merged person'); assert.notEqual(persons[0].id, persons[1].id); diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 984ead206..5a37ae91c 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -16,12 +16,13 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { nameDistinctiveness, - seatConfirmed, - publishTier, temporalStatus, localityToken, closelyHeldForm, } from './classify.mjs'; +import { openCache, readDeed, coverage } from '../tr/cache.mjs'; +import { TR_DB, TR_RAW } from '../tr/paths.mjs'; +import { evidenceVerdict, reconcileTermination, RULES_VERSION } from '../tr/evidence.mjs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; import { fingerprint, loadSuppressions, SUPPRESSION_KEY_VERSION } from './suppressions.mjs'; import { canonicalInstitution } from './institutions.mjs'; @@ -38,7 +39,15 @@ const MIGRATION_EVIDENCE = path.join( 'packages/db/migrations/0006_interest_link_evidence.sql', ); const REPORT = path.join(STAGING, 'findings.md'); -const MATCHER_VERSION = 'cnk-1+classify-1'; // bump when the normalizer or classify logic changes +// Bumped for #279: classify-2 (КДА added to the joint-stock bar) + tr-1 (identity now rests on a +// Trade Register fact, not on name distinctiveness). RULES_VERSION versions the EVIDENCE rules +// separately — §8's monotonicity gate keys on that one, not on this. +const MATCHER_VERSION = 'cnk-1+classify-2+tr-1'; +const TR_CACHE_DB = process.env.TR_CACHE_DB || TR_DB; +const TR_RAW_DIR = process.env.TR_RAW_DIR || TR_RAW; +// A deliberate, logged override for the coverage gate below. Without it a single permanently +// unreachable ЕИК would deadlock the pipeline forever; with it, the operator states that they know. +const ALLOW_PARTIAL_TR = process.argv.includes('--allow-partial-tr'); const { companyNameKey, isMatchableKey } = await import('../../packages/shared/src/company-name-key.ts'); @@ -403,6 +412,12 @@ const insLink = db.prepare( const insILA = db.prepare( 'INSERT OR IGNORE INTO interest_link_authorities(link_key,authority_id,authority_name,contract_count,value_eur,own) VALUES(?,?,?,?,?,?)', ); +// The evidence seal (#279 §8, migration 0006). Written for EVERY link, not only published ones — the +// seals on held and withdrawn links are what let the review queue explain itself. `matched_fact` is a +// closed vocabulary and must NEVER carry a name; the audit enforces that with a pattern check. +const insEvidence = db.prepare( + 'INSERT OR REPLACE INTO interest_link_evidence(link_key,evidence_kind,registry_role,matched_fact,entry_number,entry_date,lookup_date,rules_version,live_status) VALUES(?,?,?,?,?,?,?,?,?)', +); // classify one authority (whose name may be a ';'-joined blob) against the official's institutions. // exact = deterministic name equality; name_contains/locality = DISCLOSED heuristics (candidate, not proof). const OWN_RANK = { exact: 3, name_contains: 2, locality: 1, none: 0 }; @@ -426,6 +441,73 @@ function authOwn(authorityName, instNorms, instNormsLong, locTokens) { } // Distinct officials who declared each company (ЕИК). A private interest has ONE owner-declarant; a // public body's board is declared by MANY rotating members — the deterministic ex-officio tell (ADR-0019). +// ── Trade Register evidence: the candidate set, the fail-closed gate, and the deed reader ───────── +// Identity now rests on a checkable registry fact rather than on the shape of the declared name +// (#279, ADR-0033). Two consequences the loader has to enforce, both fail-closed: +// +// 1. NO cache ⇒ throw. Publishing without evidence is precisely what this change abolishes. +// 2. PARTIAL cache ⇒ throw. This is the silent one. An 80%-restored cache yields roughly 80 +// published links, which is ABOVE ship-related-persons.mjs's floor of 50 — so it would sail +// through that guard, ship a decimated surface, and wipe the rest of the live links. +// +// The candidate set is every resolved ЕИК across ALL aggregates, not just the ones that end up +// published: a link held for want of evidence still needs its deed to say so. +const candidateEiks = [...new Set([...agg.values()].map((r) => r.eik))].sort(); +fs.writeFileSync(path.join(STAGING, 'candidate-eiks.txt'), candidateEiks.join('\n') + '\n'); + +if (!fs.existsSync(TR_CACHE_DB)) { + db.close(); + throw new Error( + `REFUSE TO LOAD: no Trade Register cache at ${TR_CACHE_DB}. Every publishing decision now rests ` + + `on a registry fact (ADR-0033); without the cache there is no evidence to rest on. Run ` + + `scripts/tr/fetch-deeds.mjs --eiks-file ${path.join(STAGING, 'candidate-eiks.txt')} first.`, + ); +} +const trCache = openCache(TR_CACHE_DB); +const trCoverage = coverage(trCache, candidateEiks); +console.log( + `TR cache: ${trCoverage.covered}/${trCoverage.wanted} covered ` + + `(fetched ${trCoverage.fetched}, outside ТР ${trCoverage.outsideTr}, missing ${trCoverage.missing})`, +); +if (trCoverage.missing > 0 && !ALLOW_PARTIAL_TR) { + trCache.close(); + db.close(); + throw new Error( + `REFUSE TO LOAD: the Trade Register cache covers ${trCoverage.covered} of ${trCoverage.wanted} ` + + `candidate ЕИК. A partial cache does not fail loudly downstream — it publishes a decimated ` + + `surface that still clears the ship floor and then wipes the rest of the live links. Finish ` + + `the crawl, or pass --allow-partial-tr to state that a smaller surface is intended.`, + ); +} + +// Deeds are read from git-ignored scratch and cached in memory for the run. The parsed deed carries +// third-party names; they are used ONLY inside evidenceVerdict's boolean comparisons and never reach +// a column, a log line or the report (ADR-0033 decision 5). +// The lookup date sealed on every link: when the evidence was gathered, not when it was interpreted. +// It is the freshness bound the methodology page has to state, so it comes from the cache rather than +// from `now` — a re-run over an unchanged cache must not make the evidence look fresher than it is. +const trLookupDate = (() => { + const row = trCache.prepare('SELECT MAX(fetched_at) m FROM deeds').get(); + return row?.m ? String(row.m).slice(0, 10) : new Date().toISOString().slice(0, 10); +})(); + +const deedCache = new Map(); +function deedFor(eik) { + if (deedCache.has(eik)) return deedCache.get(eik); + const row = readDeed(trCache, eik); + let entry; + if (!row) entry = { deed: null, outsideTr: false, missing: true }; + else if (row.status === 'outside_tr') entry = { deed: null, outsideTr: true, missing: false }; + else { + const file = path.join(TR_RAW_DIR, row.rawPath ?? `${eik}.json`); + entry = fs.existsSync(file) + ? { deed: JSON.parse(fs.readFileSync(file, 'utf8')), outsideTr: false, missing: false } + : { deed: null, outsideTr: false, missing: true }; + } + deedCache.set(eik, entry); + return entry; +} + const declarantsByEik = new Map(); for (const rec of agg.values()) { if (rec.scope !== 'self') continue; // ex-officio tell counts SELF declarants of a public board only @@ -470,7 +552,20 @@ for (const rec of agg.values()) { a.count++; if (r.eur != null) a.value += r.eur; } - const seatOk = [...rec.seats].some((s) => seatConfirmed(s, rec.bidder.settlement)); + // ── the evidence ladder replaces the publish tiers (ADR-0033 decision 1) ──────────────────────── + // rec.seats is keyed on `pid|eik|scope`, so it already holds ONLY the seats this person declared for + // THIS company — which is what #279 §5 rung 3 requires: 4.9% of company-name keys carry more than one + // distinct declared seat, so a company-only key would let one person's seat confirm another's link. + const { deed, outsideTr, missing } = deedFor(rec.eik); + if (missing && !ALLOW_PARTIAL_TR) { + // Unreachable via the coverage gate above; kept as a belt-and-braces refusal so a future change + // that loosens the gate cannot silently publish a link with no evidence behind it. + trCache.close(); + db.close(); + throw new Error( + `no cached deed for ЕИК ${rec.eik} — the coverage gate should have caught this`, + ); + } // A declarant-provided ЕИК is the national unique identifier (ЗТРРЮЛНЦ) — it resolves the winner // deterministically even behind a generic or winner-colliding name, so a declared_eik match publishes // on its own basis (A_eik), never held for name-genericness. This is at least as certain as the seat @@ -478,14 +573,39 @@ for (const rec of agg.values()) { // Name-only methods (exact_name_key / extracted_name) still ride the distinctiveness/seat gate below: // a globally non-unique winner name (e.g. „Водоснабдяване и канализация ЕАД" → 2 valid ЕИК in different // towns) can never be name-distinctive, so it publishes only if the declared SEAT disambiguates, else held. + // The filters that can only WITHHOLD are retained as an AND-gate on the weakest rung only + // (ADR-0033 decision 2): a nationally shared company name cannot ride „Потвърдено". The stronger + // „Документ" rung is deliberately not gated — the register named this person in THIS company, which + // makes the name key moot. Near-zero recall cost, and it preserves ADR-0017's outcome. + // ADR-0017 carried forward, and NARROWED to what it actually held: a name backing more than one valid + // ЕИК cannot support a name-derived identity claim. It gates the SEAT leg of rung 3 only — never the + // declared-ЕИК leg (ADR-0028: the ЕИК is the identity), and never rung 2 (the register named this + // person in THIS company). nameDistinctiveness is deliberately NOT part of this gate: the seat rung + // exists precisely to rescue a generic name, so requiring distinctiveness would empty it. const nameUnique = nameGloballyUnique(rec.key); - const tier = - rec.method === 'declared_eik' - ? 'A_eik' - : publishTier({ - seatOk, - distinctiveness: nameUnique ? nameDistinctiveness(rec.key) : 'generic', - }); + // With --allow-partial-tr the operator has accepted an incomplete cache. An uncached ЕИК then yields + // no evidence at all, which is „Неизвестна" — held. It must never be read as a reason to publish. + const verdict = missing + ? { + kind: 'unknown', + publishable: false, + registryRole: null, + matchedFact: null, + entryNumber: null, + entryDate: null, + rulesVersion: RULES_VERSION, + } + : evidenceVerdict({ + deed, + outsideTr, + declarantName: rec.person, + declaredSeats: [...rec.seats], + declaredEik: rec.method === 'declared_eik', + firstDeclaredYear: declYears.length ? Math.min(...declYears) : null, + scope: rec.scope, + nameGloballyUnique: nameUnique, + }); + const tier = verdict.kind; const contemporaneous = [...years].some( (cy) => temporalStatus(declYears, cy) === 'contemporaneous', ) @@ -547,15 +667,33 @@ for (const rec of agg.values()) { // non-surfaced class ('internal'), not 'published'. cValue can legitimately be 0 with cCount>0 (contracts // whose amount is unknown/NULL) — that is a real conflict of unknown value, so gate on COUNT, not value. const surfaces = (iClass === 'private_ownership' || iClass === 'family_ownership') && cCount > 0; + // §7: „terminated" is an inference from SILENCE, and its commonest cause is a finished mandate, not a + // sale. Before ADR-0021 E11's withdrawal takes effect, a terminated OWN stake is reconciled against the + // live deed: a person still registered as an owner has not divested. Family stakes are never reconciled + // — the registered owner there is the relative, whose name we neither store nor check. + // PHASE 1 uses only `terminated`; the „и към днешна дата" label is computed and deliberately not + // rendered (ADR-0033 decision 4 — it asserts a present tense behind an LIA addendum). + const recon = + divested && rec.scope === 'self' + ? reconcileTermination({ deed, declarantName: rec.person, scope: rec.scope }) + : { terminated: divested, label: null }; + const terminatedEffective = recon.terminated; + + // Order matters and differs from the old ladder: `internal` is now decided BEFORE `published`, so a + // non-surfaced class (ex-officio board, management-only, zero-contract) can never land in `held`. + // `held` is the REVIEW QUEUE, and its population is exactly the evidence rungs that withhold — + // bar_joint_stock, unknown and outside_tr. const status = isSuppressed(linkKey) ? 'suppressed' - : divested - ? 'withdrawn' - : tier === 'C_hold' - ? 'held' - : surfaces - ? 'published' - : 'internal'; + : verdict.kind === 'refuted' + ? 'withdrawn' // §5.4 — own stakes only; evidence.mjs refuses to refute a family stake + : terminatedEffective + ? 'withdrawn' + : !surfaces + ? 'internal' + : verdict.publishable + ? 'published' + : 'held'; const yrs = [...years]; insLink.run( `il:${linkKey}`, @@ -580,6 +718,26 @@ for (const rec of agg.values()) { yrs.length ? String(Math.max(...yrs)) : null, status, ); + // live_status is RE-DERIVED every run and never treated as part of the permanent seal (ADR-0033 R3): + // it asserts a present tense whose freshness is bounded by the cache refresh cycle. + const liveStatus = !terminatedEffective + ? recon.label === 'owner_today' + ? 'terminated_owner_still' + : 'live' + : recon.label === 'manager_today' + ? 'terminated_manager_still' + : 'terminated'; + insEvidence.run( + linkKey, + verdict.kind, + verdict.registryRole, + verdict.matchedFact, + verdict.entryNumber, + verdict.entryDate, + trLookupDate, + verdict.rulesVersion, + liveStatus, + ); for (const [auth_id, a] of perAuth) insILA.run(linkKey, auth_id, a.name, a.count, a.value || null, a.own); } diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index d795bf598..d90b44a44 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -14,7 +14,7 @@ import { fingerprint } from './suppressions.mjs'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); const SUPP_SALT = 'test-salt-9f3a'; // stand-in for the CI secret SUPPRESSION_SALT -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad(extraEnv = {}) { execFileSync( @@ -22,17 +22,105 @@ function runLoad(extraEnv = {}) { ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], { cwd: ROOT, - env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING, ...extraEnv }, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + ...extraEnv, + }, stdio: 'pipe', }, ); } + +/** + * Build a Trade Register cache + raw deeds covering the fixture's winners. + * + * `spec[eik]` describes one deed: `owners` / `managers` are full names placed in SEPARATE registry + * entities (so the entity-boundary rule is exercised end to end), `form` is the numeric legalForm and + * `suffix` the ЗТРРЮЛНЦ form on fullName. Anything omitted from `spec` is still cached — as a deed + * naming somebody else — because the loader must FAIL CLOSED on a cache that does not cover every + * candidate, and a test that silently left ЕИК uncovered would exercise that path by accident. + */ +function buildTrCache(dbFile, rawDir, spec = {}, { omit = [] } = {}) { + fs.mkdirSync(rawDir, { recursive: true }); + const cache = new DatabaseSync(dbFile); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + const src = new DatabaseSync(DB, { readOnly: true }); + const eiks = src + .prepare('SELECT eik_normalized e FROM bidders WHERE eik_normalized IS NOT NULL') + .all() + .map((r) => r.e); + src.close(); + + const container = (t) => + `

${t}

`; + const joinEntities = (names) => names.map(container).join(`
`); + + for (const eik of eiks) { + if (omit.includes(eik)) continue; + const d = spec[eik] ?? {}; + if (d.outsideTr) { + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,fetched_at,outside_reason) VALUES(?,?,?,?)', + ) + .run(eik, 'outside_tr', '2026-08-05T00:00:00Z', 'HTTP 200, empty body'); + continue; + } + const fields = []; + const push = (nameCode, names, entryDate) => + names?.length && + fields.push({ + nameCode, + htmlData: joinEntities(names), + fieldEntryNumber: '20110502101007', + fieldEntryDate: `${entryDate ?? '2011-05-02'}T00:00:00`, + }); + push('CR_F_19_L', d.owners ?? ['НЯКОЙ ДРУГ СОБСТВЕНИК'], d.ownEntryDate); + push('CR_F_7_L', d.managers, d.ownEntryDate); + if (d.seat) push('CR_F_5_L', [`Населено място: ${d.seat}`], d.seatEntryDate ?? d.ownEntryDate); + const deed = { + uic: eik, + fullName: `"ФИКС" ${d.suffix ?? 'ООД'}`, + legalForm: d.form ?? 4, + sections: [{ subDeeds: [{ groups: [{ fields }] }] }], + }; + fs.writeFileSync(path.join(rawDir, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + `INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code, + legal_form_verdict,seat_normalized,latest_own_entry_date) + VALUES(?,?,?,?,?,?,?,?,?)`, + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + d.form ?? 4, + d.suffix && /АД|КДА/.test(d.suffix) ? 'joint_stock' : 'closely_held', + d.seat ? d.seat.replace(/^гр\.\s*/, '').toUpperCase() : null, + d.ownEntryDate ?? '2011-05-02', + ); + } + cache.close(); +} const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-load-')); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); fs.mkdirSync(STAGING, { recursive: true }); // minimal slice of the winner schema that load.mjs joins @@ -99,6 +187,12 @@ before(() => { -- not twice — the load-side sibling of the UI conflictHeadline per-ЕИК money dedup. INSERT INTO bidders VALUES ('eik:181818187','ПАРТНЬОРИ 5 ЕООД','181818187',1,'София'); INSERT INTO contracts VALUES ('c15','t1','eik:181818187','2022-06-01',600000); + -- FAMILY positive control (#279): identical in shape to Кмет's case but the official DECLARED the + -- company's seat, which is what confirms the company's identity when the registered owner is the + -- relative whose name we never hold. Without this case „family published: 0" would be + -- indistinguishable from a structurally dead path (ADR-0027's false-zero lesson). + INSERT INTO bidders VALUES ('eik:191919199','СЕМЕЕН ДОМ ЕООД','191919199',1,'Русе'); + INSERT INTO contracts VALUES ('c16','t4','eik:191919199','2023-07-01',180000); `); db.close(); @@ -128,7 +222,7 @@ before(() => { template: 'assets', category: '', institution: 'X', - person: 'Мария Иванова', + person: 'Мария Иванова Петрова', position: '', entity: '"ГЕНЕРИК" ООД', kind: 'shares', @@ -145,7 +239,7 @@ before(() => { template: 'assets', category: '', institution: 'Y', - person: 'Петър Николов', + person: 'Петър Иванов Николов', position: '', entity: 'СИЙ ЕООД', kind: 'shares', @@ -162,7 +256,7 @@ before(() => { template: 'assets', category: '', institution: 'Z', - person: 'Георги Стоянов', + person: 'Георги Иванов Стоянов', position: '', entity: 'СИЙ ЕООД', kind: 'shares', @@ -181,7 +275,7 @@ before(() => { template: 'assets', category: '', institution: 'W', - person: 'Стефан Колев', + person: 'Стефан Иванов Колев', position: '', entity: '"ГЕНЕРИК" ООД, ЕИК 222222229', kind: 'shares', @@ -198,7 +292,7 @@ before(() => { template: 'assets', category: '', institution: 'V', - person: 'Радка Илиева', + person: 'Радка Иванова Илиева', position: '', entity: '"ГЕНЕРИК" ООД, ЕИК 333333338', kind: 'shares', @@ -215,7 +309,7 @@ before(() => { template: 'interests', category: '', institution: 'U', - person: 'Борис Манолов', + person: 'Борис Иванов Манолов', position: 'член на съвет', entity: 'ХОЛДИНГ 9 ЕАД', kind: 'management', @@ -231,7 +325,7 @@ before(() => { template: 'interests', category: '', institution: 'U', - person: 'Виктор Асенов', + person: 'Виктор Иванов Асенов', position: 'член на съвет', entity: 'ХОЛДИНГ 9 ЕАД', kind: 'management', @@ -249,7 +343,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Николай Дивестов', + person: 'Николай Иванов Дивестов', position: '', entity: 'ДИВЕСТ 1 ЕООД', kind: 'shares', @@ -265,7 +359,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Николай Дивестов', + person: 'Николай Иванов Дивестов', position: '', entity: 'ДИВЕСТ 2 ЕООД', kind: 'shares', @@ -283,7 +377,7 @@ before(() => { template: 'assets', category: '', institution: 'ОБЩИНА ТЕСТ', - person: 'Кмет Тестов', + person: 'Кмет Иванов Тестов', position: 'кмет', entity: 'ЕВРОСТРОЙ 21 ЕООД', kind: 'shares', @@ -293,6 +387,24 @@ before(() => { holderRelation: 'related', controlHash: 'H11', }, + // FAMILY positive control: same as Кмет, but with the seat declared → rung 3 confirms the company. + { + folder: '2024', + xmlFile: 'K2.xml', + year: '2023', + template: 'assets', + category: '', + institution: 'ОБЩИНА ТЕСТ', + person: 'Кметица Иванова Втора', + position: 'кмет', + entity: 'СЕМЕЕН ДОМ ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: 'Русе', + holderRelation: 'related', + controlHash: 'H11b', + }, // SECURITIES: Акционер holds LISTED joint-stock shares (kind securities) → excluded, no ownership link. { folder: '2024', @@ -301,7 +413,7 @@ before(() => { template: 'assets', category: '', institution: 'S', - person: 'Акционер Тестов', + person: 'Акционер Иванов Тестов', position: '', entity: 'ЛИСТЕД ТЕСТ АД', kind: 'securities', @@ -320,7 +432,7 @@ before(() => { template: 'assets', category: '', institution: 'N2', - person: 'Нула Тестов', + person: 'Нула Иванов Тестов', position: '', entity: 'НУЛА ТЕХ 3 ЕООД', kind: 'shares', @@ -340,7 +452,7 @@ before(() => { template: 'assets', category: '', institution: 'T2', - person: 'Пълен Дивестов', + person: 'Пълен Иванов Дивестов', position: '', entity: 'ДИВЕСТ ЗЕРО 4 ЕООД', kind: 'shares', @@ -360,7 +472,7 @@ before(() => { template: 'assets', category: '', institution: 'N3', - person: 'Двусмислен Тестов', + person: 'Двусмислен Иванов Тестов', position: '', entity: 'ДИСТИНКТ ТЕХ 7 ЕООД', kind: 'shares', @@ -379,7 +491,7 @@ before(() => { template: 'assets', category: '', institution: 'МВР', - person: 'Канонов Тестов', + person: 'Канонов Иванов Тестов', position: '', entity: 'КАНОН ТЕХ 5 ЕООД', kind: 'shares', @@ -396,7 +508,7 @@ before(() => { template: 'assets', category: '', institution: 'Министерство на вътрешните работи', - person: 'Канонов Тестов', + person: 'Канонов Иванов Тестов', position: '', entity: 'КАНОН ТЕХ 5 ЕООД', kind: 'shares', @@ -414,7 +526,7 @@ before(() => { template: 'assets', category: '', institution: '', - person: 'Безинст Тестов', + person: 'Безинст Иванов Тестов', position: '', entity: 'БЕЗИНСТ ТЕХ 6 ЕООД', kind: 'shares', @@ -434,7 +546,7 @@ before(() => { template: 'interests', category: '', institution: 'INT', - person: 'Интер Тестов', + person: 'Интер Иванов Тестов', position: '', entity: 'ИНТЕР ТЕХ 8 ЕООД', kind: 'shares', @@ -454,7 +566,7 @@ before(() => { template: 'assets', category: '', institution: 'ТЕСТ ВЕДОМСТВО', - person: 'Алфа Партньоров', + person: 'Алфа Иванов Партньоров', position: '', entity: 'ПАРТНЬОРИ 5 ЕООД', kind: 'shares', @@ -471,7 +583,7 @@ before(() => { template: 'assets', category: '', institution: 'ДРУГО ВЕДОМСТВО', - person: 'Бета Партньоров', + person: 'Бета Иванов Партньоров', position: '', entity: 'ПАРТНЬОРИ 5 ЕООД', kind: 'shares', @@ -503,7 +615,7 @@ before(() => { xmlFile: 'ZE1.xml', year: '2023', template: 'assets', // Пълен's later EMPTY ASSET filing — same type as his 2019 asset stake ⇒ divests it - person: 'Пълен Дивестов', + person: 'Пълен Иванов Дивестов', institution: 'T2', }); // #226 (Todor B1) cross-type: Интер declares his stake ONLY in an INTERESTS declaration (2020); his later @@ -515,13 +627,39 @@ before(() => { xmlFile: 'INTA.xml', year: '2023', template: 'assets', - person: 'Интер Тестов', + person: 'Интер Иванов Тестов', institution: 'INT', }); fs.writeFileSync( path.join(STAGING, 'filings.jsonl'), filings.map((f) => JSON.stringify(f)).join('\n') + '\n', ); + + // The Trade Register evidence each link now has to rest on (#279, ADR-0033). Shaped so every + // existing case keeps the INTENT it was written for, under the new rule rather than the old one: + // • a person the register names as owner/manager → „Документ" + // • a declared seat matching the registered seat → „Потвърдено" + // • a declared ЕИК → „Потвърдено" (never name-gated, ADR-0028) + // • nobody we can match and nothing to confirm → „Неизвестна", held + buildTrCache(TR_DB, TR_RAW, { + 111111119: { managers: ['ИВАН ПЕТРОВ ТЕСТОВ'] }, // manages → document/manager (class keeps it internal) + 444444447: { seat: 'гр. Бургас' }, // Петър declared Бургас → confirmed; Георги declared none → held + 555555556: { + managers: ['БОРИС ИВАНОВ МАНОЛОВ', 'ВИКТОР ИВАНОВ АСЕНОВ'], + suffix: 'ЕАД', + form: 5, + }, + 666666665: { owners: ['СЪВСЕМ ДРУГ СОБСТВЕНИК'] }, // Николай absent → his divestment stands + 777777773: { owners: ['НИКОЛАЙ ИВАНОВ ДИВЕСТОВ'] }, // still the registered owner → document + 888888884: { owners: ['РОДНИНА КМЕТОВА'] }, // family: the RELATIVE owns it, not the official + 999999998: { suffix: 'АД', form: 5 }, + 101010104: { owners: ['ДРУГ СОБСТВЕНИК'] }, // Пълен absent → divest-to-zero stands + 131313136: { owners: ['КАНОНОВ ИВАНОВ ТЕСТОВ'] }, + 161616163: { owners: ['ИНТЕР ИВАНОВ ТЕСТОВ'] }, + 181818187: { owners: ['АЛФА ИВАНОВ ПАРТНЬОРОВ', 'БЕТА ИВАНОВ ПАРТНЬОРОВ'] }, + 121212129: { owners: ['НУЛА ИВАНОВ ТЕСТОВ'] }, + 191919199: { owners: ['РОДНИНА ВТОРА'], seat: 'гр. Русе' }, // family + declared seat → confirmed + }); }); after(() => fs.rmSync(dir, { recursive: true, force: true })); @@ -540,7 +678,7 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // management_role never surfaces → status 'internal', NOT 'published' (a direct D1 reader must not see a // non-surfaced official+company row labelled published; the served query also filters by interest_class). assert.equal(ivan.status, 'internal'); - assert.equal(ivan.publish_tier, 'B_distinctive'); + assert.equal(ivan.publish_tier, 'document'); // the register names him a manager of this company assert.equal(ivan.relation, 'manages'); assert.equal(ivan.interest_class, 'management_role'); // manages, sole declarant → ambiguous, not headline assert.equal(ivan.own_institution, 'exact'); @@ -557,8 +695,8 @@ test('resolves publish/held/quarantine tiers deterministically', () => { assert.equal(blob.value_eur, 25000); // bare collision name (no ЕИК in text) → quarantined, Мария gets no link - assert.equal(link('222222229', 'Мария Иванова'), undefined); - assert.equal(link('333333338', 'Мария Иванова'), undefined); + assert.equal(link('222222229', 'Мария Иванова Петрова'), undefined); + assert.equal(link('333333338', 'Мария Иванова Петрова'), undefined); // the only links onto the colliding ЕИК come from declared_eik (Стефан/Радка), never exact_name_key assert.equal( db @@ -569,14 +707,14 @@ test('resolves publish/held/quarantine tiers deterministically', () => { 0, ); - const petar = link('444444447', 'Петър Николов'); - assert.equal(petar.publish_tier, 'A_seat'); // generic name rescued by seat match + const petar = link('444444447', 'Петър Иванов Николов'); + assert.equal(petar.publish_tier, 'confirmed'); // declared seat == registered seat assert.equal(petar.status, 'published'); assert.equal(petar.interest_class, 'private_ownership'); // declared a share → the headline conflict signal // two officials manage the SAME company → deterministically classed ex-officio (public board), not private - const boris = link('555555556', 'Борис Манолов'); - const viktor = link('555555556', 'Виктор Асенов'); + const boris = link('555555556', 'Борис Иванов Манолов'); + const viktor = link('555555556', 'Виктор Иванов Асенов'); assert.equal(boris.interest_class, 'ex_officio_board'); assert.equal(viktor.interest_class, 'ex_officio_board'); assert.equal(boris.relation, 'manages'); @@ -584,14 +722,14 @@ test('resolves publish/held/quarantine tiers deterministically', () => { assert.equal(boris.status, 'internal'); assert.equal(viktor.status, 'internal'); - const georgi = link('444444447', 'Георги Стоянов'); - assert.equal(georgi.publish_tier, 'C_hold'); // generic, no seat → held + const georgi = link('444444447', 'Георги Иванов Стоянов'); + assert.equal(georgi.publish_tier, 'unknown'); // same company, but he declared no seat → nothing confirms assert.equal(georgi.status, 'held'); // E11 divestment: Николай's 2019 stake in ДИВЕСТ 1 is superseded by a 2022 filing that omits it → withdrawn; // his current ДИВЕСТ 2 stake stays published. A later ownership filing that drops a company ends that link. - const gone = link('666666665', 'Николай Дивестов'); - const kept = link('777777773', 'Николай Дивестов'); + const gone = link('666666665', 'Николай Иванов Дивестов'); + const kept = link('777777773', 'Николай Иванов Дивестов'); assert.equal(gone.status, 'withdrawn'); // divested — excluded from the published surface assert.equal(gone.interest_class, 'private_ownership'); assert.equal(gone.last_declared_year, '2019'); // dated to its last declaration, never asserted "current" @@ -601,15 +739,15 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // certain ЕИК (declared_eik) behind a colliding name, no seat → the declarant-provided ЕИК is the // national unique identifier, so identity is deterministic → publishes as A_eik, NOT held for // name-genericness (ADR-0016; the ЕИК is at least as certain as the seat that rescues Радка below). - const stefan = link('222222229', 'Стефан Колев'); + const stefan = link('222222229', 'Стефан Иванов Колев'); assert.equal(stefan.match_method, 'declared_eik'); // ЕИК resolution IS certain - assert.equal(stefan.publish_tier, 'A_eik'); // ЕИК = unique identifier → deterministic, not name-gated + assert.equal(stefan.publish_tier, 'confirmed'); // the declared ЕИК confirms the company, never name-gated assert.equal(stefan.status, 'published'); // private_ownership (20% ООД share) → surfaces // same colliding name, resolved by her declared ЕИК (with seat as extra corroboration) → the ЕИК is the // identity, so A_eik (not A_seat); publishable. A_seat's own path stays covered by Петър above. - const radka = link('333333338', 'Радка Илиева'); + const radka = link('333333338', 'Радка Иванова Илиева'); assert.equal(radka.match_method, 'declared_eik'); - assert.equal(radka.publish_tier, 'A_eik'); + assert.equal(radka.publish_tier, 'confirmed'); assert.equal(radka.status, 'published'); // FAMILY: a close relative's declared stake in a winner that sold to the official's OWN institution. @@ -617,10 +755,17 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // relative's declared stake in a procurement winner is the same public-interest signal. class // family_ownership, relation 'related', own_institution exact. It has real contract money (€250k, cCount>0) // so the zero-contract gate keeps it. - const family = link('888888884', 'Кмет Тестов'); + const family = link('888888884', 'Кмет Иванов Тестов'); assert.equal(family.relation, 'related'); assert.equal(family.interest_class, 'family_ownership'); - assert.equal(family.status, 'published'); // ADR-0032: family surfaces like self + // #279 NARROWS the family surface, and this is where it shows. The registered owner of a family + // stake is the RELATIVE, whose name we deliberately never store (ADR-0010 item 4, ADR-0032 #2) — so + // rung 2 („Документ") can never fire for a family link, by construction. Its identity can only be + // confirmed by something the OFFICIAL declared: the seat, or the ЕИК. Кмет declared neither, so his + // link is now HELD rather than published. ADR-0032's decision is untouched — family publishes on the + // named surface exactly like self — but it now needs the same registry evidence as everything else. + assert.equal(family.status, 'held'); + assert.equal(family.publish_tier, 'unknown'); assert.equal(family.own_institution, 'exact'); // relative's company sold to the official's own institution assert.equal(family.contemporaneous, 1); assert.equal(family.contract_value_eur, 250000); @@ -645,8 +790,8 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // SUM(contract_value_eur) over published links double-counts that winner — the load-side twin of the UI // conflictHeadline bug. Prove the fixture actually exercises the collision (naive per-link sum strictly // exceeds the per-ЕИК-deduped sum), then assert the reported totals equal the deduped figure. - const alfa = link('181818187', 'Алфа Партньоров'); - const beta = link('181818187', 'Бета Партньоров'); + const alfa = link('181818187', 'Алфа Иванов Партньоров'); + const beta = link('181818187', 'Бета Иванов Партньоров'); assert.equal(alfa.status, 'published'); assert.equal(beta.status, 'published'); assert.equal(alfa.interest_class, 'private_ownership'); @@ -683,14 +828,14 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // ZERO-CONTRACT gate (I5): a distinctive winner with NO contracts is collected but never published — the // card would read „0 договори · 0 €", which is no procurement conflict. status 'internal', not 'published'. - const zero = link('121212129', 'Нула Тестов'); + const zero = link('121212129', 'Нула Иванов Тестов'); assert.equal(zero.contract_count, 0); assert.equal(zero.interest_class, 'private_ownership'); // it IS own material ownership … - assert.equal(zero.publish_tier, 'B_distinctive'); // … and tier-B by name … + assert.equal(zero.publish_tier, 'document'); // … with registry evidence … assert.equal(zero.status, 'internal'); // … but the zero-contract gate withholds it from the surface // SECURITIES/materiality: a self holding of LISTED joint-stock shares forms NO ownership link. - assert.equal(link('999999998', 'Акционер Тестов'), undefined); + assert.equal(link('999999998', 'Акционер Иванов Тестов'), undefined); // but it is still recorded as a declared interest (census), tagged kind securities assert.equal( db.prepare("SELECT kind FROM declared_interests WHERE entity_raw='ЛИСТЕД ТЕСТ АД'").get().kind, @@ -700,7 +845,7 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // B1 divest-to-ZERO: Пълен owned ДИВЕСТ ЗЕРО in 2019, then filed an EMPTY declaration in 2023 (no holdings // row — only a filings.jsonl entry). The empty filing advances his horizon to 2023, so the 2019 stake is // WITHDRAWN. Without the filing horizon (pre-B1) his scope-max would be 2019 and this would stay published. - const divZero = link('101010104', 'Пълен Дивестов'); + const divZero = link('101010104', 'Пълен Иванов Дивестов'); assert.equal(divZero.interest_class, 'private_ownership'); assert.equal(divZero.last_declared_year, '2019'); // dated to its last declaration, never asserted current assert.equal(divZero.status, 'withdrawn'); // caught by the empty later filing (B1) @@ -710,18 +855,18 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // that asset-declaration silence as a sale and WITHDRAWS the stake; the per-type horizon must not — no later // INTERESTS filing omits the company. This link must stay PUBLISHED. (Guards against dropping a true link: // 13% of holders declare a stake only in the interests declaration.) - const crossType = link('161616163', 'Интер Тестов'); + const crossType = link('161616163', 'Интер Иванов Тестов'); assert.equal(crossType.interest_class, 'private_ownership'); assert.equal(crossType.status, 'published'); // NOT withdrawn — the later asset filing is a different type // B4 UNKNOWN holder: an ambiguous holder cell forms NO link at all (counted nowhere) — it must never // reach the leaderboard, self or family (ADR-0032). Двусмислен gets no interest_link. - assert.equal(link('111111119', 'Двусмислен Тестов'), undefined); + assert.equal(link('111111119', 'Двусмислен Иванов Тестов'), undefined); // but the person + declared_interest are still recorded (census), and it is neither self nor family. assert.equal( db .prepare( - "SELECT COUNT(*) n FROM interest_links il JOIN persons p ON p.id=il.person_id WHERE p.name='Двусмислен Тестов'", + "SELECT COUNT(*) n FROM interest_links il JOIN persons p ON p.id=il.person_id WHERE p.name='Двусмислен Иванов Тестов'", ) .get().n, 0, @@ -738,10 +883,10 @@ test('resolves publish/held/quarantine tiers deterministically', () => { .n, 1, ); - const kanon = link('131313136', 'Канонов Тестов'); + const kanon = link('131313136', 'Канонов Иванов Тестов'); assert.equal(kanon.status, 'published'); // distinctive, private ownership, has a contract // N10 empty-institution: an empty institution cannot distinguish homonyms → Безинст forms NO link. - assert.equal(link('141414141', 'Безинст Тестов'), undefined); + assert.equal(link('141414141', 'Безинст Иванов Тестов'), undefined); db.close(); }); @@ -772,14 +917,15 @@ test('re-run is idempotent and honors the suppression list (contested link stays 'suppressed', ); // idempotent: still exactly the same number of links + persons after a clean rebuild. - // 16 links: 12 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero - // 'withdrawn' + Интер's per-type-kept published link) + 1 family + Канонов's canonicalized single link + + // 17 links: 12 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero + // 'withdrawn' + Интер's per-type-kept published link) + 2 family (Кмет's, now held for want of + // registry evidence, and Кметица's seat-confirmed one) + Канонов's canonicalized single link + // Алфа & Бета (two officials on one winner, ПАРТНЬОРИ 5); Мария (quarantined), Акционер (securities), // Двусмислен (unknown holder) & Безинст (empty institution) none. - assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 16); - // 19 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, + assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 17); + // 20 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, // zero-contract Нула and the two ПАРТНЬОРИ co-owners; Канонов's two institution-variant filings fold to ONE. - assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 19); + assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 20); db.close(); }); @@ -879,3 +1025,135 @@ test('a suppression on a ROTATED key_version FAILS the build (B3 no silent salt (err) => /key_version/.test(String(err.stderr ?? '') + String(err.message ?? '')), ); }); + +// ── the fail-closed evidence gates (ADR-0033 decision 7) ───────────────────────────────────────── +// Both directions of "no evidence ⇒ no publish", because the partial case is the dangerous one: it does +// NOT fail loudly downstream. An 80%-restored cache yields roughly 80 published links, which clears +// ship-related-persons.mjs's floor of 50 — so it would ship a decimated surface and then wipe the rest +// of the live links. The loader is the only place that can still tell the difference. +test('a MISSING Trade Register cache refuses the whole load', () => { + const gone = path.join(dir, 'no-such-cache.sqlite'); + assert.throws( + () => runLoad({ TR_CACHE_DB: gone }), + /REFUSE TO LOAD[\s\S]*no Trade Register cache/, + ); +}); + +test('a PARTIAL cache refuses the load rather than publishing a decimated surface', () => { + const partialDb = path.join(dir, 'partial-cache.sqlite'); + const partialRaw = path.join(dir, 'partial-deeds'); + // Cover everything EXCEPT two winners — the shape a resumed-but-unfinished crawl leaves behind. + buildTrCache( + partialDb, + partialRaw, + { 111111119: { managers: ['ИВАН ПЕТРОВ ТЕСТОВ'] } }, + { omit: ['444444447', '777777773'] }, + ); + assert.throws( + () => runLoad({ TR_CACHE_DB: partialDb, TR_RAW_DIR: partialRaw }), + /REFUSE TO LOAD[\s\S]*covers \d+ of \d+/, + ); +}); + +test('--allow-partial-tr is the deliberate, stated override', () => { + // Without an override a single permanently unreachable ЕИК would deadlock the pipeline forever. + const partialDb = path.join(dir, 'partial-ok.sqlite'); + const partialRaw = path.join(dir, 'partial-ok-deeds'); + buildTrCache(partialDb, partialRaw, {}, { omit: ['121212129'] }); + assert.doesNotThrow(() => + execFileSync( + 'node', + [ + '--import', + path.join(HERE, 'register-ts.mjs'), + path.join(HERE, 'load.mjs'), + '--allow-partial-tr', + ], + { + cwd: ROOT, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: partialDb, + TR_RAW_DIR: partialRaw, + }, + stdio: 'pipe', + }, + ), + ); + runLoad(); // restore the full-cache state for any later reader +}); + +test('the candidate ЕИК list is written for the crawler, covering held links too', () => { + runLoad(); + const listed = fs + .readFileSync(path.join(STAGING, 'candidate-eiks.txt'), 'utf8') + .split('\n') + .filter(Boolean); + const db = open(); + const all = db + .prepare('SELECT DISTINCT eik FROM interest_links') + .all() + .map((r) => r.eik); + db.close(); + // Every resolved ЕИК, not just the published ones: a link held for want of evidence still needs a + // deed to explain why it is held. + for (const e of all) assert.ok(listed.includes(e), `${e} missing from candidate-eiks.txt`); + assert.equal(new Set(listed).size, listed.length, 'no duplicates — each ЕИК costs one request'); +}); + +test('every link carries an evidence seal, and no seal carries a name', () => { + runLoad(); + const db = open(); + const links = db.prepare('SELECT link_key, publish_tier, status FROM interest_links').all(); + const seals = db.prepare('SELECT * FROM interest_link_evidence').all(); + assert.equal(seals.length, links.length, 'a seal for EVERY link, held and withdrawn included'); + + const CLOSED = /^(?:seat:[\p{Lu} -]+|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + for (const s of seals) { + if (s.matched_fact != null) assert.match(s.matched_fact, CLOSED, s.matched_fact); + assert.ok(s.rules_version.length > 0); + assert.match(s.lookup_date, /^\d{4}-\d{2}-\d{2}$/); + } + // The rail, stated as an assertion rather than a convention. Scope it precisely: `link_key` carries + // the OFFICIAL's own identity by design — they are named on the public surface from their own + // declaration — so the rail is about everyone else. No name that exists only inside a registry deed + // (a co-owner, a manager, the relative who actually holds a family stake) may reach a sealed column. + const evidenceOnly = seals.map(({ link_key: _ignored, ...rest }) => rest); + const blob = JSON.stringify(evidenceOnly).toUpperCase(); + for (const thirdParty of [ + 'РОДНИНА', // the relative who owns the family company + 'СЪВСЕМ ДРУГ СОБСТВЕНИК', // a registry co-owner nobody declared + 'НЯКОЙ ДРУГ', + 'ДРУГ СОБСТВЕНИК', + ]) + assert.ok(!blob.includes(thirdParty), `a third-party name reached the seal: ${thirdParty}`); + // …and the declarant's name must not be duplicated into the evidence fields either. + for (const surname of ['ТЕСТОВ', 'ДИВЕСТОВ', 'ПАРТНЬОРОВ']) + assert.ok(!blob.includes(surname), `a declarant name reached an evidence column: ${surname}`); + db.close(); +}); + +test('a family stake publishes ONLY when the official confirmed the company themselves', () => { + // The positive control for the narrowest published path. A family link can never earn „Документ" — + // the registered owner is the relative, whose name we never hold — so it stands or falls on the seat + // or ЕИК the OFFICIAL declared. Without this case „family published: 0" would be indistinguishable + // from a structurally dead path (ADR-0027). + runLoad(); + const db = open(); + const withSeat = db + .prepare( + "SELECT il.status, il.publish_tier FROM interest_links il JOIN persons p ON p.id=il.person_id WHERE il.eik='191919199' AND p.name='Кметица Иванова Втора'", + ) + .get(); + assert.equal(withSeat.status, 'published'); + assert.equal(withSeat.publish_tier, 'confirmed'); + const seal = db + .prepare( + "SELECT matched_fact FROM interest_link_evidence WHERE link_key LIKE '%191919199|family'", + ) + .get(); + assert.equal(seal.matched_fact, 'seat:РУСЕ'); + db.close(); +}); diff --git a/scripts/cacbg/tr-census.mjs b/scripts/cacbg/tr-census.mjs deleted file mode 100644 index e7b8c66d7..000000000 --- a/scripts/cacbg/tr-census.mjs +++ /dev/null @@ -1,195 +0,0 @@ -// TR name-uniqueness census (ADR-0015). Promotes tier-C held interest_links — generic company names -// with a single WINNER namesake — to 'published' iff the name is GLOBALLY unique in the Trade Register. -// -// Source: the Commercial Register open-data dump on data.egov.bg (DPA-safe: ЕГН/ЛНЧ hashed out, company -// name + ЕИК retained). We build companyNameKey(name) → {distinct ЕИК} over ALL active entities using -// the SAME normalizer as the matcher, so the key spaces are identical. Promotion is deterministic: -// key count == 1 AND that ЕИК == the matched winner ЕИК. Anything else stays held. No heuristic. -// -// Field-detection is structural (find the 9/13-digit ЕИК; find the company-name field), so it is robust -// to the exact open-data column names — confirm the mapping against the real dump before a production run. -// -// Run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/tr-census.mjs --dump -import { DatabaseSync } from 'node:sqlite'; -import fs from 'node:fs'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); -const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); -const arg = (n) => { - const i = process.argv.indexOf(`--${n}`); - return i >= 0 ? process.argv[i + 1] : undefined; -}; -const DUMP = arg('dump') || process.env.TR_DUMP; -const { companyNameKey } = await import('../../packages/shared/src/company-name-key.ts'); - -const isEik = (v) => typeof v === 'string' && /^\d{9}$|^\d{13}$/.test(v.trim()); - -// The register stores CompanyName WITHOUT the legal form and LegalForm as a code; our matcher keys on the -// name WITH its Bulgarian form (bidders.name = «…» ЕООД), so we must reconstruct it or every lookup misses. -// Codes seen in the real dump: EOOD OOD ASSOC CC K EAD ET AD KCHT FOUND KD SD DPK EDPK IAD IEAD. -const LEGAL_FORM = { - EOOD: 'ЕООД', - OOD: 'ООД', - EAD: 'ЕАД', - AD: 'АД', - ET: 'ЕТ', - KD: 'КД', - K: 'КД', - SD: 'СД', - KCHT: 'КДА', - IAD: 'ИАД', - IEAD: 'ИЕАД', - DPK: 'ПК', - EDPK: 'ЕПК', - CC: 'кооперация', - ASSOC: 'СНЦ', - FOUND: 'фондация', -}; -// A Bulgarian trade name reconstructed to match the matcher's key space. Unknown form → append the raw -// code (harmless: it just forms its own key that no bidder lookup will hit). -const bgName = (companyName, legalForm) => - `${String(companyName ?? '').trim()} ${LEGAL_FORM[legalForm] ?? String(legalForm ?? '').trim()}`.trim(); - -// Extract {eik, name} from ONE TR deed's attribute object ({CompanyName, LegalForm, UIC, DeedStatus, …}). -// Also tolerates a flat {eik/uic, name} record (test fixtures). ЕИК = the 9/13-digit UIC. -export function extractEntity(rec) { - const uic = rec.UIC ?? rec.uic ?? rec.eik ?? rec.EIK; - const eik = isEik(String(uic ?? '')) ? String(uic).trim() : null; - const name = - rec.CompanyName != null ? bgName(rec.CompanyName, rec.LegalForm) : (rec.name ?? null); - return { eik, name: name || null }; -} - -// Read a TR dump as deed-attribute records. The real data.egov.bg export nests them at -// Message[].Body[].Deeds[].Deed[], each carrying its fields under `$` (fast-xml/attr style). Falls back to -// a plain JSON array / {data:[…]} / JSONL of already-flat records so tests and other shapes still work. -function* records(dump) { - const raw = fs.readFileSync(dump, 'utf8'); - const trimmed = raw.trimStart(); - if (trimmed.startsWith('{') || trimmed.startsWith('[')) { - const parsed = JSON.parse(raw); - const deeds = parsed?.Message?.[0]?.Body?.[0]?.Deeds?.[0]?.Deed; - if (Array.isArray(deeds)) { - for (const d of deeds) yield d?.['$'] ?? d; - return; - } - const arr = Array.isArray(parsed) - ? parsed - : (parsed.data ?? parsed.records ?? parsed.result ?? []); - yield* arr; - } else { - for (const line of raw.split('\n')) { - const t = line.trim(); - if (t) yield JSON.parse(t); - } - } -} - -// Build companyNameKey → Set(ЕИК) over one or more dump files (accumulates — call with every daily file to -// approach the full register). Deduped by ЕИК within a key, so the same company appearing on many days -// (or a name→one company) counts once; two DISTINCT companies sharing a name yield size>1 = not unique. -export function buildCensus(dumps) { - const census = new Map(); - for (const dump of Array.isArray(dumps) ? dumps : [dumps]) { - for (const rec of records(dump)) { - const { eik, name } = extractEntity(rec); - if (!eik || !name) continue; - const key = companyNameKey(name); - if (!census.has(key)) census.set(key, new Set()); - census.get(key).add(eik); - } - } - return census; -} - -// Promote held tier-C links the census proves globally unique: name-key maps to EXACTLY one ЕИК AND that -// ЕИК is the matched winner's (a national namesake, even at a different ЕИК, keeps the link held). With -// dryRun, report the would-promote set without writing — used to validate against a PARTIAL census, which -// must never actually promote (an un-ingested namesake would make a false-unique claim = libel). -export function promote(db, census, { dryRun = false, minEik = null, forcePartial = false } = {}) { - // Partial-census guard (libel gate). A census smaller than the real ТР register can make a genuinely - // non-unique name look unique (`eiks.size === 1`) → a false, libelous attribution. A real promote must - // ASSERT coverage in code, not by convention: pass `--min-eik ≥ register size` (the census's distinct-ЕИК - // count must meet it) or the explicit `--force-partial` escape hatch. dryRun is exempt — it writes nothing - // and exists precisely to inspect a partial census. - if (!dryRun && !forcePartial) { - if (!Number.isInteger(minEik) || minEik < 1) { - throw new Error( - 'refusing to promote without --min-eik : a partial census silently fabricates ' + - 'false-unique attributions. Pass --min-eik ≥ the known ТР register size, or --force-partial to override.', - ); - } - const distinctEik = new Set(); - for (const s of census.values()) for (const e of s) distinctEik.add(e); - if (distinctEik.size < minEik) { - throw new Error( - `refusing to promote from a partial census: ${distinctEik.size} distinct ЕИК < --min-eik ${minEik}. ` + - 'Ingest more ТР dumps until coverage meets the register size, or pass --force-partial to override.', - ); - } - } - const held = db - .prepare( - "SELECT link_key, eik, entity_key FROM interest_links WHERE status='held' AND publish_tier='C_hold'", - ) - .all(); - const upd = db.prepare( - "UPDATE interest_links SET status='published', match_method='exact_name_key+tr_census' WHERE link_key=?", - ); - const would = []; - if (!dryRun) db.exec('BEGIN'); - for (const l of held) { - const eiks = census.get(l.entity_key); - if (eiks && eiks.size === 1 && eiks.has(l.eik)) { - would.push(l.link_key); - if (!dryRun) upd.run(l.link_key); - } - } - if (!dryRun) db.exec('COMMIT'); - return { promoted: would.length, stillHeld: held.length - would.length, would }; -} - -// Resolve --dump: a single file, a comma-list, or --dump-dir (every *.json/*.jsonl inside). -function dumpFiles() { - const dir = arg('dump-dir'); - if (dir) - return fs - .readdirSync(dir) - .filter((f) => /\.(json|jsonl)$/.test(f)) - .map((f) => path.join(dir, f)); - return DUMP - ? DUMP.split(',') - .map((s) => s.trim()) - .filter(Boolean) - : []; -} - -// CLI entry (guarded so importing this module in tests has no side effects) -const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); -if (isMain) { - const files = dumpFiles(); - const dryRun = process.argv.includes('--dry-run'); - const forcePartial = process.argv.includes('--force-partial'); - const minEik = arg('min-eik') !== undefined ? Number(arg('min-eik')) : null; - if (!files.length) { - console.log( - 'no --dump/--dump-dir provided; census not run. Provide TR open-data files to promote tier-C links.', - ); - } else { - const db = new DatabaseSync(DB); - const census = buildCensus(files); - const eikTotal = [...census.values()].reduce((n, s) => n + s.size, 0); - console.log( - `census: ${files.length} file(s) → ${census.size} distinct name-keys / ${eikTotal} ЕИК`, - ); - const { promoted, stillHeld, would } = promote(db, census, { dryRun, minEik, forcePartial }); - console.log( - `${dryRun ? 'DRY-RUN would promote' : 'tier-C promotions'}: ${promoted} published, ${stillHeld} still held (non-unique or namesake mismatch)`, - ); - if (dryRun && would.length) - console.log(' would-promote link_keys:\n ' + would.join('\n ')); - db.close(); - } -} diff --git a/scripts/cacbg/tr-census.test.mjs b/scripts/cacbg/tr-census.test.mjs deleted file mode 100644 index 76d2fbb0f..000000000 --- a/scripts/cacbg/tr-census.test.mjs +++ /dev/null @@ -1,121 +0,0 @@ -// TR census — real data.egov.bg deed shape + deterministic tier-C promotion. -// Run: node --import ./scripts/cacbg/register-ts.mjs --test scripts/cacbg/tr-census.test.mjs -import { test, before, after } from 'node:test'; -import assert from 'node:assert/strict'; -import { DatabaseSync } from 'node:sqlite'; -import fs from 'node:fs'; -import path from 'node:path'; -import os from 'node:os'; -import { extractEntity, buildCensus, promote } from './tr-census.mjs'; -const { companyNameKey } = await import('../../packages/shared/src/company-name-key.ts'); - -let dir, dump, dbPath; -const K = (s) => companyNameKey(s); - -before(() => { - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-census-')); - dump = path.join(dir, 'tr.json'); - // Real export shape: Message[].Body[].Deeds[].Deed[], each with its fields under `$`; CompanyName is - // bare and LegalForm is a code — the census must reconstruct «name + Bulgarian form» to match bidders. - const deeds = [ - { CompanyName: 'СИЙ', LegalForm: 'AD', UIC: '444444447' }, // globally unique → promote - { CompanyName: 'ОБЩА ФИРМА', LegalForm: 'OOD', UIC: '555555556' }, // one of two namesakes - { CompanyName: 'Обща Фирма', LegalForm: 'OOD', UIC: '666666663' }, // second namesake → NOT unique - { CompanyName: 'СИЙ', LegalForm: 'AD', UIC: '444444447' }, // same deed on another day → deduped - ]; - fs.writeFileSync( - dump, - JSON.stringify({ Message: [{ Body: [{ Deeds: [{ Deed: deeds.map((d) => ({ $: d })) }] }] }] }), - ); - - dbPath = path.join(dir, 'db.sqlite'); - const db = new DatabaseSync(dbPath); - db.exec(` - CREATE TABLE interest_links(link_key TEXT PRIMARY KEY, eik TEXT, entity_key TEXT, status TEXT, publish_tier TEXT, match_method TEXT); - INSERT INTO interest_links VALUES ('p1|444444447','444444447','${K('СИЙ АД')}','held','C_hold','exact_name_key'); - INSERT INTO interest_links VALUES ('p2|555555556','555555556','${K('ОБЩА ФИРМА ООД')}','held','C_hold','exact_name_key'); - `); - db.close(); -}); - -after(() => fs.rmSync(dir, { recursive: true, force: true })); - -test('extractEntity reconstructs «name + Bulgarian legal form» from a TR deed and reads UIC as ЕИК', () => { - assert.deepEqual(extractEntity({ CompanyName: 'СИЙ', LegalForm: 'AD', UIC: '444444447' }), { - eik: '444444447', - name: 'СИЙ АД', - }); - assert.deepEqual(extractEntity({ CompanyName: 'ТЕСТ', LegalForm: 'EOOD', UIC: '205057459' }), { - eik: '205057459', - name: 'ТЕСТ ЕООД', - }); - assert.equal(extractEntity({ CompanyName: 'X', LegalForm: 'AD', UIC: '2018060520' }).eik, null); // 10 digits ≠ ЕИК - assert.equal(extractEntity({ name: 'flat ООД', uic: '5555555560000' }).eik, '5555555560000'); // flat fallback + 13-digit -}); - -test('buildCensus indexes name-key → set of ЕИК over the nested export (dedup by ЕИК)', () => { - const c = buildCensus(dump); - assert.equal(c.get(K('СИЙ АД')).size, 1); // appears twice, one ЕИК → deduped - assert.equal(c.get(K('ОБЩА ФИРМА ООД')).size, 2); // two distinct ЕИК fold to one key → non-unique -}); - -test('promote publishes only globally-unique tier-C links; shared names stay held', () => { - const db = new DatabaseSync(dbPath); - // census covers 3 distinct ЕИК → assert coverage with --min-eik 3 (the partial-census gate). - const res = promote(db, buildCensus(dump), { minEik: 3 }); - assert.equal(res.promoted, 1); - const rows = new Map( - db - .prepare('SELECT link_key,status,match_method FROM interest_links') - .all() - .map((r) => [r.link_key, r]), - ); - assert.equal(rows.get('p1|444444447').status, 'published'); - assert.match(rows.get('p1|444444447').match_method, /tr_census/); - assert.equal(rows.get('p2|555555556').status, 'held'); // two namesakes → stays held - db.close(); -}); - -test('partial-census gate: a real promote refuses without --min-eik, and when coverage is below it', () => { - const db = new DatabaseSync(dbPath); - // isolate: p1 back to held (an earlier test may have published it — shared dbPath) - db.exec( - "UPDATE interest_links SET status='held', match_method='exact_name_key' WHERE link_key='p1|444444447'", - ); - const census = buildCensus(dump); // 3 distinct ЕИК - // no --min-eik → refuse (can't assert completeness → could fabricate a false-unique attribution) - assert.throws(() => promote(db, census, {}), /--min-eik/); - // --min-eik above the census's coverage → refuse (partial dump) - assert.throws(() => promote(db, census, { minEik: 4 }), /partial census|distinct ЕИК/); - // the refused runs published nothing — p1 is still held - assert.equal( - db.prepare("SELECT status FROM interest_links WHERE link_key='p1|444444447'").get().status, - 'held', - ); - db.close(); -}); - -test('partial-census gate: --force-partial is the explicit override', () => { - const db = new DatabaseSync(dbPath); - db.exec( - "UPDATE interest_links SET status='held', match_method='exact_name_key' WHERE link_key='p1|444444447'", - ); - const res = promote(db, buildCensus(dump), { forcePartial: true }); // no minEik, but forced - assert.equal(res.promoted, 1); - db.close(); -}); - -test('dry-run reports would-promote set without mutating the DB', () => { - const db = new DatabaseSync(dbPath); - // reset p1 back to held for an isolated dry-run check - db.exec( - "UPDATE interest_links SET status='held', match_method='exact_name_key' WHERE link_key='p1|444444447'", - ); - const res = promote(db, buildCensus(dump), { dryRun: true }); - assert.deepEqual(res.would, ['p1|444444447']); - assert.equal( - db.prepare("SELECT status FROM interest_links WHERE link_key='p1|444444447'").get().status, - 'held', - ); // untouched - db.close(); -}); diff --git a/scripts/tr/evidence.mjs b/scripts/tr/evidence.mjs index 262e31eb0..f6619dad1 100644 --- a/scripts/tr/evidence.mjs +++ b/scripts/tr/evidence.mjs @@ -157,9 +157,16 @@ export function evidenceVerdict(input) { // The weakest publishing rung, so it carries the extra AND-gate: a nationally shared company name // cannot ride it (ADR-0017's holding, carried forward). The stronger „Документ" rung above is not // gated — the register named this person in THIS company, which makes the name key moot. + // The declared-ЕИК leg is NOT name-gated. ADR-0028: the ЕИК is the identity, not the name, so it + // resolves the company deterministically even behind a nationally shared фирма — which is exactly the + // case ADR-0017 was written about. Gating it on name uniqueness would discard the strongest + // identifier we have precisely where it is most needed. + if (declaredEik) return verdict('confirmed', true, { matchedFact: 'eik' }); + + // The SEAT leg is name-gated, and only this one. ADR-0017's holding carried forward: a name shared by + // two ЕИК cannot support a name-derived identity claim. The seat still rescues a GENERIC name — that + // is the whole point of the rung — it just cannot rescue a NATIONALLY SHARED one. if (nameGloballyUnique) { - if (declaredEik) return verdict('confirmed', true, { matchedFact: 'eik' }); - const seat = registrySeat(deed); // Empty NEVER confirms — otherwise every link with no seat data on either side rubber-stamps. if (seat.settlement !== '') { diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs index 7c43b7971..858db9194 100644 --- a/scripts/tr/evidence.test.mjs +++ b/scripts/tr/evidence.test.mjs @@ -176,6 +176,38 @@ test('rung 3 — the weakest rung ALSO requires global name uniqueness (ADR-0017 assert.notEqual(v.kind, 'confirmed', 'a nationally shared name cannot ride the weakest rung'); }); +test('rung 3 — a declared ЕИК is NOT gated by name uniqueness (ADR-0028)', () => { + // The case ADR-0017 was written about — a фирма backing two ЕИК — is exactly where a declarant-supplied + // ЕИК is most valuable. Gating it on the name would discard the strongest identifier precisely when the + // name is useless. + const other = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК'))]); + const v = evidenceVerdict({ + ...base, + deed: other, + declaredEik: true, + nameGloballyUnique: false, + }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'eik'); +}); + +test('rung 3 — the seat rung DOES rescue a merely generic name; it is uniqueness that gates it', () => { + // The seat leg exists to rescue generic names (a bare one- or two-word фирма). Requiring the name to + // be distinctive would empty the rung of its entire purpose; only NATIONAL non-uniqueness blocks it. + const generic = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: generic, + declaredSeats: ['Пловдив'], + nameGloballyUnique: true, + }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'seat:ПЛОВДИВ'); +}); + test('rung 3 — name uniqueness does NOT gate the stronger „Документ" rung', () => { const v = evidenceVerdict({ ...base, nameGloballyUnique: false }); assert.equal( From 2a087443344dade66c2b2393fbdda43b8a41af1d Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 08:41:56 +0300 Subject: [PATCH 10/50] feat(web): the surface explains the registry fact each link rests on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read path, the DTO, the card and the audit (#279, ADR-0033). This is what makes „every shown link explains itself" true rather than a promise: a reader sees which act was read, which entry, and when — and can open the same act. THE PREDICATE HAS THREE COPIES and all three moved together. SURFACED_OWNERSHIP in related-persons.ts is the one people find; the other two are precompute.sql and refresh-slice.sql, and the second of those runs on the 6-hourly cron feeding the officials search index. Missing it would leave officials findable whose links no longer surface — a stale index that contradicts the page it links to. The card's wording is load-bearing and tested as such. „лицето е вписано като съдружник/собственик" reports what the act RECORDS; it does not say the official owns anything, because that claim comes from their own declaration and is rendered separately as „дялово участие". „Потвърдено" says the COMPANY was identified from something the official declared — nobody was found in the act — so its label must not imply anyone was, and a test asserts it never contains „вписан" or „собственик". audit.mjs's C axis was dead code the moment the loader stopped emitting B_distinctive: it re-derived name distinctiveness, a rule no longer in force. It now re-derives the rule that IS in force — a published link must carry a seal, the seal must be a publishing rung, and the tier must agree with it — plus the PII rail as an audited invariant rather than a convention: matched_fact is a closed vocabulary, so a name reaching it is a hard finding. Five new cases, including a positive control, without which all four negatives would still pass if the axes fired unconditionally. Eight test fixtures needed 0006 or evidence seals. That is not incidental: the read gate now requires a seal, so a fixture without one renders an EMPTY surface and every assertion below it passes vacuously. related-persons-sql.test.ts seals its links with a SELECT over interest_links, so a row added later is sealed automatically and cannot silently drop off the surface. Two things the browser check caught that no unit test could. The dev server was serving pre-change HTML because my local vite config pinned __SIGMA_DEPLOY_TAG__ to a constant — the worker prefixes that tag into its edge-cache key, so the key never rotated and stale HTML survived every restart. That is exactly the failure the real vite.config.ts warns about, reproduced by ignoring it. And the row's
is CSS-uppercased, so the assertion had to be case-insensitive. --- apps/web/app/components/ConflictCards.tsx | 16 ++++ apps/web/app/lib/conflicts.test.ts | 37 +++++++- apps/web/app/lib/conflicts.ts | 18 ++++ .../app/routes/conflict.pages.render.test.tsx | 8 +- apps/web/app/routes/conflicts.render.test.tsx | 30 ++++++ packages/api-contract/src/index.ts | 9 ++ .../db/src/contractor-identity-sql.test.ts | 7 +- .../etl-entity-canonicalization-sql.test.ts | 3 + packages/db/src/integrity-checks.test.ts | 3 + packages/db/src/migrations.test.ts | 1 + packages/db/src/queries/related-persons.ts | 31 +++++- packages/db/src/refresh-slice.test.ts | 8 ++ packages/db/src/related-persons-sql.test.ts | 24 ++++- scripts/cacbg/audit.mjs | 50 ++++++++-- scripts/cacbg/audit.test.mjs | 95 ++++++++++++++++++- scripts/precompute.sql | 7 ++ scripts/refresh-slice.sql | 7 ++ 17 files changed, 334 insertions(+), 20 deletions(-) diff --git a/apps/web/app/components/ConflictCards.tsx b/apps/web/app/components/ConflictCards.tsx index 4bbfeba41..28783efd3 100644 --- a/apps/web/app/components/ConflictCards.tsx +++ b/apps/web/app/components/ConflictCards.tsx @@ -20,6 +20,7 @@ import { officialHref, partitionContracts, relationLabel, + registryEvidenceLabel, temporalLabel, } from '../lib/conflicts'; @@ -181,6 +182,21 @@ function ConflictCard({ )}
+ {/* The Trade Register fact the link's identity rests on (#279, ADR-0033). This is what makes + „every shown link explains itself" true rather than a promise: a reader can open the same act + we read and check it. The wording is careful — the register records a ROLE, it does not + certify the ownership claim, which comes from the official's own declaration. */} +
+
Регистър
+
+ + + {registryEvidenceLabel(l)} + {l.registryEntryDate ? ` · вписване ${l.registryEntryDate}` : ''} + {l.registryLookupDate ? ` · справка ${l.registryLookupDate}` : ''} + +
+
{l.contractCount > 0 && ( diff --git a/apps/web/app/lib/conflicts.test.ts b/apps/web/app/lib/conflicts.test.ts index a4bfd5f8b..6cec490ac 100644 --- a/apps/web/app/lib/conflicts.test.ts +++ b/apps/web/app/lib/conflicts.test.ts @@ -21,6 +21,7 @@ import { partitionContracts, relationLabel, temporalLabel, + registryEvidenceLabel, } from './conflicts'; function link(over: Partial = {}): ConflictLink { @@ -43,7 +44,13 @@ function link(over: Partial = {}): ConflictLink { contemporaneousValueEur: 40_000_000, firstContractYear: '2021', lastContractYear: '2024', - sourceUrl: 'https://register.cacbg.bg/2024/i.xml', + sourceUrl: 'https://register.cacbg.bg/2024/x.xml', + // #279: a link only reaches the DTO when its identity rests on a Trade Register fact. + evidenceKind: 'document', + registryRole: 'owner', + registryEntryNumber: '20110502101007', + registryEntryDate: '2011-05-02', + registryLookupDate: '2026-08-05', ...over, }; } @@ -523,3 +530,31 @@ describe('authorityShareDisplay', () => { }); }); }); + +describe('registryEvidenceLabel', () => { + // The wording is load-bearing. The register records a ROLE; it does not certify that the official owns + // anything — that claim comes from their own declaration and is rendered separately. A label that said + // „собственик според ТР" would assert something the evidence does not support (ADR-0033 decision 2). + it('reports what the act records, never an ownership conclusion', () => { + expect(registryEvidenceLabel({ evidenceKind: 'document', registryRole: 'owner' })).toBe( + 'лицето е вписано като съдружник/собственик', + ); + expect(registryEvidenceLabel({ evidenceKind: 'document', registryRole: 'manager' })).toBe( + 'лицето е вписано като управител', + ); + }); + + it('a seat/ЕИК confirmation claims identity, not a registry role', () => { + // „Потвърдено" means the COMPANY was identified from something the official declared — nobody was + // found in the act, so the label must not imply anyone was. + const label = registryEvidenceLabel({ evidenceKind: 'confirmed', registryRole: null }); + expect(label).toBe('самоличност, потвърдена по декларирани данни'); + expect(label).not.toMatch(/вписан/); + }); + + it('never renders the word „собственик" for a mere confirmation', () => { + expect(registryEvidenceLabel({ evidenceKind: 'confirmed', registryRole: 'owner' })).not.toMatch( + /собственик/, + ); + }); +}); diff --git a/apps/web/app/lib/conflicts.ts b/apps/web/app/lib/conflicts.ts index 50a6f16ba..5b99180ed 100644 --- a/apps/web/app/lib/conflicts.ts +++ b/apps/web/app/lib/conflicts.ts @@ -24,6 +24,24 @@ const RELATION_LABEL: Record = { }; /** Bulgarian label for a declared relation. Unknown values pass through — never invent a stronger claim. */ +/** + * How the company's identity was established, in the register's own terms (#279, ADR-0033). + * + * Deliberately does NOT say the official owns anything: „вписан съдружник/собственик" reports what the + * act RECORDS, while the ownership claim itself comes from the official's own declaration and is + * rendered separately as „дялово участие". „Потвърдено" means the company was identified by a fact the + * official declared — the seat or the ЕИК — not that anybody was found in the act. + */ +export function registryEvidenceLabel(l: { + evidenceKind: 'document' | 'confirmed'; + registryRole: 'owner' | 'manager' | null; +}): string { + if (l.evidenceKind === 'confirmed') return 'самоличност, потвърдена по декларирани данни'; + return l.registryRole === 'manager' + ? 'лицето е вписано като управител' + : 'лицето е вписано като съдружник/собственик'; +} + export function relationLabel(relation: string): string { return RELATION_LABEL[relation] ?? relation; } diff --git a/apps/web/app/routes/conflict.pages.render.test.tsx b/apps/web/app/routes/conflict.pages.render.test.tsx index c6f2498cf..fbfc4fdb4 100644 --- a/apps/web/app/routes/conflict.pages.render.test.tsx +++ b/apps/web/app/routes/conflict.pages.render.test.tsx @@ -34,7 +34,13 @@ function link(over: Partial = {}): ConflictLink { contemporaneousValueEur: 30_000_000, firstContractYear: '2020', lastContractYear: '2024', - sourceUrl: 'https://register.cacbg.bg/2024/i.xml', + sourceUrl: 'https://register.cacbg.bg/2024/x.xml', + // #279: a link only reaches the DTO when its identity rests on a Trade Register fact. + evidenceKind: 'document', + registryRole: 'owner', + registryEntryNumber: '20110502101007', + registryEntryDate: '2011-05-02', + registryLookupDate: '2026-08-05', ...over, }; } diff --git a/apps/web/app/routes/conflicts.render.test.tsx b/apps/web/app/routes/conflicts.render.test.tsx index 68dad5fbb..ce2a78dbe 100644 --- a/apps/web/app/routes/conflicts.render.test.tsx +++ b/apps/web/app/routes/conflicts.render.test.tsx @@ -36,6 +36,12 @@ function link(over: Partial = {}): ConflictLink { firstContractYear: '2020', lastContractYear: '2024', sourceUrl: 'https://register.cacbg.bg/2024/i.xml', + // #279: a link only reaches the DTO when its identity rests on a Trade Register fact. + evidenceKind: 'document', + registryRole: 'owner', + registryEntryNumber: '20110502101007', + registryEntryDate: '2011-05-02', + registryLookupDate: '2026-08-05', ...over, }; } @@ -238,3 +244,27 @@ describe('/conflicts route — render', () => { ).not.toBeNull(); }); }); + +describe('Trade Register evidence on the card (#279, ADR-0033)', () => { + it('renders the registry fact the link rests on, so the card explains itself', async () => { + await renderConflicts([link({ evidenceKind: 'document', registryRole: 'owner' })]); + const text = container.textContent ?? ''; + expect(text).toContain('Регистър'); + expect(text).toContain('лицето е вписано като съдружник/собственик'); + expect(text).toContain('вписване 2011-05-02'); // WHICH entry + expect(text).toContain('справка 2026-08-05'); // and HOW FRESH it is + }); + + it('a seat/ЕИК confirmation never implies somebody was found in the act', async () => { + await renderConflicts([link({ evidenceKind: 'confirmed', registryRole: null })]); + const text = container.textContent ?? ''; + expect(text).toContain('самоличност, потвърдена по декларирани данни'); + expect(text).not.toContain('вписано като'); + }); + + it('links out to the register so a reader can check the same act we read', async () => { + await renderConflicts([link({ eik: '201122335' })]); + const hrefs = [...container.querySelectorAll('a')].map((a) => a.getAttribute('href') ?? ''); + expect(hrefs.some((h) => h.includes('201122335'))).toBe(true); + }); +}); diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index d1d9f7092..9f6fe164d 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -705,6 +705,15 @@ export interface ConflictLink { firstContractYear: string | null; lastContractYear: string | null; sourceUrl: string | null; // a representative declaration URL — provenance, never a fabricated value + // Trade Register evidence (#279, ADR-0033). A link only reaches this DTO when its identity rests on a + // checkable registry fact, so these describe WHICH fact — the surface's whole point is that every shown + // link can explain itself. `registryRole` is the role the register records, NOT a claim about who owns + // what: the ownership claim comes from the official's own declaration. + evidenceKind: 'document' | 'confirmed'; // the only two rungs that publish + registryRole: 'owner' | 'manager' | null; // set only for evidenceKind='document' + registryEntryNumber: string | null; // TEXT — a fieldEntryNumber exceeds the exact-integer range + registryEntryDate: string | null; // the registry entry the evidence rests on + registryLookupDate: string; // when the deed was read — the freshness bound on the claim } /** One contract of a linked winner, marked by whether it was signed during the declared-stake window. diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index efd769fe3..ea4de1cef 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -16,6 +16,11 @@ const migration3 = readFileSync( resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'), 'utf8', ); +// …and 0006, which those same blocks now join for the Trade Register evidence gate (#279, ADR-0033). +const migration6 = readFileSync( + resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'), + 'utf8', +); const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); const normalize = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); const precompute = readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8'); @@ -66,7 +71,7 @@ function build(path: 'normalize' | 'refresh'): DatabaseSync { const db = new DatabaseSync(':memory:'); db.exec(schema); db.exec(migration2); - db.exec(migration3); + db.exec(migration3 + migration6); db.exec(staging); db.exec(seed); if (path === 'normalize') { diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index df67b27e4..9ffdea5b0 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -11,6 +11,8 @@ const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // refresh-slice.sql's officials block reads interest_links (0003); build it so the script doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). +const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -40,6 +42,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index 52acac255..bf81ae93c 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -28,6 +28,8 @@ const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bid const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // precompute.sql's officials block reads interest_links (0003); build it so precompute doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). +const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); const precomputePath = resolve(root, 'scripts/precompute.sql'); function sqlite(dbPath: string, sql: string): void { @@ -71,6 +73,7 @@ function freshDb(): string { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); sqlite(dbPath, CLEAN_FIXTURE); return dbPath; } diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index b92c174b4..fed25dda4 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -132,6 +132,7 @@ describe('served migrations', () => { ); readScript(dbPath, migration2); readScript(dbPath, migration3); + readScript(dbPath, migration6); readScript(dbPath, backfill); readScript(dbPath, precompute); diff --git a/packages/db/src/queries/related-persons.ts b/packages/db/src/queries/related-persons.ts index ce15ff2b6..ff99e7f51 100644 --- a/packages/db/src/queries/related-persons.ts +++ b/packages/db/src/queries/related-persons.ts @@ -82,6 +82,11 @@ interface LinkRow { first_contract_year: string | null; last_contract_year: string | null; source_url: string | null; + evidence_kind: string | null; + registry_role: string | null; + entry_number: string | null; + entry_date: string | null; + lookup_date: string | null; } // The winner's contracts, joined exactly as the ETL aggregate does (contracts→tenders→authorities→bidders, @@ -119,8 +124,15 @@ export const NEXUS_ORDER = `(il.own_institution = 'exact') DESC, (contemporaneou // The two surfaced ownership classes (ADR-0032): the official's own stake (private_ownership) and a close // relative's (family_ownership). Anchored once here + in LINK_CONTRACTS_SQL so the read gate and the // drill-down never drift. +// …and the identity must rest on a Trade Register fact (#279, ADR-0033). `status='published'` already +// encodes the loader's decision, so this EXISTS is belt-and-braces: it makes the read path refuse a link +// whose seal is missing or whose evidence is a withholding rung, even if a future writer sets status +// wrongly. 'document' and 'confirmed' are the only two rungs that publish; bar_joint_stock, unknown, +// refuted and outside_tr never reach a reader. export const SURFACED_OWNERSHIP = `il.status = 'published' - AND il.interest_class IN ('private_ownership', 'family_ownership')`; + AND il.interest_class IN ('private_ownership', 'family_ownership') + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed'))`; // Redundant-family collapse (ADR-0032, per todorkolev review). A family link is DROPPED when the SAME official // already has a published OWN stake in the SAME winner. Rendering both a self row and a family row for one // (official, ЕИК) is a de-anonymisation vector: the office-holder is himself in that company's Търговски @@ -159,8 +171,13 @@ export const LINK_SELECT = `SELECT il.link_key, il.person_id, p.name AS official -- (name, institution), ADR-0026; same subquery the search projection uses). Correlated per row, but the -- leaderboard is ≤1000 rows and hourly-cached, so the extra scan is immaterial. (SELECT d.institution FROM declarations d WHERE d.person_id = il.person_id - ORDER BY d.declared_year DESC LIMIT 1) AS institution + ORDER BY d.declared_year DESC LIMIT 1) AS institution, + -- The evidence the link rests on, so the card can explain itself (ADR-0033 decision 7). LEFT JOIN + -- rather than an inner one: SURFACED_OWNERSHIP already requires a publishing seal, and an inner join + -- here would silently re-filter rather than surface a contradiction. + ev.evidence_kind, ev.registry_role, ev.entry_number, ev.entry_date, ev.lookup_date FROM interest_links il + LEFT JOIN interest_link_evidence ev ON ev.link_key = il.link_key JOIN persons p ON p.id = il.person_id JOIN bidders b ON b.id = il.bidder_id WHERE ${SURFACED_OWNERSHIP} @@ -196,6 +213,16 @@ function toLink(r: LinkRow): ConflictLink { firstContractYear: r.first_contract_year, lastContractYear: r.last_contract_year, sourceUrl: r.source_url, + // SURFACED_OWNERSHIP guarantees a publishing seal exists, so these are non-null in practice; the + // fallbacks keep a half-migrated environment rendering rather than throwing on a null read. + evidenceKind: (r.evidence_kind === 'confirmed' ? 'confirmed' : 'document') as + | 'document' + | 'confirmed', + registryRole: + r.registry_role === 'owner' || r.registry_role === 'manager' ? r.registry_role : null, + registryEntryNumber: r.entry_number, + registryEntryDate: r.entry_date, + registryLookupDate: r.lookup_date ?? '', }; } diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 261a28747..82cc2f516 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -13,6 +13,8 @@ const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bid const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // refresh-slice.sql / precompute.sql officials block reads interest_links (0003) — build it in every chain. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). +const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const deriveAmendmentsPath = resolve(root, 'scripts/derive-amendments.sql'); @@ -185,6 +187,7 @@ function initWorkDb(dbPath: string): void { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); } @@ -576,6 +579,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -658,6 +662,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -709,6 +714,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -760,6 +766,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -897,6 +904,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index af3380416..1c88b94f4 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -22,6 +22,8 @@ import { const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006: SURFACED_OWNERSHIP now requires a Trade Register evidence seal (#279, ADR-0033). +const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }).trim(); @@ -122,6 +124,13 @@ INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, co -- Rollup row for the awarding body — the per-authority capture-share denominator the read query LEFT JOINs. INSERT INTO authority_totals (authority_id, name, spent_eur, contracts, suppliers, avg_eur) VALUES ('a:1','ОБЩИНА ТЕСТ',50000000,10,4,5000000); + +-- Every link carries a Trade Register evidence seal (#279, ADR-0033). SURFACED_OWNERSHIP now requires +-- one, so a fixture without seals renders an EMPTY surface and every assertion below passes vacuously. +-- Derived from interest_links itself, so a row added to the fixture later is sealed automatically and +-- cannot silently drop off the surface. +INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links; `; describe('свързани-лица SQL (real SQLite)', () => { @@ -131,6 +140,7 @@ describe('свързани-лица SQL (real SQLite)', () => { try { readScript(dbPath, migration0); readScript(dbPath, migration2); + readScript(dbPath, migration6); sqlite(dbPath, FIXTURE); return fn(dbPath); } finally { @@ -193,7 +203,9 @@ describe('свързани-лица SQL (real SQLite)', () => { INSERT INTO persons (id, name) VALUES ('person:zero','Нула Тестов'); INSERT INTO interest_links (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES - ('il:zerofam','person:zero|666|family','person:zero','eik:666','666','НУЛА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2021',0,0,NULL,NULL,'published');`, + ('il:zerofam','person:zero|666|family','person:zero','eik:666','666','НУЛА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2021',0,0,NULL,NULL,'published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence);`, ); const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); expect(board.some((r) => r.official === 'Нула Тестов')).toBe(false); // no live contracts → gated out @@ -217,6 +229,8 @@ describe('свързани-лица SQL (real SQLite)', () => { (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES ('il:alen','person:alen|701','person:alen','eik:701','701','АЛЕН КО ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'none',1,'2020','2021',1,1000000,'2021','2021','published'), ('il:boyan','person:boyan|702','person:boyan','eik:702','702','БОЯН КО ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'none',1,'2020','2021',1,10000000,'2021','2021','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES ('t:71','unp71','Обект А','a:1','открита процедура'),('t:72','unp72','Обект Б','a:1','открита процедура'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES @@ -322,6 +336,8 @@ describe('свързани-лица SQL (real SQLite)', () => { (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES ('il:dubl-self','person:dubl|800','person:dubl','eik:800','800','ДУБЪЛ ЕООД','exact_name_key','v1','B_distinctive','owns','private_ownership',0,'none',1,'2020','2022',2,60000,'2021','2022','published'), ('il:dubl-fam','person:dubl|800|family','person:dubl','eik:800b','800','ДУБЪЛ ЕООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2022',2,60000,'2021','2022','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES ('t:80','unp80','Обект Дубъл','a:1','открита процедура'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES ('c:80','t:80','eik:800',60000,'EUR','2021-05-01','Д-80',60000);`, ); @@ -350,6 +366,8 @@ describe('свързани-лица SQL (real SQLite)', () => { (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES ('il:asim-self-held','person:asim|850','person:asim','eik:850','850','АСИМ ЕООД','exact_name_key','v1','C_hold','owns','private_ownership',0,'none',1,'2020','2022',2,40000,'2021','2022','held'), ('il:asim-fam','person:asim|850|family','person:asim','eik:850','850','АСИМ ЕООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2022',2,40000,'2021','2022','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES ('t:85','unp85','Обект Асим','a:1','открита процедура'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES ('c:85','t:85','eik:850',40000,'EUR','2021-05-01','Д-85',40000);`, ); @@ -475,7 +493,9 @@ describe('свързани-лица SQL (real SQLite)', () => { INSERT INTO persons (id, name) VALUES ('person:praz','Празен Тестов'); INSERT INTO interest_links (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES - ('il:praz','person:praz|900','person:praz','eik:900','900','ПРАЗЕН ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'exact',1,'2020','2022',7,7000000,'2021','2022','published');`, + ('il:praz','person:praz|900','person:praz','eik:900','900','ПРАЗЕН ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'exact',1,'2020','2022',7,7000000,'2021','2022','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence);`, ); const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); expect(board.some((r) => r.official === 'Празен Тестов')).toBe(false); // gated out — no live contracts diff --git a/scripts/cacbg/audit.mjs b/scripts/cacbg/audit.mjs index 50ff29334..a072a6d03 100644 --- a/scripts/cacbg/audit.mjs +++ b/scripts/cacbg/audit.mjs @@ -7,7 +7,6 @@ import { DatabaseSync } from 'node:sqlite'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { nameDistinctiveness } from './classify.mjs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); @@ -31,12 +30,21 @@ const published = db ` SELECT il.id, il.link_key, il.person_id, il.eik, il.entity_key, il.match_method, il.publish_tier, il.bidder_id, il.relation, il.contemporaneous, il.contract_value_eur, - b.name AS bidder_name, b.eik_normalized AS bidder_eik, b.eik_valid AS bidder_eik_valid + b.name AS bidder_name, b.eik_normalized AS bidder_eik, b.eik_valid AS bidder_eik_valid, + -- LEFT JOIN, deliberately: a published link with NO seal is the finding, so it must reach the + -- loop rather than be filtered out of it. + e.evidence_kind, e.registry_role, e.matched_fact FROM interest_links il JOIN bidders b ON b.id = il.bidder_id + LEFT JOIN interest_link_evidence e ON e.link_key = il.link_key WHERE il.status = 'published'`, ) .all(); +// The only two evidence rungs that publish (ADR-0033 decision 1). Everything else withholds. +const PUBLISHING_EVIDENCE = new Set(['document', 'confirmed']); +// The closed vocabulary for a sealed matched_fact. Anything else is a potential name leak. +const MATCHED_FACT = /^(?:seat:[\p{Lu} -]+|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + const findings = []; const flag = (link, axis, detail) => findings.push({ axis, link_key: link.link_key, eik: link.eik, detail }); @@ -50,12 +58,16 @@ for (const l of published) { // l.eik is a valid winner BEARING this name key (a stray/mis-attached ЕИК is still caught). The ЕИК+name // double-lock itself is re-proven in the provenance pass below (A_eik_no_provenance). if (!rec) flag(l, 'A_key_missing', `entity_key ${l.entity_key} not found in live bidder set`); - else if (l.publish_tier === 'A_eik') { + else if (l.match_method === 'declared_eik') { + // Identity here is the declarant-provided ЕИК, not the name (ADR-0028), so a name shared by more + // than one winner is legitimate — the ЕИК picks exactly one. Require instead that l.eik is a valid + // winner BEARING this name key, which still catches a stray or mis-attached ЕИК. The ЕИК+name + // double-lock is re-proven independently in the provenance pass below. if (!rec.valid.has(l.eik)) flag( l, 'A_eik_not_winner', - `A_eik published ${l.eik}, not among key ${l.entity_key}'s valid winners {${[...rec.valid].join(',')}}`, + `declared_eik published ${l.eik}, not among key ${l.entity_key}'s valid winners {${[...rec.valid].join(',')}}`, ); } else if (rec.valid.size !== 1) flag( @@ -72,12 +84,34 @@ for (const l of published) { if (!l.bidder_eik_valid) flag(l, 'B_eik_invalid', `published on eik_valid=0 bidder ${l.bidder_name}`); - // C. Tier honesty: B_distinctive must actually be distinctive by the same classifier that gated it. - if (l.publish_tier === 'B_distinctive' && nameDistinctiveness(l.entity_key) !== 'distinctive') + // C. Evidence honesty (#279, ADR-0033). The tier IS the evidence kind now, so the axis that used to + // re-derive name distinctiveness re-derives the publishing rule instead: a published link must + // carry a seal, and that seal must be one of the two rungs that publish. `nameDistinctiveness` + // no longer gates publication at all — it survives only as a withholding filter inside the + // loader — so re-checking it here would assert a rule that is no longer in force. + if (!l.evidence_kind) + flag(l, 'C_no_evidence', `published with no Trade Register evidence seal (${l.bidder_name})`); + else if (!PUBLISHING_EVIDENCE.has(l.evidence_kind)) + flag( + l, + 'C_withholding_evidence', + `published on evidence_kind='${l.evidence_kind}', which withholds (${l.bidder_name})`, + ); + else if (l.evidence_kind !== l.publish_tier) + flag( + l, + 'C_tier_evidence_mismatch', + `publish_tier='${l.publish_tier}' but sealed evidence_kind='${l.evidence_kind}'`, + ); + + // C2. The PII rail, audited rather than assumed: matched_fact is a CLOSED vocabulary and can never + // carry a name. The registry deed's names are read only to produce a boolean and must never reach + // a served column (#279 §9, ADR-0033 decision 5). A schema cannot enforce this; this does. + if (l.matched_fact != null && !MATCHED_FACT.test(l.matched_fact)) flag( l, - 'C_not_distinctive', - `tier B_distinctive but nameDistinctiveness=${nameDistinctiveness(l.entity_key)} (${l.bidder_name})`, + 'C_matched_fact_shape', + `matched_fact='${l.matched_fact}' is outside the closed vocabulary — a name may have leaked`, ); } diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index df36baa84..46d42121e 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -26,7 +26,7 @@ const dirs = []; // Build a fixture DB (bidders + declarations + declared_interests + interest_links), run audit.mjs against // it as a subprocess, and return { threw, out } — threw=true iff the audit exited non-zero (a hard finding). -function buildAndAudit({ bidders, decls = [], dis = [], links }) { +function buildAndAudit({ bidders, decls = [], dis = [], links, seals = [] }) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-audit-')); dirs.push(dir); const DB = path.join(dir, 'fixture.sqlite'); @@ -38,10 +38,16 @@ function buildAndAudit({ bidders, decls = [], dis = [], links }) { CREATE TABLE interest_links( id TEXT PRIMARY KEY, link_key TEXT, person_id TEXT, eik TEXT, entity_key TEXT, match_method TEXT, publish_tier TEXT, bidder_id TEXT, relation TEXT, contemporaneous INT, contract_value_eur REAL, status TEXT); + -- The evidence seal (#279, migration 0006). The audit LEFT JOINs it, so it must exist even when a + -- case deliberately leaves a link unsealed — an unsealed published link is itself a finding. + CREATE TABLE interest_link_evidence( + link_key TEXT PRIMARY KEY, evidence_kind TEXT, registry_role TEXT, matched_fact TEXT, + entry_number TEXT, entry_date TEXT, lookup_date TEXT, rules_version TEXT, live_status TEXT); ${bidders.map((b) => `INSERT INTO bidders VALUES (${b});`).join('\n')} ${decls.map((d) => `INSERT INTO declarations VALUES (${d});`).join('\n')} ${dis.map((d) => `INSERT INTO declared_interests(declaration_id, entity_raw) VALUES (${d});`).join('\n')} ${links.map((l) => `INSERT INTO interest_links VALUES (${l});`).join('\n')} + ${seals.map((e) => `INSERT INTO interest_link_evidence(link_key,evidence_kind,registry_role,matched_fact,lookup_date,rules_version,live_status) VALUES (${e});`).join('\n')} `); db.close(); @@ -73,8 +79,9 @@ test('A_eik behind a colliding name, backed by a real ЕИК+name double-lock, P // the declarant wrote BOTH the ЕИК and the фирма → the double-lock the loader required dis: [`'d1','„ОБЩ" ЕООД, ЕИК 100000001'`], links: [ - `'il1','p1|100000001','p1','100000001','${KEY}','declared_eik','A_eik','b1','owns',0,1000,'published'`, + `'il1','p1|100000001','p1','100000001','${KEY}','declared_eik','confirmed','b1','owns',0,1000,'published'`, ], + seals: [`'p1|100000001','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], }); assert.equal( threw, @@ -93,8 +100,9 @@ test('A_eik published on a ЕИК that is NOT a winner bearing the name key → dis: [`'d1','„ОБЩ" ЕООД, ЕИК 300000003'`], // entity_key claims the colliding name, but the published eik/bidder is the unrelated 300000003 links: [ - `'il1','p1|300000003','p1','300000003','${KEY}','declared_eik','A_eik','b3','owns',0,1000,'published'`, + `'il1','p1|300000003','p1','300000003','${KEY}','declared_eik','confirmed','b3','owns',0,1000,'published'`, ], + seals: [`'p1|300000003','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], }); assert.equal(threw, true, 'a stray-ЕИК A_eik link must fail the gate'); assert.equal( @@ -110,8 +118,9 @@ test('A_eik with no declaration carrying its ЕИК+name → A_eik_no_provenance decls: [`'d1','p1'`], dis: [`'d1','нещо съвсем друго без ЕИК'`], // no ЕИК, no фирма → the double-lock cannot be re-proven links: [ - `'il1','p1|100000001','p1','100000001','${K('„ОБЩ" ЕООД')}','declared_eik','A_eik','b1','owns',0,1000,'published'`, + `'il1','p1|100000001','p1','100000001','${K('„ОБЩ" ЕООД')}','declared_eik','confirmed','b1','owns',0,1000,'published'`, ], + seals: [`'p1|100000001','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], }); assert.equal( threw, @@ -130,9 +139,85 @@ test('a NON-A_eik (name-based) colliding link STILL fails A_multi_eik — the na bidders: COLLIDING_BIDDERS, // exact_name_key published on a name that maps to 2 ЕИК — exactly what the gate must keep rejecting links: [ - `'il1','p1|100000001','p1','100000001','${KEY}','exact_name_key','B_distinctive','b1','owns',0,1000,'published'`, + `'il1','p1|100000001','p1','100000001','${KEY}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'`, ], }); assert.equal(threw, true, 'a name-based colliding published link must still fail'); assert.equal(/A_multi_eik/.test(out), true, 'invariant A still fires for non-A_eik links'); }); + +// ── C: evidence honesty (#279, ADR-0033) ───────────────────────────────────────────────────────── +// The axis that used to re-derive name distinctiveness now re-derives the PUBLISHING rule, because the +// publishing rule changed: identity rests on a registry fact, and nameDistinctiveness survives only as +// a withholding filter inside the loader. Re-checking distinctiveness here would assert a rule that is +// no longer in force — a test that passes while guarding nothing. +test('a published link with NO evidence seal is a hard finding', () => { + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [], // the whole point: nothing sealed it + }); + assert.equal(threw, true, 'publishing without evidence must fail the gate'); + assert.equal(/C_no_evidence/.test(out), true, out); +}); + +test('a link published on a WITHHOLDING rung is a hard finding', () => { + // bar_joint_stock / unknown / outside_tr never publish. If one ever reaches status='published', the + // loader and the seal disagree and a claim is on the surface that no evidence supports. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','unknown','b1','owns',0,1000,'published'`, + ], + seals: [`'p1|100000001','unknown',NULL,NULL,'2026-08-05','tr-rules-1','live'`], + }); + assert.equal(threw, true); + assert.equal(/C_withholding_evidence/.test(out), true, out); +}); + +test('a tier that disagrees with its own seal is a hard finding', () => { + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [`'p1|100000001','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], + }); + assert.equal(threw, true); + assert.equal(/C_tier_evidence_mismatch/.test(out), true, out); +}); + +test('a matched_fact outside the closed vocabulary is a hard finding — the name-leak rail', () => { + // #279 §9: the deed's names are read only to produce a boolean and must never reach a served column. + // A schema cannot enforce a vocabulary, so the audit does. This is the shape a leak would take. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','ИВАН ПЕТРОВ ТЕСТОВ','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'a name in matched_fact must fail the gate'); + assert.equal(/C_matched_fact_shape/.test(out), true, out); +}); + +test('a well-formed seal passes every C axis (positive control)', () => { + // Without this, all four negatives above would still pass if the axes fired unconditionally. + const { threw } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, false, 'a correctly sealed link must not fail the gate'); +}); diff --git a/scripts/precompute.sql b/scripts/precompute.sql index c79a26ef1..0c10e7375 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -245,6 +245,13 @@ FROM interest_links il JOIN persons p ON p.id = il.person_id -- rendering both re-identifies the relative via a ТР owner lookup, and the company is already surfaced -- by the self row. WHERE il.status = 'published' AND il.interest_class IN ('private_ownership', 'family_ownership') + -- …and the identity rests on a Trade Register fact (#279, ADR-0033). This predicate is the THIRD copy + -- of the surface gate — the other two are SURFACED_OWNERSHIP in packages/db/src/queries/related-persons.ts + -- and the sibling block in the other of precompute.sql / refresh-slice.sql. All three must move + -- together: this one feeds the officials search index, so omitting it would keep officials findable + -- whose links no longer surface. + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed')) AND EXISTS (SELECT 1 FROM contracts cc JOIN bidders bb ON bb.id = cc.bidder_id WHERE bb.eik_normalized = il.eik) AND NOT (il.interest_class = 'family_ownership' AND EXISTS ( diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 3f4544bea..c3ff6fdbc 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -1863,6 +1863,13 @@ FROM interest_links il JOIN persons p ON p.id = il.person_id -- rendering both re-identifies the relative via a ТР owner lookup, and the company is already surfaced -- by the self row. WHERE il.status = 'published' AND il.interest_class IN ('private_ownership', 'family_ownership') + -- …and the identity rests on a Trade Register fact (#279, ADR-0033). This predicate is the THIRD copy + -- of the surface gate — the other two are SURFACED_OWNERSHIP in packages/db/src/queries/related-persons.ts + -- and the sibling block in the other of precompute.sql / refresh-slice.sql. All three must move + -- together: this one feeds the officials search index, so omitting it would keep officials findable + -- whose links no longer surface. + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed')) AND EXISTS (SELECT 1 FROM contracts cc JOIN bidders bb ON bb.id = cc.bidder_id WHERE bb.eik_normalized = il.eik) AND NOT (il.interest_class = 'family_ownership' AND EXISTS ( From 796903ae4646abc97c48e3a82dd1b12ce02f173a Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 09:20:04 +0300 Subject: [PATCH 11/50] docs/ci: publish the rule, and put the decisions on a cadence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #279 §9 asks for the publishing decisions to run on a schedule and for the rule to be public. ADR-0021 E10 makes the second one a libel-defence obligation, not a nicety: a reader must be able to re-derive any link. The methodology page now carries the evidence ladder VERBATIM — all six rungs, including the two that hide a link — and, more importantly, the disclosures that make it honest rather than merely stated: • the register carries NO ЕГН, so a three-name match is not proof of identity and a namesake is possible. Said in those words, not implied. • which is why the match must be all three names inside ONE registry entry; initials and Latin-script names are refused. • seats move, so a registered seat only confirms a period it predates. • erased entries are skipped everywhere. • what the evidence proves and what it does not: the register confirms the IDENTITY OF THE COMPANY; it does not certify that the official owns it — that claim is their own declaration. This is why the card says „вписано като съдружник/собственик" and never „собственик според ТР". • the monthly cadence, the entry number and lookup date on every link, and the 35-day retention with no third-party name kept. The divestment promise is corrected too: withdrawal on a later silent filing is an INFERENCE FROM SILENCE whose commonest cause is a finished mandate, so it is now reconciled against the live act before it takes effect. Cadence, deliberately on the EXISTING workflow rather than a second one. A duplicate would carry its own copy of the credential guards, the D1-target guard and the ship floor — and the copy is the one nobody exercises. A scheduled run takes the same path, targeting STAGING (production stays manual, so a prod write remains a deliberate act) and with full_crawl false, so it never touches the register: decisions are a pure function of declarations, cached deeds and contracts, which is exactly what §9 means. Registry lookups get their own monthly workflow because they are the only step with a network budget, and separating them is what keeps the decision run zero-network by construction. It ships nothing, caches the deeds between runs so a refresh stays ~400 requests rather than a full crawl, and treats exit 2 (the rate limiter) as a WARNING, not a failure — the run stopped politely and the cache is resumable, and a red X there would train everyone to ignore the one signal that means „the register asked us to stop". The uploaded artifact is the cache INDEX only; the raw deeds carry third-party names and never leave the runner. Spec §3.3's Phase-0 is closed with what was measured, including the three facts that contradicted the issue: the sustained 429, „not in the register" being an HTTP 200 with an empty body, and fetch failing where node:https succeeds. The LIA gains an addendum for the new source and the runbook two new takedown grounds — a court-annulled entry (чл. 29 ЗТРРЮЛНЦ) and a namesake match. Neither looks like a bug: the pipeline works correctly and the claim is still wrong, so neither is fixed by a rules bump. --- .github/workflows/related-persons-data.yml | 23 +++- .../workflows/related-persons-tr-refresh.yml | 114 ++++++++++++++++++ apps/web/app/routes/conflict.methodology.tsx | 64 +++++++++- docs/runbooks/related-persons-suppression.md | 15 +++ docs/spec/related-persons-foundation.md | 8 ++ docs/spec/related-persons-lia.md | 34 ++++++ 6 files changed, 252 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/related-persons-tr-refresh.yml diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index 16f35c7e1..c09bed4d9 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -16,6 +16,19 @@ name: Related-persons data foundation (build + ship) # XML (fetch.mjs skips files already on disk) — it fills gaps, not a from-scratch rebuild unless the cache is cleared. on: + # DAILY, and deliberately on the EXISTING workflow rather than a second one. #279 §9 asks for the + # publishing decisions to run on a cadence; duplicating the job would create a second ship path with + # its own copy of the credential guards, the D1-target guard and the ship floor — and the copy is the + # one nobody exercises. A scheduled run takes the same path as a manual one, with two differences: + # • it targets STAGING (the environment default below). Production stays manual, so the + # assertD1TargetAuthorized guard keeps a prod write a deliberate act. + # • `full_crawl` is empty and therefore false, so it never crawls the register — it re-runs the + # decisions over the cached corpus, which is exactly what #279 §9 means by a pure, zero-network + # decision. Registry lookups have their own workflow (related-persons-tr-refresh.yml). + # If the corpus cache has been evicted, extract yields nothing, the ship floor refuses, and the run + # fails loudly rather than wiping the surface. + schedule: + - cron: '0 3 * * *' workflow_dispatch: inputs: environment: @@ -34,7 +47,7 @@ permissions: concurrency: # Never let two data-foundation writes to the same environment overlap; do not cancel one in flight. - group: related-persons-data-${{ inputs.environment }} + group: related-persons-data-${{ inputs.environment || 'staging' }} cancel-in-progress: false jobs: @@ -46,7 +59,7 @@ jobs: # (stalled gov-server I/O) — fail rather than burn the slot. The raw corpus is cached (below) so a # re-run resumes instead of re-crawling from scratch. timeout-minutes: 300 - environment: ${{ inputs.environment }} + environment: ${{ inputs.environment || 'staging' }} env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} @@ -54,7 +67,7 @@ jobs: SIGMA_D1_NAME: ${{ vars.SIGMA_D1_NAME }} # The DECLARED intended environment — the independent anchor the ship guard checks the (name, id) pair # against (T48), so a consistent-but-wrong pair can't wipe the wrong DB. See assertD1TargetAuthorized. - SIGMA_SHIP_ENV: ${{ inputs.environment }} + SIGMA_SHIP_ENV: ${{ inputs.environment || 'staging' }} # Suppression (takedown) list keying — load.mjs fingerprints link-suppressions.jsonl under this salt and # refuses to build if the list is non-empty but the salt is unset (fail-closed, ADR-0031). Without this # the first suppression entry would break every ship; the key version guards a coordinated salt rotation. @@ -93,14 +106,14 @@ jobs: sigma-blue|sigma-green) IS_PROD_SLOT=1 ;; *) IS_PROD_SLOT=0 ;; esac - if [ "${{ inputs.environment }}" = "production" ]; then + if [ "${{ inputs.environment || 'staging' }}" = "production" ]; then if [ "$IS_PROD_SLOT" != "1" ]; then echo "::error::production requires SIGMA_D1_NAME to be a prod slot (sigma-blue|sigma-green), got '${SIGMA_D1_NAME}' — refusing to ship." exit 1 fi else if [ "$IS_PROD_SLOT" = "1" ]; then - echo "::error::SIGMA_D1_NAME='${SIGMA_D1_NAME}' (a PRODUCTION slot) on a '${{ inputs.environment }}' run — a rebuild would WIPE prod. Set a non-production SIGMA_D1_NAME variable for this Environment." + echo "::error::SIGMA_D1_NAME='${SIGMA_D1_NAME}' (a PRODUCTION slot) on a '${{ inputs.environment || 'staging' }}' run — a rebuild would WIPE prod. Set a non-production SIGMA_D1_NAME variable for this Environment." exit 1 fi fi diff --git a/.github/workflows/related-persons-tr-refresh.yml b/.github/workflows/related-persons-tr-refresh.yml new file mode 100644 index 000000000..807c103c3 --- /dev/null +++ b/.github/workflows/related-persons-tr-refresh.yml @@ -0,0 +1,114 @@ +name: Related-persons — Trade Register cache refresh (monthly) + +# Refreshes the Търговски регистър deed cache the publishing decision rests on (#279, ADR-0033). +# +# WHY THIS IS ITS OWN WORKFLOW, separate from the decision run: this is the only step in the project +# that touches a public register at volume, and it is the only one with a network budget. The decisions +# themselves are a pure function of declarations, cached deeds and contracts — they must be able to run +# daily without ever calling the register. Splitting them keeps that true by construction. +# +# It SHIPS NOTHING. It writes only the cache under scratch/tr/, which is git-ignored and never leaves +# the runner except as an artifact. A run that fails leaves the previous cache intact. +# +# Pace and refusals live in scripts/tr/fetch-deeds.mjs and are not configurable here on purpose: +# 1 request / 3 s, sequential, a closed candidate set, and a 429 ends the run (exit 2) without marking +# anything. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the limiter is the +# operator's only way to state a rate preference, so we do not tune around it. + +on: + schedule: + # 02:00 UTC on the 1st. Monthly, per #279 §9 — a deed changes rarely, and every extra pass is + # ~400 requests against somebody else's service for almost no new information. + - cron: '0 2 1 * *' + workflow_dispatch: + inputs: + max_age_days: + description: Re-fetch deeds older than this many days (default 30). Use a large value to force a full refresh. + type: string + default: '30' + limit: + description: Stop after this many lookups (blank = all pending). Bound a first run while watching for a 429. + type: string + default: '' + +concurrency: + # Never two crawls at once: they would double the request rate against the register, which is the one + # thing the pacing exists to prevent. Queue rather than cancel — a cancelled crawl wastes its progress. + group: tr-cache-refresh + cancel-in-progress: false + +permissions: + contents: read + +jobs: + refresh: + runs-on: ubuntu-latest + # ~400 candidates at 1 per 3 s is ~20 minutes; the ceiling allows retries and a slower day without + # letting a hung run sit for the 6-hour default. + timeout-minutes: 60 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24 + cache: pnpm + + # The cache is the whole point of the workflow: without restoring it, every run would re-fetch + # every deed and turn a 400-request refresh into a 400-request full crawl each month. + - name: Restore the deed cache + id: cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.2 + with: + path: scratch/tr + key: tr-deed-cache-${{ github.run_id }} + restore-keys: | + tr-deed-cache- + + - run: pnpm install --frozen-lockfile + + # The closed candidate set. It is produced by the decision run (load.mjs writes it), so a cache + # refresh can never invent an ЕИК to look up — it can only refresh what the corpus already + # resolved. Absent on a first run, in which case there is nothing to refresh yet. + - name: Check for a candidate list + id: candidates + run: | + if [ -s scratch/cacbg/staging/candidate-eiks.txt ]; then + echo "count=$(wc -l < scratch/cacbg/staging/candidate-eiks.txt)" >> "$GITHUB_OUTPUT" + echo "ok=true" >> "$GITHUB_OUTPUT" + else + echo "No candidate-eiks.txt — run the decision workflow first. Nothing to refresh." + echo "ok=false" >> "$GITHUB_OUTPUT" + fi + + - name: Refresh deeds + if: steps.candidates.outputs.ok == 'true' + env: + MAX_AGE: ${{ inputs.max_age_days || '30' }} + LIMIT: ${{ inputs.limit }} + run: | + set -euo pipefail + ARGS=(--eiks-file scratch/cacbg/staging/candidate-eiks.txt --max-age-days "$MAX_AGE") + [ -n "$LIMIT" ] && ARGS+=(--limit "$LIMIT") + set +e + node scripts/tr/fetch-deeds.mjs "${ARGS[@]}" + code=$? + set -e + # Exit 2 is the rate limiter, and it is NOT a build failure: the run stopped politely, marked + # nothing, and the partial cache is valid and resumable. Failing the workflow here would train + # everyone to ignore a red X on the one signal that means "the register asked us to stop". + if [ "$code" -eq 2 ]; then + echo "::warning::The register rate-limited us; the run stopped and the cache is resumable." + exit 0 + fi + exit "$code" + + - name: Publish the cache index as an artifact (no raw deeds — they carry third-party data) + if: always() && steps.candidates.outputs.ok == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: tr-cache-index + # The SQLite index only. scratch/tr/deeds/*.json holds owner and manager names and company + # addresses (ADR-0033 decision 5) and must not leave the runner. + path: scratch/tr/tr-cache.sqlite + retention-days: 7 diff --git a/apps/web/app/routes/conflict.methodology.tsx b/apps/web/app/routes/conflict.methodology.tsx index 32936a301..c9a432b06 100644 --- a/apps/web/app/routes/conflict.methodology.tsx +++ b/apps/web/app/routes/conflict.methodology.tsx @@ -182,6 +182,64 @@ export default function ConflictMethodology() { връзката е двусмислена и не се публикува — никога не отгатваме кой от субектите е имал предвид деклараторът.

+ +

Проверка в Търговския регистър

+

+ Съвпадението по име вече не е достатъчно, за да се публикува + връзка. За всяко дружество правим отделна справка в{' '} + Търговския регистър (публична, без регистрация) и връзката се + показва само ако самоличността на дружеството е потвърдена от{' '} + проверим факт в акта. Редът е следният — прилага се първото, което + е изпълнено: +

+
    +
  1. + Акционерна форма (АД, ЕАД, КДА) — не се показва никога. Книгата + на акционерите не е публична, тоест твърдението е непроверимо; освен това дялът + може да е миноритарен или борсов, тоест несъществен. +
    2. +
  2. + Документ. И трите имена на лицето се срещат в живо поле на акта — + управители, съдружници, едноличен собственик или физическо лице-търговец. Пазим + ролята (съдружник/собственик или само управител), защото тя не е едно и също. +
    3. +
  3. + Потвърдено. Лицето не е намерено в акта, но декларирано от него + седалище съвпада с регистрираното за този ЕИК, или самият + декларатор е изписал ЕИК. Тук се потвърждава{' '} + кое е дружеството, а не кой го притежава. +
    4. +
  4. + Оборена. Само за собствен дял: лицето не фигурира никъде в акта, + а най-късното вписване за собственост е преди декларирания период + — тоест записът покрива целия период и не назовава лицето. Връзката отпада. +
    5. +
  5. + Неизвестна — всичко останало. Остава скрита. +
    6. +
+

+ Какво доказва това и какво не. Регистърът потвърждава{' '} + самоличността на дружеството; той не удостоверява, че лицето притежава дял + — това твърдение идва от неговата собствена декларация. Затова на картата пише + „лицето е вписано като съдружник/собственик", а не „лицето е собственик според ТР". +

+

+ Ограниченията, честно. Търговският регистър{' '} + не съдържа ЕГН, затова съвпадението по трите имена не е абсолютно + доказателство за самоличност — възможен е съименник. Затова се + изисква пълно съвпадение и на трите имена в{' '} + един и същ запис на акта (две от три не се приема; имена с инициали + или с латински букви не се приемат). Седалищата се променят, затова регистрираното + седалище се приема само ако е вписано преди декларирания период. Заличените записи + се пропускат навсякъде. +

+

+ Справките се опресняват месечно, а датата на справката и номерът на + вписването се показват на всяка връзка — така всяко твърдение носи своя източник и + своята давност. Актовете се пазят само служебно, за срок до 35 дни; имена на трети + лица от тях не се съхраняват и не влизат в никакъв публичен запис. +

@@ -267,7 +325,11 @@ export default function ConflictMethodology() { Прекратен дял. Ако лице подаде по-късна декларация за собственост, която вече не изброява дадено дружество, връзката се смята за приключила и се оттегля от публичната част — така остарял запис - не внушава конфликт, който е приключил. + не внушава конфликт, който е приключил. Това обаче е извод от мълчание: + най-честата причина да спреш да декларираш е изтекъл мандат, а не продажба. Затова + преди оттеглянето сверяваме с живия акт в Търговския регистър — + ако лицето и днес е вписано като съдружник или собственик, дялът не е прекратен и + връзката остава.

diff --git a/docs/runbooks/related-persons-suppression.md b/docs/runbooks/related-persons-suppression.md index ac2b140c9..2aaa2d975 100644 --- a/docs/runbooks/related-persons-suppression.md +++ b/docs/runbooks/related-persons-suppression.md @@ -4,6 +4,21 @@ A contested, corrected, or legally-challenged link on the свързани-ли the version-controlled suppression list. See [ADR-0031](../adr/0031-suppressions-version-controlled-fingerprinted.md) for why this is a git list keyed on an HMAC fingerprint, not a DB row. +## Grounds that require a takedown + +Two of these are specific to the Trade Register evidence (#279, ADR-0033) and are easy to miss, because +neither shows up as a bug: the pipeline is working correctly and the claim is still wrong. + +- **A court-annulled entry (чл. 29 ЗТРРЮЛНЦ).** A registry entry set aside by a court is void, but the + deed we cached may still carry it, and the rules that read it are unchanged — so a rules-version bump + will NOT remove the link. This is an evidence-invalidation ground, and suppression is the mechanism. +- **A match against the wrong person (a namesake).** The register carries no ЕГН, so a three-name match + inside one entity is strong evidence of identity but not proof. If the named official is not the person + in the act, the link is a false public claim about them and must come down immediately — do not wait for + a rules change or a re-crawl. +- The ordinary grounds are unchanged: a contested link under review, a correction the source has since + published, or a legal challenge. + ## What you need - The exact `link_key`. Self links are `pid|eik`; a close-relative link is `pid|eik|family`. Get it from the diff --git a/docs/spec/related-persons-foundation.md b/docs/spec/related-persons-foundation.md index 8c50fa949..61d399f52 100644 --- a/docs/spec/related-persons-foundation.md +++ b/docs/spec/related-persons-foundation.md @@ -70,6 +70,13 @@ Of #128's five checkboxes: **1** (shared owners) contingently unblocked via §3. **Per-ЕИК public lookup** (`portal.registryagency.bg`) is free, no fee. Recoverable per ЕИК: **управители, съдружници/собственици на капитала + stakes, капитал, legal form, seat.** **ЕГН masked** → owner identity name-based. Run **only** for the bounded set (bidders + resolved declared/donor companies) — thousands, not ~900k. **Hard constraints (DPA-verified):** **NEVER bulk-scrape TR** (КЗЛД ruled bulk provision unlawful; CJEU C-200/23 climate). **Never store scraped ЕГН**, even if leaked in a document image. Phase-0 confirms endpoint, fields, and rate limits. +**Phase-0 is CLOSED — measured 2026-08-04/05 (#279, ADR-0033).** Endpoint: `GET portal.registryagency.bg/CR/api/Deeds/{ЕИК}`, public JSON, no authentication. Facts worth recording because three of them contradict what the issue predicted: +- **The register rate-limits, and the block is sustained.** HTTP 429 at ~50 cumulative requests ending in a burst, then 429 to everything — no `Retry-After`, no quota header. So a 429 is an instruction to stop, never a transient to retry through: the crawler ends the run, marks nothing, and resumes later. The implemented pace is 1 request / 3 s, sequential, over a **closed** candidate set the crawler cannot extend (it never follows a link out of a deed). +- **„Not in the register" is an HTTP 200 with a ZERO-BYTE body**, not a 404 and not HTML. Verified on Община София (`000696327`): empty twice while a real company returned its full deed in the same window. Permanence therefore keys on the **status**, not the empty body — empty under 200 is an answer and may be cached; empty under 5xx is a failure and stays transient. +- **`fetch` does not work against this host.** The identical request returns 500/empty via undici and 200 with the full deed via `node:https`. +- Field values are HTML fragments inside the JSON envelope, and one field routinely holds several people separated by `

`; erasure is marked structurally (`erasure-text-inline`), not textually. Matching must happen **inside one entity**, or one person's given name combines with another's surname. +- **ЕГН is absent** from every payload examined, and is never stored either way. Raw deeds live only under git-ignored `scratch/tr/` with a 35-day retention; the cache index holds no name at all. + ## 4. Data model New domain tables, built by `normalize-raw.sql` from persistent staging; `interest_links` written by a **JS resolver pass** (§5), NOT by fuzzy SQL. @@ -182,6 +189,7 @@ The public site MUST carry a plain-language methodology page. It is part of the 1. **Sources + legal basis** — CACBG declarations (ЗСП чл. 75, Сметна палата); ЕРИК (Изборен кодекс); targeted TR per-ЕИК (public company facts). Each with a direct source link. 2. **What is shown and what is not** — officials' own declared holdings; **family holdings are NOT published** (v1); addresses/ЕГН never shown. 3. **The matching rule, verbatim** — declared company **full name** (incl. legal form) is normalized and matched **exactly** against the winner's registered name, which is paired 1:1 with its ЕИК in the procurement source; because Bulgarian trade names are **nationally unique** (ЗТРРЮЛНЦ/ТЗ), an exact match is the same legal entity. Ambiguous cases (truncated names, no exact match) are **excluded or human-confirmed, never guessed**. Every link shows its full evidence chain. + **Superseded in part by #279 / ADR-0033:** a name match is no longer *sufficient* to publish. Every link additionally rests on a checkable **Търговски регистър** fact, and the methodology page carries the six-rung evidence ladder verbatim, together with the disclosures that make it honest: the register carries **no ЕГН**, so a three-name match is not proof of identity and a namesake is possible; the match must therefore be a **full three-token subset inside a single registry entity**; seats move, so a registered seat only confirms a period it predates; erased entries are skipped; and every published link shows its **entry number and lookup date**. 4. **Certainty & framing** — only deterministic exact matches (and human-confirmed cases) appear; each is a „**модел за проверка, не обвинение**"; the tool shows *declared* links only and **does not claim to find hidden ownership** (it is a floor, not the full picture). 5. **Temporal meaning** — a declaration is a point-in-time snapshot; how overlap with a contract date is (and isn't) interpreted. 6. **Known gaps** — the recall holes (§5), self-reporting limits, sources it cannot see. diff --git a/docs/spec/related-persons-lia.md b/docs/spec/related-persons-lia.md index 1309c42a3..075239405 100644 --- a/docs/spec/related-persons-lia.md +++ b/docs/spec/related-persons-lia.md @@ -128,6 +128,40 @@ v Poland* (2428/05) приема онлайн публикуване на дек регистър. Приемаме този оспорим елемент съзнателно. **Резервен план при оспорване:** един филтър (сваляне на семейните връзки към еднолични дружества), не спиране на повърхността. +## 9а. Допълнение: справки в Търговския регистър (#279, ADR-0033) + +Нов **източник** и нова **обработка**, затова оценката се допълва, а не само се препотвърждава. + +**Каква е обработката.** За всяко дружество-кандидат правим една справка в публичния Търговски регистър и +четем живия акт. Актът съдържа **лични данни на трети лица** — имена на съдружници, собственици и +управители, които не заемат публична длъжност, както и адрес на дружеството. + +**Цел и основание.** Целта е тясна и проверима: да се установи **самоличността на дружеството** зад +декларирано име, преди срещу конкретно лице да бъде публикувано твърдение. Това е мярка, която *намалява* +риска за субектите на данни — досега самоличността се решаваше по евристика за отличителност на името, а +сега — по документ. Основанието е същият надделяващ обществен интерес по ЗДОИ чл. 41и, ал. 4, приложен към +по-точна, а не към по-широка публикация. + +**Минимизиране — какво НЕ правим.** +- Имената от акта се ползват **само в булеви сравнения** и не влизат в никаква публична таблица, отговор или + лог. Индексът на кеша не съдържа нито едно име — само ЕИК, дати, кодове и хеш на тялото. +- Адресът **не се извлича**: от полето за седалище се чете единствено сегментът „Населено място". +- Полетата за **действителен собственик** (`CR_F_550_L`) и за контрол (`537/538`) **не се ползват** — това + би било претенция за действителна собственост, оставена настрана след CJEU C-37/20 (ADR-0007, т. 1). +- Суровите актове се пазят само служебно, в git-игнорирана директория, със **срок 35 дни**. +- Обхватът е **затворен**: справка се прави само за ЕИК, които вече са резултат от съпоставянето; обхождащият + никога не следва връзка навън от акт. Това не е масово изтегляне (спец. §3.3). + +**Остатъчен риск — съименник.** Регистърът **не съдържа ЕГН**, затова съвпадението по трите имена е силно, но +не абсолютно доказателство за самоличност. Смекчаване: изисква се пълно съвпадение и на трите имена в **един и +същ запис** на акта; имена с инициали или с латински букви не се приемат; фамилни съвпадения не се ползват +никъде. Грешката тук не измисля твърдение за собственост — тя **закача длъжностното лице за друго дружество**, +което е също толкова невярно и се сваля по реда на ADR-0031 (виж runbook-а: „съвпадение с грешно лице"). + +**Отложено съзнателно.** Етикетите „и към днешна дата" по т. 7 на #279 **не се пускат** в тази фаза: те +твърдят сегашно време за поименно лице, а давността им е ограничена от цикъла на опресняване. Влизат само +след отделен преглед на настоящата оценка. + ## 10. Статус и преглед Draft за правен преглед; изпраща се паралелно с пускането, не като блокер (ADR-0032, т. 7). Ревизира се при промяна From d50a723b00d03a7abef00670c23e326e8689d008 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 09:39:12 +0300 Subject: [PATCH 12/50] fix(test): two fixtures that only passed because of local state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI caught both; local runs could not, and the reasons are worth recording because they will recur. load-ambiguous.test.mjs ran load.mjs WITHOUT setting TR_CACHE_DB, so it fell back to the repo's real scratch/tr/tr-cache.sqlite — a file that exists on my machine only because the live smoke run created it, and that exists nowhere in CI. The test was silently running against a developer artifact. It now builds its own empty cache and passes the path explicitly, like every other loader test. refresh-officials.test.ts executes the real refresh-slice.sql, which now joins interest_link_evidence, but built its schema from 0000/0001/0003 only. Locally `pnpm test` reported 6/6 because turbo REPLAYED A CACHED RESULT for @sigma/ingest — my change was to scripts/refresh-slice.sql, which is not among that package's declared inputs, so the cache key did not move. `turbo run test --force` fails the same way CI does. Fixture now applies 0006 and seals its links. The seal is derived with a SELECT over interest_links rather than written row by row, so a link added to the fixture later is sealed automatically. That matters more than it looks: the officials batch requires a publishing rung, so an unsealed fixture indexes NOBODY and every assertion below it passes vacuously. Coverage baseline ratcheted up, as check-coverage nudged: apps/web +1.33pp (the new jsdom card tests and the evidence-label tests), packages/db +0.34pp, packages/ingest +0.57pp, packages/shared +0.83pp branches. --- coverage-baseline.json | 16 ++++++------- packages/ingest/src/refresh-officials.test.ts | 8 +++++++ scripts/cacbg/load-ambiguous.test.mjs | 23 ++++++++++++++++++- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/coverage-baseline.json b/coverage-baseline.json index 5abe85c95..064f448fe 100644 --- a/coverage-baseline.json +++ b/coverage-baseline.json @@ -6,24 +6,24 @@ "branches": 58.2 }, "apps/web": { - "lines": 89.7, - "branches": 81.8 + "lines": 91, + "branches": 82.4 }, "packages/config": { "lines": 92.8, "branches": 72.2 }, "packages/db": { - "lines": 94.2, - "branches": 79 + "lines": 94.5, + "branches": 79.3 }, "packages/ingest": { - "lines": 85.8, - "branches": 80 + "lines": 86.3, + "branches": 80.4 }, "packages/shared": { - "lines": 95.4, - "branches": 80 + "lines": 95.5, + "branches": 80.8 } } } diff --git a/packages/ingest/src/refresh-officials.test.ts b/packages/ingest/src/refresh-officials.test.ts index 55170ef21..6f3eb4c60 100644 --- a/packages/ingest/src/refresh-officials.test.ts +++ b/packages/ingest/src/refresh-officials.test.ts @@ -20,6 +20,9 @@ const migrations = [ '0000_init.sql', '0001_flow_pairs_bidder_index.sql', '0003_related_persons_foundation.sql', + // The officials batch now joins the Trade Register evidence seal (#279, ADR-0033) — without 0006 the + // real refresh-slice.sql this test executes cannot parse. + '0006_interest_link_evidence.sql', ].map((f) => resolve(root, 'packages/db/migrations', f)); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); @@ -67,6 +70,11 @@ INSERT INTO interest_links -- her — dropping the s.status='published' guard would collapse it behind an invisible self stake (false neg). ('il:ys','person:ЯНА|444','person:ЯНА','eik:444','444','ДЕЛТА ООД','exact_name_key','v1','C_hold','owns','private_ownership',0,'none',1,'2020','2023',1,20000,'2021','2021','held'), ('il:yf','person:ЯНА|444|family','person:ЯНА','eik:444','444','ДЕЛТА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',1,'none',1,'2020','2023',1,20000,'2021','2021','published'); +-- Every link carries an evidence seal (#279, ADR-0033): the officials batch requires a publishing rung, +-- so a fixture without seals indexes NOBODY and every assertion below passes vacuously. Derived from +-- interest_links, so a row added later is sealed automatically. +INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links; `; describe('ETL refresh-slice officials batch (the live parse-then-execute path)', () => { diff --git a/scripts/cacbg/load-ambiguous.test.mjs b/scripts/cacbg/load-ambiguous.test.mjs index 336cfb307..01cf8f7ad 100644 --- a/scripts/cacbg/load-ambiguous.test.mjs +++ b/scripts/cacbg/load-ambiguous.test.mjs @@ -45,12 +45,33 @@ function buildAndLoad(bidderRows) { fs.writeFileSync(path.join(STAGING, 'holdings.jsonl'), ''); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + // An EMPTY Trade Register cache, pointed at explicitly (#279, ADR-0033). This fixture declares no + // holdings, so it resolves no links and the coverage gate is satisfied by an empty candidate set — + // but the cache FILE must still exist, because a missing one refuses the whole load. + // + // Explicit rather than defaulted, and that matters: load.mjs falls back to the repo's real + // scratch/tr/tr-cache.sqlite, so a test that omits TR_CACHE_DB silently runs against whatever the + // developer's last live crawl left behind. That is exactly how this test passed locally and failed + // in CI, where no such file exists. + const trDb = path.join(dir, 'tr-cache.sqlite'); + new DatabaseSync(trDb).close(); + let threw = false; try { execFileSync( 'node', ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], - { cwd: ROOT, env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, stdio: 'pipe' }, + { + cwd: ROOT, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: trDb, + TR_RAW_DIR: path.join(dir, 'tr-deeds'), + }, + stdio: 'pipe', + }, ); } catch { threw = true; // execFileSync throws on a non-zero exit code (a fired hard-gate) From 7a424bda21620fa3da71a1050636e932e5722aea Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:29:59 +0300 Subject: [PATCH 13/50] fix(tr): accept double-quoted attributes in the deed parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The entity split and the erasure strip both matched class='...' only. The live register emits single quotes today and every fixture is verbatim from a real deed, so nothing is broken now — but the failure mode if that ever changes is the worst one this module has: the split stops firing, all owners in a field collapse into one token pool, and a name assembled from two different people starts matching. That is a named public claim about someone who is not there. deed.mjs's own doctrine is "refuse loudly, never guess". Silently mis-splitting is neither, so match either quote style instead. The new split test strips the
separators first. With them present it passes on the
rule alone and proves nothing about the quote handling — verified by watching it pass before the fix. --- scripts/tr/deed.mjs | 4 ++-- scripts/tr/deed.test.mjs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/scripts/tr/deed.mjs b/scripts/tr/deed.mjs index 7709e31c0..fdb3c1bd8 100644 --- a/scripts/tr/deed.mjs +++ b/scripts/tr/deed.mjs @@ -64,7 +64,7 @@ export function entityBlocks(html, { strict = false } = {}) { const decoded = decodeEntities(html); // decode BEFORE splitting — see the order above const chunks = decoded .split(/]*>/i) - .flatMap((part) => part.split(/(?=]*class='[^']*record-container)/i)) + .flatMap((part) => part.split(/(?=]*class=['"][^'"]*record-container)/i)) .map((c) => c.trim()) .filter((c) => c !== ''); @@ -74,7 +74,7 @@ export function entityBlocks(html, { strict = false } = {}) { // Read the visible text WITHOUT the erasure notice, so „Заличено обстоятелство." never counts as // content and an erased block reads as empty. const withoutNotice = chunk.replace( - /]*class='[^']*erasure-text-inline[^']*'[^>]*>.*?<\/div>/gis, + /]*class=['"][^'"]*erasure-text-inline[^'"]*['"][^>]*>.*?<\/div>/gis, ' ', ); const text = stripTags(withoutNotice); diff --git a/scripts/tr/deed.test.mjs b/scripts/tr/deed.test.mjs index 6ec634eff..9a7f4c97c 100644 --- a/scripts/tr/deed.test.mjs +++ b/scripts/tr/deed.test.mjs @@ -75,6 +75,36 @@ test('T1 — tokens from two DIFFERENT people never combine into a match', () => assert.equal(fullSubsetMatch(frankenstein, F19_THREE), true, 'naive whole-field match fires'); }); +// The live register emits single-quoted attributes today, and every fixture above is verbatim from a +// real deed. But the entity split is the ONLY thing standing between three people and one merged +// token pool, and it must not be quiet about a markup change: if the register ever switches to +// class="…", a quote-specific pattern stops splitting, all three owners collapse into one block, and +// the frankenstein match above starts firing — a named public claim about a person who is not there. +// R7's doctrine for this module is „refuse loudly, never guess"; silently mis-splitting is neither. +const dq = (s) => s.replace(/class='([^']*)'/g, 'class="$1"'); + +test('T1 — the entity split survives double-quoted attributes (markup-drift hardening)', () => { + // Strip the
separators first. entityBlocks splits on BOTH
and record-container precisely + // because either may be absent; with the
present this test would pass on the
rule alone + // and prove nothing about the quote handling it is here to pin. + const blocks = entityBlocks(dq(F19_THREE).replace(/]*>/gi, '')); + assert.equal(blocks.length, 3, 'a quote style change must not merge three owners into one block'); + assert.equal( + blocks.some((b) => fullSubsetMatch('ПЕНКО КОСТАДИНОВ ФИЛИПОВ', b.text)), + false, + 'cross-entity match under double quotes — the libel bug via markup drift', + ); + assert.ok(blocks.some((b) => fullSubsetMatch('ПЕНКО НЕСТОРОВ НЕСТОРОВ', b.text))); +}); + +test('T1 — erasure is still detected and stripped under double-quoted attributes', () => { + const [block] = entityBlocks(dq(F23_ERASED)); + assert.equal(block.erased, true); + assert.equal(block.text, '', 'the erasure notice must not survive as content'); + // The strict contradiction check must not fire merely because the quote style changed. + assert.doesNotThrow(() => entityBlocks(dq(F23_ERASED), { strict: true })); +}); + test('T1 positive control — the CORRECT declarant does match', () => { // Without this, a matcher that always returns false passes every negative test above (ADR-0027). const blocks = entityBlocks(F19_THREE); From 299400e61dfe0136b58498b18525546faccca600 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:30:29 +0300 Subject: [PATCH 14/50] =?UTF-8?q?fix(tr):=20anchor=20the=20=D0=95=D0=93?= =?UTF-8?q?=D0=9D=20guard=20so=20a=2013-digit=20=D0=95=D0=98=D0=9A=20canno?= =?UTF-8?q?t=20abort=20the=20crawl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assertNoEgnShape refused any value matching /\d{10}/, justified in three places by "an ЕИК is 9 or 13 digits, never 10 — so it cannot reject a legitimate identifier". That reasoning only holds if the run is matched as a WHOLE. A 13-digit ЕИК (клон) CONTAINS ten-digit substrings, and on the fetched path rawPath is `.json`, derived from that very ЕИК: /\d{10}/.test("1155361790001.json") === true upsertDeed sits past fetch-deeds' JSON.parse/assertUicEcho try-catch, so the throw propagated and killed the whole run on the first branch office that returned a deed — after its paced request was spent. Fail-closed for correctness, but the crawl is the data source for the entire feature. The existing test asserted this exact case was safe and passed anyway: markOutsideTr sets no rawPath, and the fetched fixture used a 9-digit ЕИК, so the one path that throws was never exercised. Anchor the pattern, and screen by EXCLUSION rather than a hand-maintained list of four field names — upsertDeed binds thirteen values and the next one added would have bypassed the rail silently. Two fields stay exempt for structural reasons, not convenience. `eik` is already shape-validated by safeEik. `bodySha256` is 64 chars of [0-9a-f]: measured over 200k digests, a standalone ten-digit run occurs in 7.2% of them (18.1% unanchored), so screening it would refuse roughly one deed in fourteen for no privacy gain. A digest is not personal data — it exists precisely so that no deed content reaches the index. --- scripts/tr/cache.mjs | 35 ++++++++++++++++++++++++----------- scripts/tr/cache.test.mjs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/scripts/tr/cache.mjs b/scripts/tr/cache.mjs index 673c6b477..846c4eb98 100644 --- a/scripts/tr/cache.mjs +++ b/scripts/tr/cache.mjs @@ -7,9 +7,11 @@ // exist only in the raw JSON under git-ignored scratch/, are read only to produce a boolean, and // never enter a public table, a response or a log. A hash rather than an excerpt, because an // excerpt of a deed IS third-party personal data. -// 2. Nothing may carry a ten-digit run. That is the ЕГН shape, and the check is sound precisely -// because an ЕИК is 9 or 13 digits — never 10 — so it cannot reject a legitimate identifier. -// ЕГН was absent from every payload examined; this is the rail for the day one leaks. +// 2. Nothing may carry a STANDALONE ten-digit run. That is the ЕГН shape, and the check is sound +// precisely because an ЕИК is 9 or 13 digits — never 10 — so it cannot reject a legitimate +// identifier. „Standalone" is load-bearing: a 13-digit ЕИК contains ten-digit substrings, so an +// unanchored match would refuse every клон. ЕГН was absent from every payload examined; this is +// the rail for the day one leaks. // // Resumability is the other job: the register throttles hard and a 429 ends the run (client.mjs), so a // crawl must be able to pick up exactly where it stopped without re-requesting what it already has. @@ -49,21 +51,33 @@ export function openCache(file) { } // ── the ЕГН rail ────────────────────────────────────────────────────────────── -const TEN_DIGITS = /\d{10}/; +// ANCHORED, and that is the whole correctness of the rail. An ЕИК is 9 or 13 digits — never 10 — so +// a ten-digit run cannot be a legitimate identifier here. But that reasoning only holds when the run +// is matched as a WHOLE: an unanchored /\d{10}/ matches INSIDE the 13-digit ЕИК of a клон, and +// rawPath on the fetched path is `.json`, so every branch office would be refused and the crawl +// would abort on the first one. +const EGN_SHAPE = /(? assert.doesNotThrow(() => markOutsideTr(db, '1155361790001', 'клон')); })); +// A 13-digit ЕИК (клон/подразделение) CONTAINS ten-digit substrings, so an unanchored /\d{10}/ +// rejects it — and rawPath on the fetched path is `.json`, derived from that very ЕИК. This is +// the exact shape fetch-deeds.mjs writes (path.relative(rawDir, deedPath(eik))), and upsertDeed sits +// past its JSON.parse/assertUicEcho try-catch, so a throw here aborts the whole crawl on the first +// branch office that returns a deed. The guard's own stated soundness ("an ЕИК is 9 or 13 digits, +// never 10") only holds if the run is matched as a WHOLE, which is why the pattern is anchored. +test('a 13-digit ЕИК does not trip the ЕГН guard through its own derived rawPath', () => + withCache((db) => { + assert.doesNotThrow(() => + upsertDeed(db, deed({ eik: '1155361790001', rawPath: '1155361790001.json' })), + ); + assert.equal(readDeed(db, '1155361790001').eik, '1155361790001'); + })); + +// The rail must not be a hand-maintained allowlist of four field names: upsertDeed binds thirteen +// values, and the next one added would bypass the check silently. Every bound value is screened. +test('the ЕГН guard screens EVERY bound value, not a hand-picked subset', () => + withCache((db) => { + for (const field of ['seatEntryDate', 'latestOwnEntryDate', 'fetchedAt']) + assert.throws(() => upsertDeed(db, deed({ [field]: '8001014567' })), /ten-digit|ЕГН/i, field); + })); + +// ...with one exemption, and it is measured rather than assumed: a sha256 hex digest is 64 chars of +// [0-9a-f], so a standalone ten-digit run occurs in ~7% of hashes (18% unanchored). Screening it +// would refuse roughly one deed in fourteen for no privacy gain — a hash is not an ЕГН, and it is a +// hash precisely so that no deed content reaches the index. +test('the body hash is exempt from the ЕГН guard — a digit run there is arithmetic, not an ЕГН', () => + withCache((db) => { + assert.doesNotThrow(() => upsertDeed(db, deed({ bodySha256: `8001014567${'a'.repeat(54)}` }))); + })); + test('the schema exposes no column that could hold a person name', () => withCache((db) => { const cols = db From 7f2e174e1a731cc83ce7c1f274f833c16b584db0 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:30:44 +0300 Subject: [PATCH 15/50] feat(tr): enforce the 35-day deed retention with a purge step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0033 decision 5 promises the deed cache is kept "with a 35-day retention (one refresh cycle plus slack) and a purge step in the same job". Neither existed. --max-age-days defaulted to null and only made a cached row pending again — it re-REQUESTS, it never deletes — so nothing ever unlinked the raw JSON that holds co-owner and manager names and company addresses. Third-party personal data accumulated on disk unbounded, with a documented TTL that deleted nothing. Freshness and retention are now separate flags because they are separate obligations: --max-age-days decides what gets re-requested, --retention-days decides what stops being kept. Retention defaults to the ADR's 35 days rather than to "off", so the rail holds for an operator who passes neither. purgeExpired runs in `finally`, and that placement is the point: retention is an obligation about other people's data, not a reward for a clean run, so it must also happen on the paths that leave early — a 429 (exit 2), a tripped breaker, an unresolved candidate. A failed purge is loud but never masks the run's own exit code, least of all the 429 that tells the operator to back off. It is a privacy rail, not cache eviction. Under normal operation it removes nothing, because the monthly refresh rewrites each row well inside the window; what it actually catches is residue — a company that dropped out of the candidate set, or a refresh that failed — plus orphaned raw files that no read path can reach and that are therefore pure retained personal data. purgeExpired was written before its tests, so RED was established afterwards by mutation: stubbing the expiry query kills 4 tests, disabling the orphan sweep kills 2, and moving the call out of `finally` kills the 429 case. --- .../workflows/related-persons-tr-refresh.yml | 5 ++ scripts/tr/cache.mjs | 72 +++++++++++++++++- scripts/tr/cache.test.mjs | 73 ++++++++++++++++++- scripts/tr/fetch-deeds.mjs | 35 ++++++++- scripts/tr/fetch-deeds.test.mjs | 54 ++++++++++++++ 5 files changed, 234 insertions(+), 5 deletions(-) diff --git a/.github/workflows/related-persons-tr-refresh.yml b/.github/workflows/related-persons-tr-refresh.yml index 807c103c3..9cc09add0 100644 --- a/.github/workflows/related-persons-tr-refresh.yml +++ b/.github/workflows/related-persons-tr-refresh.yml @@ -10,6 +10,11 @@ name: Related-persons — Trade Register cache refresh (monthly) # It SHIPS NOTHING. It writes only the cache under scratch/tr/, which is git-ignored and never leaves # the runner except as an artifact. A run that fails leaves the previous cache intact. # +# It also PURGES: raw deeds past the 35-day retention (ADR-0033 decision 5) are deleted at the end of +# every run, including one a 429 cut short. Retention is not the same knob as freshness — --max-age-days +# decides what gets re-requested, --retention-days decides what stops being kept — and the default is +# the ADR's 35 days, so the rail holds even when the workflow passes neither. +# # Pace and refusals live in scripts/tr/fetch-deeds.mjs and are not configurable here on purpose: # 1 request / 3 s, sequential, a closed candidate set, and a 429 ends the run (exit 2) without marking # anything. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the limiter is the diff --git a/scripts/tr/cache.mjs b/scripts/tr/cache.mjs index 846c4eb98..206687181 100644 --- a/scripts/tr/cache.mjs +++ b/scripts/tr/cache.mjs @@ -153,10 +153,16 @@ export function readDeed(db, eik) { }; } +/** ADR-0033 decision 5: one monthly refresh cycle plus slack. See purgeExpired. */ +export const RETENTION_DAYS = 35; + /** * Which of `wanted` still need a request — the resumability primitive. - * A cached row past `maxAgeDays` becomes pending again (the 35-day retention of ADR-0033 decision 5); - * pass no `maxAgeDays` to treat any cached row as current. + * + * `maxAgeDays` is a FRESHNESS knob, not the retention rail: past it a cached row becomes pending and + * is re-requested. Retention — actually deleting the personal data — is purgeExpired below. Refreshing + * a deed rewrites it; only purging removes it, and conflating the two is how a documented TTL ends up + * never deleting anything. Pass no `maxAgeDays` to treat any cached row as current. */ export function pendingEiks(db, wanted, { maxAgeDays = null, now = new Date() } = {}) { const cutoff = maxAgeDays == null ? null : now.getTime() - maxAgeDays * 86_400_000; @@ -198,3 +204,65 @@ export function coverage(db, wanted) { ratio: wanted.length === 0 ? 1 : covered / wanted.length, }; } + +/** + * Delete deed data past its retention — ADR-0033 decision 5's purge step. + * + * This is a PRIVACY rail, not a cache-eviction policy, and the distinction decides the design. The raw + * JSON under scratch/tr/deeds/ is the only place third-party names live: co-owners and managers who + * hold no public office, and the company's street address. We keep it because re-deriving a boolean is + * cheaper than re-requesting somebody else's register — not because we are entitled to hold it. + * + * Retention is 35 days: one monthly refresh cycle plus slack. Under normal operation nothing is ever + * purged, because the refresh at ~30 days rewrites the row first. What this actually catches is the + * residue — a company that dropped out of the candidate set, or a refresh that failed — which is + * exactly the data with no remaining reason to exist. + * + * Orphaned files are removed too: a raw deed with no index row is unreachable by every read path here, + * so it is pure retained personal data. + * + * @returns {{rows:number, files:number, orphans:number}} what was removed + */ +export function purgeExpired( + db, + rawDir, + { retentionDays = RETENTION_DAYS, now = new Date() } = {}, +) { + const cutoff = new Date(now.getTime() - retentionDays * 86_400_000).toISOString(); + const expired = db.prepare('SELECT eik, raw_path FROM deeds WHERE fetched_at < ?').all(cutoff); + + let files = 0; + for (const row of expired) { + if (!row.raw_path) continue; + // Resolve through the same safeEik path-traversal rail the writer used, never the stored string. + try { + fs.unlinkSync(path.join(rawDir, `${safeEik(row.eik)}.json`)); + files++; + } catch (e) { + // Already gone is the goal state, not a failure. Anything else must surface — a purge that + // silently fails to delete is worse than no purge, because it reports success. + if (e.code !== 'ENOENT') throw e; + } + } + db.prepare('DELETE FROM deeds WHERE fetched_at < ?').run(cutoff); + + let orphans = 0; + let names = []; + try { + names = fs.readdirSync(rawDir); + } catch (e) { + if (e.code !== 'ENOENT') throw e; // no raw dir yet — nothing to orphan + } + const known = new Set( + db + .prepare('SELECT eik FROM deeds') + .all() + .map((r) => `${r.eik}.json`), + ); + for (const name of names) { + if (!name.endsWith('.json') || known.has(name)) continue; + fs.unlinkSync(path.join(rawDir, name)); + orphans++; + } + return { rows: expired.length, files, orphans }; +} diff --git a/scripts/tr/cache.test.mjs b/scripts/tr/cache.test.mjs index 4b675e872..8046c1ada 100644 --- a/scripts/tr/cache.test.mjs +++ b/scripts/tr/cache.test.mjs @@ -9,7 +9,16 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { openCache, upsertDeed, markOutsideTr, pendingEiks, readDeed, coverage } from './cache.mjs'; +import { + openCache, + upsertDeed, + markOutsideTr, + pendingEiks, + readDeed, + coverage, + purgeExpired, + RETENTION_DAYS, +} from './cache.mjs'; function tmpDb() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-cache-')); @@ -158,3 +167,65 @@ test('markOutsideTr is permanent-by-intent and records WHY', () => assert.equal(row.status, 'outside_tr'); assert.match(row.outsideReason, /ДЗЗД/); })); + +// ── retention (ADR-0033 decision 5) ─────────────────────────────────────────── +// The ADR promises „a 35-day retention and a purge step in the same job". Freshness (pendingEiks' +// maxAgeDays) only makes a row pending again — it re-REQUESTS, it never deletes. Only this purge +// removes the third-party names in the raw deed, so these tests are the difference between a stated +// TTL and an enforced one. +function withRaw(fn) { + const { dir, file } = tmpDb(); + const rawDir = path.join(dir, 'deeds'); + fs.mkdirSync(rawDir, { recursive: true }); + const db = openCache(file); + try { + return fn(db, rawDir); + } finally { + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } +} +const writeRaw = (rawDir, eik) => + fs.writeFileSync(path.join(rawDir, `${eik}.json`), '{"names":"third-party PII"}'); + +test('purgeExpired deletes the raw deed AND its row past the retention window', () => + withRaw((db, rawDir) => { + const now = new Date('2026-08-05T00:00:00Z'); + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-05-01T00:00:00Z' })); // ~96 days old + upsertDeed(db, deed({ eik: '201122335', fetchedAt: '2026-08-01T00:00:00Z' })); // 4 days old + writeRaw(rawDir, '115536179'); + writeRaw(rawDir, '201122335'); + + const res = purgeExpired(db, rawDir, { retentionDays: 35, now }); + assert.equal(res.rows, 1); + assert.equal(res.files, 1); + assert.equal(fs.existsSync(path.join(rawDir, '115536179.json')), false, 'PII must be gone'); + assert.equal(readDeed(db, '115536179'), null); + // The in-window deed is untouched — a purge that also evicted live cache would force a re-crawl, + // which is the one thing the pacing exists to avoid. + assert.equal(fs.existsSync(path.join(rawDir, '201122335.json')), true); + assert.ok(readDeed(db, '201122335')); + })); + +test('purgeExpired removes orphaned raw deeds — unreachable data is pure retained PII', () => + withRaw((db, rawDir) => { + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-08-01T00:00:00Z' })); + writeRaw(rawDir, '115536179'); + writeRaw(rawDir, '204556676'); // no index row: no read path can ever reach it + const res = purgeExpired(db, rawDir, { now: new Date('2026-08-05T00:00:00Z') }); + assert.equal(res.orphans, 1); + assert.equal(fs.existsSync(path.join(rawDir, '204556676.json')), false); + assert.equal(fs.existsSync(path.join(rawDir, '115536179.json')), true); + })); + +test('purgeExpired defaults to the 35-day window and tolerates an already-missing file', () => + withRaw((db, rawDir) => { + assert.equal(RETENTION_DAYS, 35); + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-06-25T00:00:00Z' })); // 41 days + upsertDeed(db, deed({ eik: '201122335', fetchedAt: '2026-07-15T00:00:00Z' })); // 21 days + // No raw file on disk for the expired row: already gone is the goal state, not an error. + const res = purgeExpired(db, rawDir, { now: new Date('2026-08-05T00:00:00Z') }); + assert.equal(res.rows, 1); + assert.equal(res.files, 0); + assert.ok(readDeed(db, '201122335'), 'the 21-day-old deed is inside the window'); + })); diff --git a/scripts/tr/fetch-deeds.mjs b/scripts/tr/fetch-deeds.mjs index 0e14915d6..78c63192c 100644 --- a/scripts/tr/fetch-deeds.mjs +++ b/scripts/tr/fetch-deeds.mjs @@ -24,7 +24,14 @@ import crypto from 'node:crypto'; import { assertTrScratchIgnored, TR_DB, TR_RAW, safeEik, deedPath } from './paths.mjs'; import { eikChecksumValid } from './eik.mjs'; import { deedUrl, politeTrGet, RateLimitError, httpsGet } from './client.mjs'; -import { openCache, upsertDeed, markOutsideTr, pendingEiks } from './cache.mjs'; +import { + openCache, + upsertDeed, + markOutsideTr, + pendingEiks, + purgeExpired, + RETENTION_DAYS, +} from './cache.mjs'; import { assertUicEcho, registryLegalForm, @@ -74,11 +81,17 @@ export function parseTrOptions(argv) { ); } const maxAgeRaw = get('max-age-days', ''); + // --max-age-days is FRESHNESS (re-request past it); --retention-days is RETENTION (delete past it). + // They are separate flags because they are separate obligations: refreshing rewrites the personal + // data, only purging removes it. Retention defaults to the ADR's 35 days rather than to „off", so + // the rail holds for an operator who passes neither. + const retentionRaw = get('retention-days', ''); return { eiksFile, limit: limitRaw ? posInt(limitRaw, 'limit') : Infinity, minIntervalMs, maxAgeDays: maxAgeRaw ? posInt(maxAgeRaw, 'max-age-days') : null, + retentionDays: retentionRaw ? posInt(retentionRaw, 'retention-days') : RETENTION_DAYS, }; } @@ -117,7 +130,7 @@ export async function run({ argv = process.argv, } = {}) { guard(); - const { eiksFile, limit, minIntervalMs, maxAgeDays } = parseTrOptions(argv); + const { eiksFile, limit, minIntervalMs, maxAgeDays, retentionDays } = parseTrOptions(argv); const requested = readEiksFile(eiksFile); // A shape- or checksum-invalid code is dropped BEFORE any request: it cannot name a real company, @@ -241,6 +254,24 @@ export async function run({ } return 0; } finally { + // The purge step ADR-0033 decision 5 puts „in the same job" — in `finally`, and that placement is + // the point. Retention is an obligation about other people's data, not a reward for a clean run, + // so it must also happen on the paths that leave early: a 429 (exit 2), a tripped breaker, an + // unresolved candidate. Under normal operation it removes nothing, because the monthly refresh + // rewrites each row well inside the window; anything it does delete is residue — a company that + // left the candidate set, or a refresh that never landed. + try { + const purged = purgeExpired(db, rawDir, { retentionDays, now: now() }); + if (purged.rows || purged.files || purged.orphans) { + console.log( + `purged ${purged.rows} row(s), ${purged.files} raw deed(s), ${purged.orphans} orphan(s) past ${retentionDays}d retention`, + ); + } + } catch (e) { + // A failed purge must be loud but must not mask the run's own outcome — especially not a 429, + // whose exit code is what tells the operator to back off. + console.error(`purge failed: ${e.message}`); + } db.close(); } } diff --git a/scripts/tr/fetch-deeds.test.mjs b/scripts/tr/fetch-deeds.test.mjs index 8b8296d7b..3a4b1dc01 100644 --- a/scripts/tr/fetch-deeds.test.mjs +++ b/scripts/tr/fetch-deeds.test.mjs @@ -371,3 +371,57 @@ test('the circuit breaker aborts a sustained wall of failures', async () => { c.cleanup(); } }); + +// ── retention (ADR-0033 decision 5: „a purge step in the same job") ─────────── +test('the job purges past-retention deeds, and does so even when a 429 ends the run', async () => { + // Retention is an obligation about other people's data, not a reward for a clean run: the paths that + // leave early — a 429, a tripped breaker — are exactly the ones where a naive placement after the + // fetch loop would skip it and let third-party names sit on disk indefinitely. + for (const [label, route] of [ + ['clean run', ok(A)], + ['429 stops the run', status(429)], + ]) { + const c = ctx(); + try { + // An old deed with its raw file — well past the 35-day window at the harness's fixed clock. + fs.mkdirSync(c.rawDir, { recursive: true }); + fs.writeFileSync(path.join(c.rawDir, `${C}.json`), '{"owner":"ТРЕТО ЛИЦЕ"}'); + const db = new DatabaseSync(c.dbFile); + db.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT); + INSERT INTO deeds (eik,status,fetched_at,raw_path) VALUES ('${C}','fetched','2026-01-01T00:00:00Z','${C}.json');`); + db.close(); + + await harness(c, [A], { [A]: route }).promise; + + assert.equal( + fs.existsSync(path.join(c.rawDir, `${C}.json`)), + false, + `${label}: the past-retention raw deed must be deleted`, + ); + assert.equal( + rows(c.dbFile).some((r) => r.eik === C), + false, + `${label}: its index row must go with it`, + ); + } finally { + c.cleanup(); + } + } +}); + +test('the purge leaves in-window deeds alone — it is a privacy rail, not a cache eviction', async () => { + // If it evicted live cache, every run would re-request deeds it already holds, which is precisely + // the volume against the register the pacing exists to avoid. + const c = ctx(); + try { + await harness(c, [A], { [A]: ok(A) }).promise; + assert.equal(rows(c.dbFile).length, 1, 'the deed just fetched must survive its own job'); + assert.equal(fs.existsSync(path.join(c.rawDir, `${A}.json`)), true); + } finally { + c.cleanup(); + } +}); From 2327e6265cf5ea909ba7c8b028210c5aa3c926e3 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:31:03 +0300 Subject: [PATCH 16/50] feat(related-persons): implement the monotonicity gate ADR-0033 specifies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rules_version was only ever written (load.mjs) and never read for a comparison. Every "§8's monotonicity gate keys on this" comment — in load.mjs, evidence.mjs and migration 0006 — pointed at code that did not exist: no pre-wipe export, no snapshot compare, no finding. ship-related-persons.mjs's count floor cannot stand in for it. It compares a COUNT, so a one-for-one swap — one true published link silently dropped, one gained — leaves it perfectly quiet. Two halves: load.mjs exports the currently published set with each link's rules_version immediately before it drops the CACBG tables. The rebuild is total, so that set exists only in this instant. On a first run the tables are absent and the export is an empty array — written rather than skipped, so a missing file keeps meaning "the loader never ran", not "nothing was published". Any error other than "no such table" propagates; swallowing it would turn a broken export into a permanently silent gate. audit.mjs compares against it. A link published last run and not published now under an UNCHANGED rules_version is a hard finding: nothing licensed the removal. Under a changed version it is an intentional event and degrades to a printed diff, so a rules bump still shows a human which named claims it withdrew. Held and withdrawn links are excluded from the export — they were never a public claim, so their absence is not a regression. The failure direction this protects against is recall, not libel: a missing gate risks withholding a true link, never fabricating a false one. But the surface it guards is about to triple, so a rules-neutral change must not be able to shrink it silently. Also drops the dead nameDistinctiveness import from load.mjs (referenced only in a comment since the ladder replaced it) and replaces it with a note on why it stays in classify.mjs. --- scripts/cacbg/audit.mjs | 55 +++++++++++++++++++++++++++ scripts/cacbg/audit.test.mjs | 73 +++++++++++++++++++++++++++++++++++- scripts/cacbg/load.mjs | 43 ++++++++++++++++++--- scripts/cacbg/load.test.mjs | 63 +++++++++++++++++++++++++++++++ 4 files changed, 226 insertions(+), 8 deletions(-) diff --git a/scripts/cacbg/audit.mjs b/scripts/cacbg/audit.mjs index a072a6d03..2a20db6af 100644 --- a/scripts/cacbg/audit.mjs +++ b/scripts/cacbg/audit.mjs @@ -7,10 +7,14 @@ import { DatabaseSync } from 'node:sqlite'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import fs from 'node:fs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; +import { RULES_VERSION } from '../tr/evidence.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); +const STAGING = process.env.CACBG_STAGING || path.join(ROOT, 'scratch/cacbg/staging'); +const SNAPSHOT = path.join(STAGING, 'published-snapshot.json'); const { companyNameKey } = await import('../../packages/shared/src/company-name-key.ts'); const db = new DatabaseSync(DB, { readOnly: true }); @@ -160,6 +164,40 @@ for (const l of nonExact) { db.close(); +// D. Monotonicity — ADR-0033 decision 6, the correction of #279 §8. +// +// §8 asked for a seal kept „forever" and strictly-additive recomputation. That is unachievable: labels +// flip, the deed cache expires, and a court can annul an entry (чл. 29 ЗТРРЮЛНЦ) with no rules change +// at all. So the seal is NOT a store — it is re-derived every run — and monotonicity is enforced HERE, +// as a gate, against the export load.mjs writes immediately before it drops the CACBG tables. +// +// The rule: a link that was published last run and is not published now is a REGRESSION unless the +// rules themselves changed. Under an unchanged rules_version nothing licensed the removal, so it is a +// hard finding. Under a changed one it is an intentional event and degrades to a printed diff. +// +// ship-related-persons.mjs's count floor cannot do this job: it compares a COUNT, so a one-for-one +// swap — one true link silently dropped, one gained — leaves it perfectly quiet. +const publishedNow = new Set(published.map((l) => l.link_key)); +let priorPublished = null; +try { + priorPublished = JSON.parse(fs.readFileSync(SNAPSHOT, 'utf8')); +} catch (e) { + // ENOENT is the legitimate first run — there is no prior surface to regress from. Anything else + // (unreadable, malformed) must not be swallowed into a silent pass of the gate. + if (e.code !== 'ENOENT') throw e; +} +const vanished = (priorPublished ?? []).filter((p) => !publishedNow.has(p.link_key)); +const regressions = vanished.filter((p) => p.rules_version === RULES_VERSION); +for (const p of regressions) + findings.push({ + axis: 'D_monotonicity', + link_key: p.link_key, + eik: p.link_key.split('|')[1] ?? '', + // The link_key is named explicitly: it is the only handle a human has to go and look at which + // claim disappeared, and the shared axis report prints the ЕИК alone. + detail: `${p.link_key} published last run under rules_version ${p.rules_version} (unchanged) and is not published now — nothing licensed this removal`, + }); + // Report const byAxis = {}; for (const f of findings) (byAxis[f.axis] ??= []).push(f); @@ -171,6 +209,23 @@ for (const [axis, fs] of Object.entries(byAxis)) { for (const f of fs.slice(0, 20)) console.log(` - [${f.eik}] ${f.detail}`); console.log(''); } +if (priorPublished === null) { + console.log(`## monotonicity — no prior export at ${SNAPSHOT}; treating this as a first run\n`); +} else { + const declared = vanished.filter((p) => p.rules_version !== RULES_VERSION); + console.log( + `## monotonicity — ${priorPublished.length} published last run, ${publishedNow.size} now; ` + + `${regressions.length} regression(s), ${declared.length} declared removal(s)\n`, + ); + // Declared removals are not findings, but they are never silent: a rules bump is exactly when a + // human should read which named claims it withdrew. + for (const p of declared) + console.log( + ` - ${p.link_key} removed under a rules change (${p.rules_version} → ${RULES_VERSION})`, + ); + if (declared.length) console.log(''); +} + console.log(`## non-exact provenance (${provenance.length}) — verify each cross-check by eye`); for (const p of provenance) { console.log(` - ${p.method} [${p.eik}] winner="${p.winner}"`); diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index 46d42121e..38f4c346d 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -26,10 +26,15 @@ const dirs = []; // Build a fixture DB (bidders + declarations + declared_interests + interest_links), run audit.mjs against // it as a subprocess, and return { threw, out } — threw=true iff the audit exited non-zero (a hard finding). -function buildAndAudit({ bidders, decls = [], dis = [], links, seals = [] }) { +function buildAndAudit({ bidders, decls = [], dis = [], links, seals = [], snapshot = null }) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-audit-')); dirs.push(dir); const DB = path.join(dir, 'fixture.sqlite'); + const staging = path.join(dir, 'staging'); + fs.mkdirSync(staging, { recursive: true }); + // The pre-wipe export load.mjs writes before it drops the CACBG tables. Absent === a first run. + if (snapshot) + fs.writeFileSync(path.join(staging, 'published-snapshot.json'), JSON.stringify(snapshot)); const db = new DatabaseSync(DB); db.exec(` CREATE TABLE bidders(id TEXT PRIMARY KEY, name TEXT, eik_normalized TEXT, eik_valid INT); @@ -57,7 +62,12 @@ function buildAndAudit({ bidders, decls = [], dis = [], links, seals = [] }) { out = execFileSync( 'node', ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'audit.mjs')], - { cwd: ROOT, env: { ...process.env, CACBG_DB: DB }, encoding: 'utf8', stdio: 'pipe' }, + { + cwd: ROOT, + env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: staging }, + encoding: 'utf8', + stdio: 'pipe', + }, ); } catch (e) { threw = true; // execFileSync throws on non-zero exit (a hard finding fired) @@ -221,3 +231,62 @@ test('a well-formed seal passes every C axis (positive control)', () => { }); assert.equal(threw, false, 'a correctly sealed link must not fail the gate'); }); + +// ── D. Monotonicity (ADR-0033 decision 6) ──────────────────────────────────────────────────────── +// #279 §8 asked for a seal kept „forever" and strictly-additive recomputation. ADR-0033 showed that is +// false (labels flip, the cache expires, a court can annul an entry) and replaced the STORE with a +// GATE: seals are re-derived every run, and the audit compares the current published set against the +// pre-wipe export. A link that vanishes under an UNCHANGED rules_version is a hard finding — nothing +// about the rules changed, so its disappearance is a regression, not a decision. +// +// Why this needs its own gate and ship's count floor cannot serve: assertShipFloor checks a COUNT. +// A one-for-one swap — one link lost, one gained — leaves the count identical and the floor silent +// while a true published link is dropped. Only a per-key comparison sees it. +const SEALED = [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'`, +]; +const ONE_LINK = { + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`, `'b2','ВТОРА ФИРМА ЕООД','200000002',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: SEALED, +}; + +test('D — a previously published link that vanished under an UNCHANGED rules_version is a hard finding', () => { + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, // published last run, gone now + ], + }); + assert.equal(threw, true, 'a silent recall regression must fail the build'); + assert.match(out, /D_monotonicity/); + assert.match(out, /p9\|200000002/); +}); + +test('D — the same disappearance under a CHANGED rules_version is a printed diff, not a finding', () => { + // Removal stays an intentional event: bumping the rules version is how you declare one. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [{ link_key: 'p9|200000002', rules_version: 'tr-rules-0' }], + }); + assert.equal(threw, false, 'a declared rules change must not fail the build'); + assert.match(out, /p9\|200000002/, 'but it must still be reported'); +}); + +test('D — no snapshot (a first run) neither fires nor crashes', () => { + const { threw } = buildAndAudit(ONE_LINK); + assert.equal(threw, false); +}); + +test('D positive control — an unchanged published set produces no monotonicity finding', () => { + // Without this, a gate that never fires would pass both negatives above. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [{ link_key: 'p1|100000001', rules_version: 'tr-rules-1' }], + }); + assert.equal(threw, false); + assert.doesNotMatch(out, /D_monotonicity/); +}); diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 5a37ae91c..3fe9d2940 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -14,12 +14,9 @@ import { DatabaseSync } from 'node:sqlite'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { - nameDistinctiveness, - temporalStatus, - localityToken, - closelyHeldForm, -} from './classify.mjs'; +// nameDistinctiveness is deliberately NOT imported: the evidence ladder replaced it in the publish +// path (ADR-0033). It survives in classify.mjs for the review queue, not for a publishing decision. +import { temporalStatus, localityToken, closelyHeldForm } from './classify.mjs'; import { openCache, readDeed, coverage } from '../tr/cache.mjs'; import { TR_DB, TR_RAW } from '../tr/paths.mjs'; import { evidenceVerdict, reconcileTermination, RULES_VERSION } from '../tr/evidence.mjs'; @@ -84,6 +81,40 @@ const suppressedFp = new Set(suppEntries.map((e) => e.fp)); // B3 unused-suppression gate: every listed fingerprint MUST match exactly one built link. Track which get // used; a fingerprint that matched nothing (a stale/mis-keyed takedown) fails the build after the load loop. const usedSuppressions = new Set(); +// Export the CURRENT published surface before anything is dropped — ADR-0033 decision 6. +// +// The rebuild below is total, so the previous run's published set exists only in this instant. The +// audit compares against this file and hard-fails on a link that was published last run and is not +// published now under an UNCHANGED rules_version: nothing licensed that removal, so it is a silent +// recall regression. rules_version travels per key, because a link that vanished under a rules BUMP is +// an intentional removal and must degrade to a printed diff instead. +// +// Held and withdrawn links are deliberately excluded: they were never a public claim, so their absence +// next run is not a regression. On a first run the tables do not exist yet and the export is an empty +// set — written, not skipped, so a missing file means „the loader never ran", not „nothing published". +const priorPublished = (() => { + try { + return db + .prepare( + `SELECT il.link_key AS link_key, e.rules_version AS rules_version + FROM interest_links il + LEFT JOIN interest_link_evidence e ON e.link_key = il.link_key + WHERE il.status = 'published'`, + ) + .all() + .map((r) => ({ link_key: r.link_key, rules_version: r.rules_version ?? RULES_VERSION })); + } catch (e) { + // A first run: interest_links does not exist yet. Any OTHER failure must surface — swallowing it + // would turn a broken export into a permanently silent gate. + if (!/no such table/i.test(e.message)) throw e; + return []; + } +})(); +fs.writeFileSync( + path.join(STAGING, 'published-snapshot.json'), + JSON.stringify(priorPublished, null, 2) + '\n', +); + // Full idempotent rebuild that also picks up schema changes: drop the CACBG tables (children first — // FK-safe) and re-apply the migration. Nothing to preserve — suppressions are external now. for (const t of [ diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index d90b44a44..bd5e5bc8c 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -1157,3 +1157,66 @@ test('a family stake publishes ONLY when the official confirmed the company them assert.equal(seal.matched_fact, 'seat:РУСЕ'); db.close(); }); + +test('the published surface is exported BEFORE the wipe, so the audit can gate monotonicity', () => { + // ADR-0033 decision 6. The loader rebuilds the CACBG tables from scratch every run, so the previous + // published set exists only in the instant before the DROP. Without this export the audit has + // nothing to compare against and the monotonicity gate can never fire — which is precisely the state + // review found: rules_version was written and never read. + // A FIRST run, on its own database: interest_links does not exist yet, so the export must be an + // empty set that is WRITTEN rather than skipped — a missing file has to keep meaning „the loader + // never ran", not „nothing was published". + const fresh = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-load-first-')); + fs.mkdirSync(path.join(fresh, 'staging'), { recursive: true }); + for (const f of fs.readdirSync(STAGING)) + fs.copyFileSync(path.join(STAGING, f), path.join(fresh, 'staging', f)); + // The base DB (bidders, contracts) comes from the main pipeline and is a precondition of the load — + // a genuinely empty file is not a first run, it is a broken one. So: copy the base and drop the + // CACBG tables, which is exactly the state before the loader has ever run against it. + const firstDb = path.join(fresh, 'first.sqlite'); + fs.copyFileSync(DB, firstDb); + const fdb = new DatabaseSync(firstDb); + for (const t of [ + 'interest_link_evidence', + 'interest_link_authorities', + 'interest_links', + 'declared_interests', + 'related_persons_internal', + 'declarations', + 'persons', + ]) + fdb.exec(`DROP TABLE IF EXISTS ${t}`); + fdb.close(); + runLoad({ CACBG_DB: firstDb, CACBG_STAGING: path.join(fresh, 'staging') }); + assert.deepEqual( + JSON.parse(fs.readFileSync(path.join(fresh, 'staging', 'published-snapshot.json'), 'utf8')), + [], + ); + fs.rmSync(fresh, { recursive: true, force: true }); + + const SNAP = path.join(STAGING, 'published-snapshot.json'); + runLoad(); + const db = open(); + const published = db + .prepare("SELECT link_key FROM interest_links WHERE status='published'") + .all() + .map((r) => r.link_key); + db.close(); + assert.ok(published.length > 0, 'fixture must publish something for this test to mean anything'); + + runLoad(); + const snap = JSON.parse(fs.readFileSync(SNAP, 'utf8')); + assert.deepEqual( + snap.map((s) => s.link_key).sort(), + [...published].sort(), + 'the export must hold exactly what was published before the wipe', + ); + // rules_version travels WITH each key: the gate distinguishes „vanished under unchanged rules" + // (a regression) from „vanished under a rules bump" (an intentional removal), and it cannot do that + // from the current version alone. + for (const s of snap) + assert.ok(s.rules_version?.length > 0, 'each exported key carries its rules'); + // Held and withdrawn links must NOT be exported — they were never a public claim, so their absence + // next run is not a regression to gate on. + assert.equal(snap.length, published.length); +}); From 76ed3b2bf16514ff99c80a3be7a74941bdd57b0a Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:31:03 +0300 Subject: [PATCH 17/50] docs(adr): record ADR-0033's amendments on the ADRs it amends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Superseded ADRs gain a "Superseded by" back-pointer; amended ones gained nothing. A reader of ADR-0007 — the foundational certainty bar — had no way to discover that ADR-0033 carved an explicit exception to its decisions 2 and 3. Adds an "Amended by" line to 0007, 0010, 0021 and 0028, each naming which decisions moved. A header breadcrumb, so the convention that accepted ADRs are not rewritten still holds. --- docs/adr/0007-scope-and-certainty-bar.md | 1 + docs/adr/0010-pii-posture.md | 1 + docs/adr/0021-methodology-page-and-temporal-freshness.md | 1 + docs/adr/0028-declared-eik-is-a-determining-identifier.md | 1 + 4 files changed, 4 insertions(+) diff --git a/docs/adr/0007-scope-and-certainty-bar.md b/docs/adr/0007-scope-and-certainty-bar.md index 18fcb4462..c41f42554 100644 --- a/docs/adr/0007-scope-and-certainty-bar.md +++ b/docs/adr/0007-scope-and-certainty-bar.md @@ -1,6 +1,7 @@ # ADR-0007: Scope and certainty bar - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — решения 2 и 3 (изричното изключение за „евристика, която твърди") - Date: 2026-07-05 - Deciders: lb (Head of AI, ИО), Claude - Related: spec §1–2, §8 diff --git a/docs/adr/0010-pii-posture.md b/docs/adr/0010-pii-posture.md index 8f09ead85..cf9898a04 100644 --- a/docs/adr/0010-pii-posture.md +++ b/docs/adr/0010-pii-posture.md @@ -1,6 +1,7 @@ # ADR-0010: PII posture - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — решения 3 и 6 (кешът от актове и срокът на съхранение) - Date: 2026-07-05 - Deciders: lb, Claude - Related: spec §8; КЗЛД guidance; GDPR Art. 6(1)(c)/(f), Art. 85 diff --git a/docs/adr/0021-methodology-page-and-temporal-freshness.md b/docs/adr/0021-methodology-page-and-temporal-freshness.md index 810a74b21..d2f0d9cd2 100644 --- a/docs/adr/0021-methodology-page-and-temporal-freshness.md +++ b/docs/adr/0021-methodology-page-and-temporal-freshness.md @@ -1,6 +1,7 @@ # ADR-0021: Public methodology/corrections page (E10) + temporal freshness & divestment expiry (E11) - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — E11 (етикетите за настояще време се отлагат за фаза 2) - Date: 2026-07-06 - Deciders: lb, Claude - Related: [ADR-0007](0007-scope-and-certainty-bar.md), [ADR-0019](0019-private-interest-vs-ex-officio-classification.md), [ADR-0020](0020-conflict-explorer-surface-posture.md); spec §5/§8/§9; `scripts/cacbg/load.mjs`, `apps/web/app/routes/conflict.methodology.tsx`, `packages/db/src/queries/related-persons.ts` diff --git a/docs/adr/0028-declared-eik-is-a-determining-identifier.md b/docs/adr/0028-declared-eik-is-a-determining-identifier.md index 9f123482e..9ad76597b 100644 --- a/docs/adr/0028-declared-eik-is-a-determining-identifier.md +++ b/docs/adr/0028-declared-eik-is-a-determining-identifier.md @@ -1,6 +1,7 @@ # ADR-0028: Declared ЕИК is a determining identifier — tier `A_eik`, exempt from the TR census - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — декларираният ЕИК остава определящ, но вече иска и регистърно доказателство - Date: 2026-07-11 - Deciders: lb, Claude - Related: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md), [ADR-0015](0015-tr-name-uniqueness-census.md), [ADR-0016](0016-free-text-entity-resolution.md); `scripts/cacbg/load.mjs`, `scripts/cacbg/classify.mjs` From 935ce1c1a2475a7f8afe8a89a059f8d4876244e1 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:31:03 +0300 Subject: [PATCH 18/50] test(web): scope the missing-source assertion to the card that has none MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit expect(text()).toContain('—') ran against the whole page, where the value and date cells render em-dashes of their own. It would have survived the source branch being deleted outright. Scope it to the card whose sourceUrl is null and assert the declaration anchor is absent from it. --- apps/web/app/routes/conflicts.render.test.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/web/app/routes/conflicts.render.test.tsx b/apps/web/app/routes/conflicts.render.test.tsx index ce2a78dbe..6be5ea6ee 100644 --- a/apps/web/app/routes/conflicts.render.test.tsx +++ b/apps/web/app/routes/conflicts.render.test.tsx @@ -194,8 +194,16 @@ describe('/conflicts route — render', () => { (a) => a.textContent === 'декларация', ); expect(sourceAnchor?.getAttribute('href')).toBe('https://register.cacbg.bg/2024/i.xml'); - // the zero-contract link has no toggle (contractCount === 0) and a muted „—" source - expect(text()).toContain('—'); + // Scoped to the card that actually has sourceUrl: null. A page-wide toContain('—') passes on any + // em-dash anywhere — including the ones the value and date cells render — so it would survive the + // source branch being deleted outright. + const cards = container.querySelectorAll('.conflict-card'); + const noSourceCard = [...cards].find((c) => c.textContent?.includes('ПРАЗЕН ООД'))!; + expect(noSourceCard).toBeTruthy(); + expect( + [...noSourceCard.querySelectorAll('a')].some((a) => a.textContent === 'декларация'), + ).toBe(false); + expect(noSourceCard.textContent).toContain('—'); }); it('a zero-contract link renders no „Виж договорите" toggle', async () => { From f4d73dcdb592bac882ab44be648440335c80125a Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 6 Aug 2026 14:39:16 +0300 Subject: [PATCH 19/50] =?UTF-8?q?docs(adr):=20resolve=20the=20=C2=A75/?= =?UTF-8?q?=C2=A710=20control-number=20gap=20in=20ADR-0033?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It was parked as "resolve with the author". It does not need the author — the issue's own numbers determine it. §5 gives each rung a control count: 4 barred + 281 document + 102 seat + 3 ЕИК + 21 refuted + 156 unknown = 567, plus the 4 links whose ЕИК is not in the register at all (§5 scopes the ladder to links "с изтеглен акт", so those reach no rung) = 571. §10's identity row is 281/102/156/21/4/4 = 568. The difference is exactly the "3 по ЕИК". The labels give the cause away. The identity row calls its second bucket "седалище" — one leg of rung 3 — while the evidence row two lines below calls the same rung "потвърдено". Rung 3 has two legs; the row tallied one and presented the result as a partition of the resolved set. A histogram that dropped a category, not a disputed measurement. The stated total 568 equals that row's sum exactly, so it is derived from the histogram rather than measured independently. Both therefore move together: the bucket becomes "потвърдено: 105", the resolved total becomes 571. F8 measures against those and reports disagreement rather than adopting whichever reading makes the reconciliation pass. The identified-vs-surfaced gap moves with it — 57 links on the corrected reading, 54 on the row as written — so that figure independently distinguishes the two, which is worth more than the correction itself. Also records what is NOT a discrepancy, so nobody "fixes" it: "извън ТР" is 4 in §10 and 3 in §3/§11 because the units differ — 3 ЕИК not in the register, carrying 4 links between them. §10's row counts links. --- ...-evidence-replaces-name-distinctiveness.md | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md index b3315c993..e75c28c38 100644 --- a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -261,11 +261,28 @@ None of these are follow-ups. - **Precision is proven by a hand-labelled sample, not by the control totals.** A human verifies against the portal: every reconciled link, every refuted link, and random samples of „Документ" and „Потвърдено". **Pass mark: zero wrong-company and zero wrong-person errors.** The control numbers of #279 §10 are a - reproducibility check — and they are usable as one only after the §5/§10 discrepancy is resolved with the - author (§5's „102 по седалище, 3 по ЕИК" leaves those three ЕИК confirmations in no bucket of the §10 - row, which sums to exactly 568 without them), and re-measured on top of the §12 phantom-row fix, which - changed the corpus — that fix has landed as #281 (87 phantom rows across 15 sets; 256,286 announced − - 87 = 256,199, covered exactly by 255,582 fetched + 617 missing at source). + reproducibility check, and two things had to be settled before they could serve as one. + + **The §5/§10 discrepancy is a dropped histogram category, not a disputed measurement.** §5's rungs carry + their own control counts: 4 barred + 281 document + 102 seat + 3 ЕИК + 21 refuted + 156 unknown = 567, + plus the 4 links whose ЕИК is not in the register at all (§5 scopes the ladder to links „с изтеглен акт", + so those reach no rung) = **571**. §10's identity row is 281 / 102 / 156 / 21 / 4 / 4 = **568**. The + difference is exactly the „3 по ЕИК". + + The cause is visible in the labels. The identity row names its second bucket **„седалище"** — one leg of + rung 3 — while the evidence row two lines below names the same rung **„потвърдено"** (251 / 78). Rung 3 + has two legs; the row tallied one and presented the result as a partition of the resolved set. And the + stated total 568 equals that row's sum exactly, which makes it a figure derived from the histogram rather + than measured independently. So both move together: the bucket becomes **„потвърдено: 105"** and the + resolved total becomes **571**. F8 measures against those and reports if its own count disagrees — the + reconciliation is not permitted to adopt whichever reading makes it pass. + + **Not a discrepancy, and not to be „fixed":** „извън ТР" is 4 in §10 and 3 in §3/§11 because the units + differ — 3 ЕИК that are not in the register, carrying 4 links between them. §10's row counts links. + + The numbers must also be re-measured on top of the §12 phantom-row fix, which changed the corpus — that + fix has landed as #281 (87 phantom rows across 15 sets; 256,286 announced − 87 = 256,199, covered exactly + by 255,582 fetched + 617 missing at source). - **Every anti-false-zero control is a positive control.** „Sofia seat confirmations: 0" is indistinguishable from a broken normalizer without one; so is a joint-stock bar with no marginal effect over `closelyHeldForm`, and a matcher that always returns false (ADR-0027's lesson). @@ -304,7 +321,8 @@ None of these are follow-ups. as-of evidence via `?entryDate=`, and the §7 „и към днешна дата" labels with their post-period contracts. - The seat rung is structurally capped and we accept the cap: a declared seat is captured **only** from the ООД/ЕООД holdings table of an *asset* declaration (`parse.mjs`), so it can never rescue a link declared - solely in an interests declaration. This is probably consistent with the measured 102 — rung 1 bars the + solely in an interests declaration. This is probably consistent with the 102 measured for the seat leg + specifically (not the 105 of the whole rung, whose other three come from the ЕИК leg) — rung 1 bars the joint-stock cases anyway — but it is confirmed against the corpus before launch, not assumed. ### Measured outcome — recorded on completion @@ -313,8 +331,10 @@ This ADR is accepted on the design. Exactly one amendment is permitted afterward filled in with the measured result, and nothing else in the file is rewritten (repo convention — an accepted ADR is superseded, not edited). To be recorded: -- the realized ladder split against the #279 §10 control row, with each number naming the SQL that produced - it, and the 54-link gap between identified (281 + 102) and surfaced (329) attributed bucket by bucket; +- the realized ladder split against the #279 §10 control row **as corrected above** (281 / 105 / 156 / 21 / + 4 / 4 = 571), with each number naming the SQL that produced it, and the gap between identified (281 + 105) + and surfaced (329) attributed bucket by bucket — 57 links on the corrected reading, 54 on the row as + written, so the figure itself distinguishes the two; - the true candidate-ЕИК count (#279's „~400" is an assumption; the figure is `COUNT(DISTINCT eik)` over all links, not just published ones); - the hand-labelled sample result; From 2b06b7638e754c6cb331b2452b4a1c9f078f4ecc Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 13:50:13 +0300 Subject: [PATCH 20/50] fix(tr): survive a malformed numeric entity instead of killing the crawl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit String.fromCodePoint throws RangeError above U+10FFFF, and the argument comes straight off the wire: `�` in any deed field threw out of decodeEntities, out of entityBlocks and registrySeat, past the crawl loop's refuse-and-continue block — which covered only JSON.parse and assertUicEcho — and out of run(). One malformed escape ended a paced crawl that had already spent its request budget, and did the same to load.mjs at decision time. An out-of-range code point is not a character and cannot be part of a name, so it now decodes to nothing and the rest of the entity still parses. The in-range path is unchanged and tested, so the guard bounds the decoder without disabling it. Widen the two refuse-and-continue blocks to match what they already claimed: the crawl loop now wraps the HTML parsing as well as the JSON, and load.mjs holds the single link whose deed it cannot read rather than failing the whole run — one bad payload out of ~400 must not decide every other link's fate and then trip the ship floor. --- scripts/cacbg/load.mjs | 37 +++++++++++++++++++++---------- scripts/tr/deed.mjs | 15 +++++++++++-- scripts/tr/deed.test.mjs | 35 +++++++++++++++++++++++++++++ scripts/tr/fetch-deeds.mjs | 45 ++++++++++++++++++++++---------------- 4 files changed, 100 insertions(+), 32 deletions(-) diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 3fe9d2940..7900a17e0 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -614,19 +614,23 @@ for (const rec of agg.values()) { // person in THIS company). nameDistinctiveness is deliberately NOT part of this gate: the seat rung // exists precisely to rescue a generic name, so requiring distinctiveness would empty it. const nameUnique = nameGloballyUnique(rec.key); + // „Неизвестна" — the withholding verdict, used for every way of ending up with no usable evidence. + const noEvidence = () => ({ + kind: 'unknown', + publishable: false, + registryRole: null, + matchedFact: null, + entryNumber: null, + entryDate: null, + rulesVersion: RULES_VERSION, + }); // With --allow-partial-tr the operator has accepted an incomplete cache. An uncached ЕИК then yields // no evidence at all, which is „Неизвестна" — held. It must never be read as a reason to publish. - const verdict = missing - ? { - kind: 'unknown', - publishable: false, - registryRole: null, - matchedFact: null, - entryNumber: null, - entryDate: null, - rulesVersion: RULES_VERSION, - } - : evidenceVerdict({ + let verdict; + if (missing) verdict = noEvidence(); + else { + try { + verdict = evidenceVerdict({ deed, outsideTr, declarantName: rec.person, @@ -636,6 +640,17 @@ for (const rec of agg.values()) { scope: rec.scope, nameGloballyUnique: nameUnique, }); + } catch (err) { + // A deed we cannot parse is a deed we cannot reason about — so this link withholds, exactly as an + // uncached one does, and the run continues. Failing the whole load instead would let one malformed + // deed out of ~400 decide the fate of every other link, and the ship floor would then refuse the + // reduced surface — turning a single bad payload into a total outage. Loud, per link, fail-closed. + console.error( + ` ${rec.eik}: evidence UNREADABLE — ${err instanceof Error ? err.message : err} (link held)`, + ); + verdict = noEvidence(); + } + } const tier = verdict.kind; const contemporaneous = [...years].some( (cy) => temporalStatus(declYears, cy) === 'contemporaneous', diff --git a/scripts/tr/deed.mjs b/scripts/tr/deed.mjs index fdb3c1bd8..71686ed8e 100644 --- a/scripts/tr/deed.mjs +++ b/scripts/tr/deed.mjs @@ -29,12 +29,23 @@ const ENTITIES = { '>': '>', ' ': ' ', }; +// `String.fromCodePoint` THROWS RangeError above U+10FFFF (and on a surrogate half), and the input is +// whatever the register put on the wire. An unguarded throw here does not stay local: it escapes +// entityBlocks and registrySeat, past the crawl loop's refuse-and-continue block (which covers only +// JSON.parse + assertUicEcho) and out of run() — one malformed escape in one deed ends a paced crawl +// that has already spent its request budget, and does the same to load.mjs at decision time. +// Out of range is not a character and cannot be part of a name, so it decodes to nothing: the rest of +// the entity still parses, which is the difference between losing a glyph and losing the run. +const MAX_CODE_POINT = 0x10ffff; +const codePoint = (n) => + Number.isInteger(n) && n >= 0 && n <= MAX_CODE_POINT ? String.fromCodePoint(n) : ''; + /** Decode the entity set the register actually emits, plus numeric escapes. FIRST step, always. */ function decodeEntities(s) { return String(s) .replace(/&(?:quot|apos|amp|lt|gt|nbsp);/g, (m) => ENTITIES[m]) - .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n))) - .replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCodePoint(parseInt(n, 16))); + .replace(/&#(\d+);/g, (_, n) => codePoint(Number(n))) + .replace(/&#x([0-9a-f]+);/gi, (_, n) => codePoint(parseInt(n, 16))); } /** Strip tags and collapse whitespace, INSIDE one already-isolated entity. */ diff --git a/scripts/tr/deed.test.mjs b/scripts/tr/deed.test.mjs index 9a7f4c97c..c45611c87 100644 --- a/scripts/tr/deed.test.mjs +++ b/scripts/tr/deed.test.mjs @@ -172,6 +172,41 @@ test('an empty htmlData yields no entities and does not throw', () => { assert.deepEqual(entityBlocks(null), []); }); +// A numeric entity is attacker-shaped input in the only sense that matters here: it comes off the +// wire, and `String.fromCodePoint` throws RangeError above U+10FFFF. That throw escapes decodeEntities, +// escapes entityBlocks and registrySeat, and — because the crawl loop's try/catch covers only +// JSON.parse + assertUicEcho — escapes run() and kills the process. One malformed entity in one deed +// would end a paced crawl that has already spent its request budget. Out of range is not a person, so +// the only defensible reading is „no character": drop it and keep parsing the rest of the entity. +test('an out-of-range numeric entity is dropped, never thrown out of the parser', () => { + const overflow = F19_THREE.replace('ПЕНКО', '�ПЕНКО'); + const blocks = entityBlocks(overflow); + assert.equal(blocks.length, 3, 'the deed still parses into its three entities'); + assert.match(blocks[1].text, /ПЕНКО НЕСТОРОВ НЕСТОРОВ/, 'the surrounding name survives intact'); + assert.ok(!blocks[1].text.includes('&#'), 'the escape itself does not survive as literal text'); +}); + +test('the hex numeric form is guarded too — both decode lines, not just the decimal one', () => { + const overflow = F19_THREE.replace('ИЛИЯН', '�ИЛИЯН'); + const blocks = entityBlocks(overflow); + assert.equal(blocks.length, 3); + assert.match(blocks[2].text, /ИЛИЯН КОСТАДИНОВ ФИЛИПОВ/); +}); + +test('an in-range numeric entity still decodes — the guard bounds, it does not disable', () => { + // П is „П". A guard that dropped every numeric escape would silently mangle real names. + const blocks = entityBlocks(F19_THREE.replace('ПЕНКО', 'ПЕНКО')); + assert.match(blocks[1].text, /ПЕНКО НЕСТОРОВ НЕСТОРОВ/); +}); + +test('registrySeat survives the same malformed entity rather than aborting the load', () => { + // registrySeat sits OUTSIDE the crawl loop's refuse-and-continue block (fetch-deeds.mjs), and + // load.mjs calls it again at decision time — so an unguarded throw here takes down both legs. + const d = deedOf([field('CR_F_5_L', F5_SEAT.replace('с. Марково', '�с. Марково'))]); + const seat = registrySeat(d); + assert.equal(seat.settlement, 'МАРКОВО'); +}); + test('liveFields keeps only the requested codes and reports entry date/number', () => { const d = deedOf([ field('CR_F_7_L', F7_MANAGER, { nameCode: 'CR_F_7_L', fieldEntryDate: '2017-09-15T00:00:00' }), diff --git a/scripts/tr/fetch-deeds.mjs b/scripts/tr/fetch-deeds.mjs index 78c63192c..cdea4f73f 100644 --- a/scripts/tr/fetch-deeds.mjs +++ b/scripts/tr/fetch-deeds.mjs @@ -216,35 +216,42 @@ export async function run({ continue; } - let deed; + // ONE refuse-and-continue block around EVERYTHING derived from the response — the JSON, the UIC + // echo, and the HTML parsing alike. The parsing used to sit outside it, which made the block's + // own promise false: a throw from deed.mjs escaped the loop, escaped run(), and killed the + // process, so one malformed deed ended a crawl that had already spent its paced request budget. + // The decode guard in deed.mjs removes the one throw we know of; this is the rail that holds when + // the next one appears, and the cost of being wrong here is measured in hours of pacing. try { - deed = JSON.parse(res.body.toString('utf8')); + const deed = JSON.parse(res.body.toString('utf8')); // The deed we got back must be the deed we asked for, or every claim derived from it names // the wrong company (R8). assertUicEcho(deed, eik); + + // The raw response is the ONLY place names live; it stays under git-ignored scratch. Written + // only after the echo check, so a deed for the wrong company never lands on disk. + atomicWrite(deedPath(eik, rawDir), res.body); + const seat = registrySeat(deed); + const form = registryLegalForm(deed); + upsertDeed(db, { + eik, + status: 'fetched', + httpStatus: 200, + fetchedAt: now().toISOString(), + rawPath: path.relative(rawDir, deedPath(eik, rawDir)), + bodySha256: crypto.createHash('sha256').update(res.body).digest('hex'), + legalFormCode: form.code, + legalFormVerdict: form.verdict, + seatNormalized: seat.settlement || null, + seatEntryDate: seat.entryDate, + latestOwnEntryDate: latestOwnershipEntryDate(deed), + }); } catch (err) { console.error(` ${eik}: REFUSED — ${err instanceof Error ? err.message : err}`); unresolved++; consecutive++; continue; } - - // The raw response is the ONLY place names live; it stays under git-ignored scratch. - atomicWrite(deedPath(eik, rawDir), res.body); - const seat = registrySeat(deed); - upsertDeed(db, { - eik, - status: 'fetched', - httpStatus: 200, - fetchedAt: now().toISOString(), - rawPath: path.relative(rawDir, deedPath(eik, rawDir)), - bodySha256: crypto.createHash('sha256').update(res.body).digest('hex'), - legalFormCode: registryLegalForm(deed).code, - legalFormVerdict: registryLegalForm(deed).verdict, - seatNormalized: seat.settlement || null, - seatEntryDate: seat.entryDate, - latestOwnEntryDate: latestOwnershipEntryDate(deed), - }); consecutive = 0; } From 95d8c513444e499c570bd8793931b89961de8be5 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 13:52:10 +0300 Subject: [PATCH 21/50] test(db): put the evidence seal gate under actual test pressure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deleting the whole `EXISTS (… interest_link_evidence …)` clause out of SURFACED_OWNERSHIP left the db suite green at 361/361. Every fixture sealed every link 'document', so the one piece of SQL standing between a withheld link and a named public claim — that a specific official owns a specific company — had no test on it at all. Seed each real withholding rung evidence.mjs can emit (refuted, bar_joint_stock, unknown, outside_tr) plus a link with no seal row, all on the same winner and otherwise identical, and assert absence across all four public queries: the leaderboard, the official page, the company page and the contract drill-down. The 'confirmed' positive control is the load-bearing part. Without it an absent row could be absent for an unrelated reason and every assertion would pass vacuously — which is the failure mode being fixed, not a shape to reproduce. Verified by mutation: with the gate removed, six of these fail and the positive control still passes. --- packages/db/src/related-persons-sql.test.ts | 101 ++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index 1c88b94f4..380949a0a 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -524,4 +524,105 @@ describe('свързани-лица SQL (real SQLite)', () => { ); }); }); + + // ── the evidence seal gate (#279, ADR-0033 decision 1) ────────────────────── + // + // This block exists because the gate had NO test pressure at all: deleting the entire + // `EXISTS (… interest_link_evidence …)` clause out of SURFACED_OWNERSHIP left the whole db suite + // green (361/361), because every fixture above seals every link 'document'. That clause is the only + // SQL between a WITHHELD link and a named public claim that a specific official owns a specific + // company, so an untested one is the most expensive kind of dead rail. + // + // Each rung below is a REAL withholding outcome of evidence.mjs, not an invented value: + // refuted — the register contradicts the declared stake + // bar_joint_stock — an АД/ЕАД, where a declared parcel of shares is not a material conflict + // unknown — no rung reached; the deed proves nothing either way + // outside_tr — ДЗЗД/BULSTAT, not in the Trade Register at all + // …plus a link with NO seal row whatsoever, which is what a half-loaded run produces. + describe('a link withheld by its evidence seal never reaches any public query', () => { + // All six share the SAME winner (eik 111, which has live contracts) and the same shape, so the + // ONLY thing that differs is the seal. Without that, an absent row could be absent for an unrelated + // reason and every assertion here would pass vacuously — which is exactly the bug being fixed. + const WITHHELD = [ + ['refuted', 'person:ref', 'Оборен Тестов'], + ['bar_joint_stock', 'person:bar', 'Акционер Тестов'], + ['unknown', 'person:unk', 'Неясен Тестов'], + ['outside_tr', 'person:out', 'Извън Тестов'], + ] as const; + + function seedRungs(dbPath: string): void { + const people = [...WITHHELD.map(([, id, name]) => [id, name] as const)]; + people.push(['person:none', 'Безпечатен Тестов'], ['person:ok', 'Потвърден Тестов']); + const links = [...WITHHELD.map(([, id]) => id), 'person:none', 'person:ok']; + sqlite( + dbPath, + `INSERT INTO persons (id, name) VALUES ${people.map(([id, n]) => `('${id}','${n}')`).join(',')}; + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES + ${links + .map( + (p) => + `('il:${p.split(':')[1]}','${p}|111','${p}','eik:111','111','ТРЕЙС ГРУП ХОЛД АД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'none',1,'2019','2023',3,1000,'2020','2021','published')`, + ) + .join(',')}; + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) VALUES + ${WITHHELD.map(([kind, id]) => `('${id}|111','${kind}',NULL,NULL,'2026-08-05','tr-rules-1','live')`).join(',')}, + ('person:ok|111','confirmed',NULL,'seat:СОФИЯ','2026-08-05','tr-rules-1','live');`, + ); + // person:none deliberately gets NO evidence row at all. + } + + it('the POSITIVE CONTROL surfaces — so every absence below is caused by the seal, not the fixture', () => { + withDb((dbPath) => { + seedRungs(dbPath); + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => r.official === 'Потвърден Тестов')).toBe(true); + expect(rows(dbPath, lit(OFFICIAL_SQL, 'person:ok'))).toHaveLength(1); + }); + }); + + for (const [kind, personId, name] of WITHHELD) { + it(`'${kind}' is withheld from the leaderboard, the official page, the company page and the drill-down`, () => { + withDb((dbPath) => { + seedRungs(dbPath); + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => r.official === name)).toBe(false); + expect(board.some((r) => r.link_key === `${personId}|111`)).toBe(false); + // The official's own page must 404 rather than render a withheld claim under their name… + expect(rows(dbPath, lit(OFFICIAL_SQL, personId))).toHaveLength(0); + // …the company page must not list them among that winner's office-holders… + expect(rows(dbPath, lit(COMPANY_SQL, '111')).some((r) => r.official === name)).toBe( + false, + ); + // …and the drill-down must not enumerate the contracts of a link it may not name. + expect(rows(dbPath, lit(LINK_CONTRACTS_SQL, `${personId}|111`))).toHaveLength(0); + }); + }); + } + + it('a link with NO seal row at all is withheld too — a half-loaded run must not publish', () => { + // The LEFT JOIN in LINK_SELECT would happily return this row with NULL evidence columns; only the + // EXISTS gate keeps it off the surface. This is the case a partial load actually produces. + withDb((dbPath) => { + seedRungs(dbPath); + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => r.official === 'Безпечатен Тестов')).toBe(false); + expect(rows(dbPath, lit(OFFICIAL_SQL, 'person:none'))).toHaveLength(0); + expect( + rows(dbPath, lit(COMPANY_SQL, '111')).some((r) => r.official === 'Безпечатен Тестов'), + ).toBe(false); + expect(rows(dbPath, lit(LINK_CONTRACTS_SQL, 'person:none|111'))).toHaveLength(0); + }); + }); + + it('the company page shows ONLY the sealed office-holders of that winner', () => { + // Four withheld links and one confirmed link all point at eik 111, alongside the base fixture's + // Иван. A gate that let any withheld rung through would show up here as an extra name. + withDb((dbPath) => { + seedRungs(dbPath); + const names = rows(dbPath, lit(COMPANY_SQL, '111')).map((r) => r.official); + expect(names.sort()).toEqual(['Иван Минев', 'Потвърден Тестов']); + }); + }); + }); }); From 0a6131795978e56bde7882ce5663f48787ceabec Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 13:54:46 +0300 Subject: [PATCH 22/50] fix(db): withhold an unrecognised evidence seal instead of upgrading it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit toLink read `evidence_kind === 'confirmed' ? 'confirmed' : 'document'`, so NULL, 'refuted', 'bar_joint_stock', 'outside_tr', a rung added by a later rules_version, or a typo all became 'document' — the strongest claim the card can make, rendering „лицето е вписано като съдружник/собственик": that the register names this specific person in this specific company. The SQL gate makes that unreachable today, but the failure direction was backwards at the one mapping in the codebase where a wrong default is a defamatory statement about a named human being rather than a rendering glitch. The LEFT JOIN in LINK_SELECT was deliberately left non-inner so a contradiction would surface here; the fallback then converted exactly that contradiction into the strongest available label. Rows are now filtered to the two publishing rungs before mapping, and filtered before the emptiness check so an official whose every link is withheld 404s rather than rendering an empty page under their name. One withheld row no longer affects its sealed siblings. --- .../db/src/queries/related-persons.test.ts | 51 +++++++++++++++++++ packages/db/src/queries/related-persons.ts | 35 ++++++++++--- 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/packages/db/src/queries/related-persons.test.ts b/packages/db/src/queries/related-persons.test.ts index 519856517..8ba09404e 100644 --- a/packages/db/src/queries/related-persons.test.ts +++ b/packages/db/src/queries/related-persons.test.ts @@ -33,6 +33,13 @@ function row(over: Record = {}) { first_contract_year: '2021', last_contract_year: '2024', source_url: 'https://register.cacbg.bg/2024/i.xml', + // SURFACED_OWNERSHIP only returns rows carrying a publishing seal, so the fixture carries one too — + // a fixture without it would exercise a row shape the SQL cannot produce. + evidence_kind: 'document', + registry_role: 'owner', + entry_number: '20110502101007', + entry_date: '2011-05-02', + lookup_date: '2026-08-05', ...over, }; } @@ -197,3 +204,47 @@ describe('conflict reads soft-fail on an un-migrated env (no 500)', () => { } }); }); + +// The evidence seal is what licenses the card's registry sentence, and the two rungs assert very +// different things: 'document' renders „лицето е вписано като съдружник/собственик" — that the register +// names THIS person in THIS company — while 'confirmed' claims only that the declared data matched. +// The mapper used to read `kind === 'confirmed' ? 'confirmed' : 'document'`, so NULL, 'refuted', a rung +// added later, or a typo all became the STRONGEST claim we can make about a named human being. The SQL +// gate makes that unreachable today; the failure DIRECTION is still wrong, and this is the one place in +// the codebase where being wrong by default is libel rather than a rendering glitch. +describe('an unrecognised evidence seal withholds the link instead of upgrading it', () => { + for (const kind of ['refuted', 'unknown', 'bar_joint_stock', 'outside_tr', 'future_rung_v9']) + it(`'${kind}' never renders as a registry claim`, async () => { + const db = fakeDb({ '10': [row({ evidence_kind: kind })] }); + expect(await getConflictLeaderboard(db, 10)).toEqual([]); + }); + + it('a NULL seal — the half-migrated read the old comment invited — withholds too', async () => { + const db = fakeDb({ '10': [row({ evidence_kind: null })] }); + expect(await getConflictLeaderboard(db, 10)).toEqual([]); + }); + + it('both publishing rungs still map, and to DIFFERENT kinds — the guard bounds, it does not flatten', async () => { + for (const kind of ['document', 'confirmed']) { + const db = fakeDb({ '10': [row({ evidence_kind: kind })] }); + const links = await getConflictLeaderboard(db, 10); + expect(links).toHaveLength(1); + expect(links[0]!.evidenceKind).toBe(kind); + } + }); + + it('the official page 404s rather than render a page under a name with nothing left to show', async () => { + const db = fakeDb({ 'person:ivan': [row({ evidence_kind: 'refuted' })] }); + expect(await getOfficialConflicts(db, 'person:ivan')).toBeNull(); + expect( + await getCompanyConflicts(fakeDb({ '111': [row({ evidence_kind: null })] }), '111'), + ).toBeNull(); + }); + + it('one withheld row does not take its sealed siblings down with it', async () => { + const db = fakeDb({ + '10': [row({ link_key: 'ok|111' }), row({ link_key: 'bad|111', evidence_kind: 'refuted' })], + }); + expect((await getConflictLeaderboard(db, 10)).map((l) => l.linkKey)).toEqual(['ok|111']); + }); +}); diff --git a/packages/db/src/queries/related-persons.ts b/packages/db/src/queries/related-persons.ts index ff99e7f51..8451b01a1 100644 --- a/packages/db/src/queries/related-persons.ts +++ b/packages/db/src/queries/related-persons.ts @@ -190,6 +190,25 @@ export const LINK_SELECT = `SELECT il.link_key, il.person_id, p.name AS official -- …and drop a family link redundant with the official's own stake in the same winner (de-anon guard). AND ${NOT_REDUNDANT_FAMILY}`; +// The two rungs that license a public claim (ADR-0033 decision 1), and the ONLY two the card knows how +// to render. Anything else — 'refuted', 'unknown', 'bar_joint_stock', 'outside_tr', a rung added by a +// later rules_version, a typo, or NULL from a row with no seal — is withheld, never mapped. +// +// The mapper used to read `kind === 'confirmed' ? 'confirmed' : 'document'`, which turned every one of +// those into 'document' — the STRONGEST claim on the surface, rendering „лицето е вписано като +// съдружник/собственик": that the register names this specific person in this specific company. The SQL +// gate makes it unreachable today, but the direction was wrong, and this is the one mapping in the +// codebase where a default is a defamatory statement about a named human being rather than a glitch. +// The LEFT JOIN in LINK_SELECT is deliberately not an inner one so a contradiction SURFACES here; the +// old fallback then converted exactly that contradiction into the strongest possible label. +const PUBLISHING_EVIDENCE = new Set(['document', 'confirmed']); + +/** Rows whose seal licenses a public claim. Withholding is silent by design — the SQL already filters + * these out, so anything reaching here is a contradiction to drop, not a condition to report per row. */ +function sealed(rows: LinkRow[]): LinkRow[] { + return rows.filter((r) => PUBLISHING_EVIDENCE.has(String(r.evidence_kind))); +} + // own_institution is a 4-value verdict; only the deterministic 'exact' surfaces as true (the // name_contains/locality heuristics are disclosed elsewhere, never asserted as fact). function toLink(r: LinkRow): ConflictLink { @@ -213,11 +232,9 @@ function toLink(r: LinkRow): ConflictLink { firstContractYear: r.first_contract_year, lastContractYear: r.last_contract_year, sourceUrl: r.source_url, - // SURFACED_OWNERSHIP guarantees a publishing seal exists, so these are non-null in practice; the - // fallbacks keep a half-migrated environment rendering rather than throwing on a null read. - evidenceKind: (r.evidence_kind === 'confirmed' ? 'confirmed' : 'document') as - | 'document' - | 'confirmed', + // Narrowed, not defaulted — `sealed()` above has already dropped every other value, so this asserts + // what the filter guarantees instead of inventing a rung the row never carried. + evidenceKind: r.evidence_kind as 'document' | 'confirmed', registryRole: r.registry_role === 'owner' || r.registry_role === 'manager' ? r.registry_role : null, registryEntryNumber: r.entry_number, @@ -233,7 +250,7 @@ export const LEADERBOARD_SQL = `${LINK_SELECT} * relative's) in a procurement winner, ranked NEXUS-first (own-institution → contemporaneous → value). */ export async function getConflictLeaderboard(db: D1Database, limit = 100): Promise { try { - const rows = (await db.prepare(LEADERBOARD_SQL).bind(limit).all()).results; + const rows = sealed((await db.prepare(LEADERBOARD_SQL).bind(limit).all()).results); return rows.map(toLink); } catch (e) { if (conflictSchemaAbsent(e, 'leaderboard')) return []; // un-migrated env → empty surface, not a 500 @@ -251,7 +268,9 @@ export async function getOfficialConflicts( personId: string, ): Promise { try { - const rows = (await db.prepare(OFFICIAL_SQL).bind(personId).all()).results; + // Filtered BEFORE the emptiness check, so a person whose every link is withheld 404s rather than + // rendering an empty page under their name. + const rows = sealed((await db.prepare(OFFICIAL_SQL).bind(personId).all()).results); if (rows.length === 0) return null; const links = rows.map(toLink); return { official: links[0]!.official, links }; @@ -270,7 +289,7 @@ export async function getCompanyConflicts( eik: string, ): Promise { try { - const rows = (await db.prepare(COMPANY_SQL).bind(eik).all()).results; + const rows = sealed((await db.prepare(COMPANY_SQL).bind(eik).all()).results); if (rows.length === 0) return null; return { company: rows[0]!.company, eik, links: rows.map(toLink) }; } catch (e) { From df76ae742d61de6ff077c7683879ea1a96daf631 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 13:57:11 +0300 Subject: [PATCH 23/50] fix(cacbg): bound the seat matched_fact so a name cannot ride the seal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The closed-vocabulary rail admitted exactly what it exists to catch. `seat:` is a legitimate prefix and the settlement pattern was unbounded, so `seat:ИВАН ПЕТРОВ ГЕОРГИЕВ` — a full three-part Bulgarian name (ЗГР чл. 9) wearing an allowed prefix — passed the audit. That is the value a mis-split of the seat field would produce, which is the leak the rail was written for. A Bulgarian settlement is one or two tokens („СОФИЯ", „ВЕЛИКО ТЪРНОВО", „ГЕНЕРАЛ ТОШЕВО"); a three-part name is exactly three. Bounding at two separates them, and a rarer three-token seat stops the run for a human rather than publishing — the right direction for a rail whose failure mode is putting somebody's name on a served column. Moved to evidence.mjs next to the code that writes matched_fact, so the gate and the writer cannot drift apart, and enforced at write time as well: the audit runs after the whole domain is built, by which point the name is already in a table. --- scripts/cacbg/audit.mjs | 10 ++++++---- scripts/cacbg/audit.test.mjs | 35 +++++++++++++++++++++++++++++++++++ scripts/cacbg/load.mjs | 16 +++++++++++++++- scripts/tr/evidence.mjs | 24 ++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 5 deletions(-) diff --git a/scripts/cacbg/audit.mjs b/scripts/cacbg/audit.mjs index 2a20db6af..9c8355f6e 100644 --- a/scripts/cacbg/audit.mjs +++ b/scripts/cacbg/audit.mjs @@ -9,7 +9,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import fs from 'node:fs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; -import { RULES_VERSION } from '../tr/evidence.mjs'; +import { RULES_VERSION, isSealedFact } from '../tr/evidence.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); @@ -46,8 +46,10 @@ const published = db // The only two evidence rungs that publish (ADR-0033 decision 1). Everything else withholds. const PUBLISHING_EVIDENCE = new Set(['document', 'confirmed']); -// The closed vocabulary for a sealed matched_fact. Anything else is a potential name leak. -const MATCHED_FACT = /^(?:seat:[\p{Lu} -]+|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; +// The closed vocabulary for a sealed matched_fact lives in evidence.mjs, next to the code that WRITES +// it — one definition, so the gate can never permit a shape the writer has stopped emitting (or, worse, +// the other way round). `isSealedFact` bounds the seat leg to a settlement's one or two tokens, which is +// what stops a three-part Bulgarian name riding through behind the legitimate `seat:` prefix. const findings = []; const flag = (link, axis, detail) => @@ -111,7 +113,7 @@ for (const l of published) { // C2. The PII rail, audited rather than assumed: matched_fact is a CLOSED vocabulary and can never // carry a name. The registry deed's names are read only to produce a boolean and must never reach // a served column (#279 §9, ADR-0033 decision 5). A schema cannot enforce this; this does. - if (l.matched_fact != null && !MATCHED_FACT.test(l.matched_fact)) + if (!isSealedFact(l.matched_fact)) flag( l, 'C_matched_fact_shape', diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index 38f4c346d..712178c31 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -218,6 +218,41 @@ test('a matched_fact outside the closed vocabulary is a hard finding — the nam assert.equal(/C_matched_fact_shape/.test(out), true, out); }); +// …and the rail must catch the shape a leak would ACTUALLY take. `seat:` is a legitimate member +// of the vocabulary, so an unbounded `seat:[\p{Lu} -]+` admits `seat:ИВАН ПЕТРОВ ГЕОРГИЕВ` — a full +// three-part Bulgarian name (ЗГР чл. 9) wearing the prefix of a fact we allow. That is precisely the +// value the rail exists to reject, and it is the one a mis-split of the seat field would produce. +// A Bulgarian settlement is one or two tokens („СОФИЯ", „ВЕЛИКО ТЪРНОВО", „ГЕНЕРАЛ ТОШЕВО"); the +// three-part name is exactly three. The bound is deliberately tight: a rarer 3-token seat trips the +// audit and a human adjudicates, which is the correct direction for a rail whose failure mode is +// publishing somebody's name. +test('a THREE-TOKEN seat is a name shape, not a settlement — the rail must catch it', () => { + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','confirmed','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','confirmed',NULL,'seat:ИВАН ПЕТРОВ ГЕОРГИЕВ','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'a three-part name behind seat: must fail the gate'); + assert.equal(/C_matched_fact_shape/.test(out), true, out); +}); + +test('a real two-token settlement still passes — the bound must not empty the seat rung', () => { + const { threw } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','confirmed','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','confirmed',NULL,'seat:ВЕЛИКО ТЪРНОВО','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, false, 'ВЕЛИКО ТЪРНОВО is a settlement and must survive the rail'); +}); + test('a well-formed seal passes every C axis (positive control)', () => { // Without this, all four negatives above would still pass if the axes fired unconditionally. const { threw } = buildAndAudit({ diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 7900a17e0..df5845dc9 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -19,7 +19,12 @@ import { fileURLToPath } from 'node:url'; import { temporalStatus, localityToken, closelyHeldForm } from './classify.mjs'; import { openCache, readDeed, coverage } from '../tr/cache.mjs'; import { TR_DB, TR_RAW } from '../tr/paths.mjs'; -import { evidenceVerdict, reconcileTermination, RULES_VERSION } from '../tr/evidence.mjs'; +import { + evidenceVerdict, + isSealedFact, + reconcileTermination, + RULES_VERSION, +} from '../tr/evidence.mjs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; import { fingerprint, loadSuppressions, SUPPRESSION_KEY_VERSION } from './suppressions.mjs'; import { canonicalInstitution } from './institutions.mjs'; @@ -773,6 +778,15 @@ for (const rec of agg.values()) { : recon.label === 'manager_today' ? 'terminated_manager_still' : 'terminated'; + // Refuse at WRITE time, not only in the post-hoc audit. The audit runs after the whole domain is + // built; by then the name is already in a table, and a run that dies on a suppressed-by-default axis + // could ship it. `matched_fact` is the one sealed column derived from a deed's text, so it is the one + // place a third-party name can reach a served row — the rail belongs where the value is produced. + if (!isSealedFact(verdict.matchedFact)) + throw new Error( + `REFUSE TO SEAL: matched_fact for ${linkKey} is outside the closed vocabulary — a name may have ` + + `leaked out of a deed (#279 §9, ADR-0033 decision 5)`, + ); insEvidence.run( linkKey, verdict.kind, diff --git a/scripts/tr/evidence.mjs b/scripts/tr/evidence.mjs index f6619dad1..01fce0e89 100644 --- a/scripts/tr/evidence.mjs +++ b/scripts/tr/evidence.mjs @@ -40,6 +40,30 @@ export const RULES_VERSION = 'tr-rules-1'; /** Rung 2 needs a real three-part Bulgarian name (ЗГР чл. 9). Two tokens is the homonym risk itself. */ const MIN_NAME_TOKENS = 3; +/** + * The CLOSED vocabulary a sealed `matched_fact` may take: `seat:`, `role:owner:`, + * `role:manager:`, or `eik`. It must NEVER carry the matched NAME — the deed's names are read + * only to produce a boolean and never leave git-ignored scratch (#279 §9, ADR-0033 decision 5). + * + * The seat token bound is the whole rail. `seat:` is a legitimate prefix, so an unbounded settlement + * pattern admits `seat:ИВАН ПЕТРОВ ГЕОРГИЕВ` — a full three-part Bulgarian name (ЗГР чл. 9) wearing an + * allowed prefix, which is exactly the value a mis-split of the seat field would produce and exactly + * what the rail exists to reject. A settlement is one or two tokens („СОФИЯ", „ВЕЛИКО ТЪРНОВО", + * „ГЕНЕРАЛ ТОШЕВО"); a three-part name is exactly three. Bounding at two separates them cleanly, and a + * rarer 3-token seat stops the run for a human rather than publishing — the correct direction for a rail + * whose failure mode is putting somebody's name on a served column. + * + * Defined ONCE and consumed by both the writer (load.mjs) and the audit, so the two cannot drift into + * a state where the gate permits what the writer emits. + */ +export const MATCHED_FACT_RE = + /^(?:seat:\p{Lu}[\p{Lu}-]*(?: \p{Lu}[\p{Lu}-]*)?|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + +/** True when `fact` is a member of the closed vocabulary. `null` is legal — a rung may match no fact. */ +export function isSealedFact(fact) { + return fact == null || MATCHED_FACT_RE.test(String(fact)); +} + // Court-registered companies were re-registered into the Търговски регистър in a single administrative // push, which flattened their entry dates into this window. „Strictly before the declared period" // certifies nothing when the date is an artefact of the migration rather than of the ownership, so the From 2d3c06420a60122a2b3f87175bfb79333fb444c0 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 13:58:30 +0300 Subject: [PATCH 24/50] fix(seed): seal the dev fixture so /conflicts is not empty on a fresh DB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seed exists for one reason — without it `/conflicts` renders an empty surface and the feature is unverifiable in a browser. Then #279 added the evidence seal requirement to SURFACED_OWNERSHIP, the seed wrote no seals, and the fixture stopped surfacing: this PR's own dev fixture died under this PR's own gate. Seal each seeded link with the rung it represents — document for the official's own stake, confirmed for the relative's seat-proved link, bar_joint_stock for the withheld АД — so the local surface shows the same three outcomes as production and keeps its negative case. The new test runs the real migrations, the real seed and the real exported queries, because the seed is a promise about the read gate and nothing was checking it. A future change to the publishing rule that the seed does not keep up with now fails here instead of in a browser days later. --- packages/db/src/seed-surface.test.ts | 95 ++++++++++++++++++++++++++++ scripts/seed.sql | 21 ++++++ 2 files changed, 116 insertions(+) create mode 100644 packages/db/src/seed-surface.test.ts diff --git a/packages/db/src/seed-surface.test.ts b/packages/db/src/seed-surface.test.ts new file mode 100644 index 000000000..e9484d25a --- /dev/null +++ b/packages/db/src/seed-surface.test.ts @@ -0,0 +1,95 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { COMPANY_SQL, LEADERBOARD_SQL, OFFICIAL_SQL } from './queries/related-persons'; + +// The dev seed exists for ONE reason: `/conflicts` renders an empty surface without it, so the whole +// feature is unverifiable in a browser. That makes the seed a promise about the read gate, and a promise +// nothing checked — it was written against SURFACED_OWNERSHIP as it stood, then #279 added the evidence +// seal requirement to that same predicate and the fixture silently stopped surfacing. A fresh dev DB +// rendered empty, which is exactly the state the seed was created to prevent. +// +// So: run the REAL migrations, the REAL seed.sql, and the REAL exported queries. Any future change to +// the publishing gate that the seed does not keep up with now fails here rather than in someone's +// browser, days later, as „the local app looks broken". + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migrations = [ + 'packages/db/migrations/0000_init.sql', + 'packages/db/migrations/0003_related_persons_foundation.sql', + 'packages/db/migrations/0006_interest_link_evidence.sql', +].map((m) => resolve(root, m)); +const seed = resolve(root, 'scripts/seed.sql'); + +function seededDb(fn: (dbPath: string) => T): T { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-seed-')); + const dbPath = resolve(dir, 'seed.sqlite'); + try { + for (const file of [...migrations, seed]) + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${file}\n`, stdio: 'pipe' }); + return fn(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} +function rows(dbPath: string, sql: string): Record[] { + const out = execFileSync('sqlite3', ['-json', dbPath], { input: sql, encoding: 'utf8' }).trim(); + return out ? JSON.parse(out) : []; +} +function lit(sql: string, ...vals: (string | number)[]): string { + let i = 0; + return sql.replace(/\?/g, () => { + const v = vals[i++]; + return typeof v === 'number' ? String(v) : `'${String(v).replace(/'/g, "''")}'`; + }); +} + +describe('scripts/seed.sql produces a surface a developer can actually see', () => { + it('the leaderboard is NOT empty — the seed survives every gate in SURFACED_OWNERSHIP', () => { + seededDb((dbPath) => { + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.length).toBeGreaterThan(0); + // Both published outcomes the seed is built around: the official's own stake and a relative's. + expect(board.map((r) => r.relation).sort()).toEqual(['owns', 'related']); + }); + }); + + it('every surfaced seed link carries a real publishing seal, not a coincidence', () => { + seededDb((dbPath) => { + for (const r of rows(dbPath, lit(LEADERBOARD_SQL, 100))) + expect(['document', 'confirmed']).toContain(r.evidence_kind); + }); + }); + + it('the held joint-stock link stays off the surface — the seed keeps its negative case', () => { + // ГАМА ИНВЕСТ АД is seeded as a withheld link precisely so a change to the publishing rule shows up + // as a before/after rather than as an empty page either way. If it ever surfaces, the materiality + // bar (ADR-0022) or the seal gate has quietly stopped working. + seededDb((dbPath) => { + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => String(r.company).includes('ГАМА ИНВЕСТ'))).toBe(false); + expect(rows(dbPath, lit(COMPANY_SQL, '204556676'))).toHaveLength(0); + }); + }); + + it('the official and company pages render too — not just the leaderboard', () => { + seededDb((dbPath) => { + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + const own = board.find((r) => r.relation === 'owns')!; + expect(rows(dbPath, lit(OFFICIAL_SQL, String(own.person_id))).length).toBeGreaterThan(0); + expect(rows(dbPath, lit(COMPANY_SQL, String(own.eik))).length).toBeGreaterThan(0); + }); + }); + + it('re-running the seed is idempotent — INSERT OR IGNORE, no duplicated links', () => { + seededDb((dbPath) => { + const before = rows(dbPath, lit(LEADERBOARD_SQL, 100)).length; + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${seed}\n`, stdio: 'pipe' }); + expect(rows(dbPath, lit(LEADERBOARD_SQL, 100))).toHaveLength(before); + }); + }); +}); diff --git a/scripts/seed.sql b/scripts/seed.sql index 9da64854b..3d26a0da1 100644 --- a/scripts/seed.sql +++ b/scripts/seed.sql @@ -28,6 +28,7 @@ INSERT OR IGNORE INTO bidders (id, name, bulstat) VALUES -- • a LIVE contract for that ЕИК — the read-time zero-contract gate -- • signed_at inside [first_declared_year, last_declared_year] — IN_WINDOW ⇒ the „в периода" chip -- • no published SELF stake on the same (official, ЕИК) for a family row — NOT_REDUNDANT_FAMILY +-- • a Trade Register evidence seal of a PUBLISHING rung — the #279 seal gate -- One link per outcome, including the non-surfaced ones, so a change to the publishing rule is -- visible as a before/after rather than as an empty page either way. @@ -95,3 +96,23 @@ VALUES INSERT OR IGNORE INTO interest_link_authorities (link_key, authority_id, authority_name, contract_count, value_eur, own) VALUES ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'auth-plovdiv', 'Община Пример', 1, 429000, 'exact'), ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'auth-sofia', 'Община София', 1, 190000, 'exact'); + +-- Trade Register evidence seals (#279, ADR-0033, migration 0006). NOT optional decoration: since the +-- evidence ladder landed, SURFACED_OWNERSHIP requires a publishing seal, so a seeded link without one +-- renders NOTHING — which is the exact failure this fixture exists to prevent. A seal per link, one per +-- rung, so the dev surface shows the same three outcomes as production: +-- document — the register names this person in this company (the strongest rung) +-- confirmed — identity confirmed by declared data (seat), no person found in the act itself +-- bar_joint_stock — an АД: a declared parcel of shares is not a material ownership conflict +-- matched_fact stays inside the closed vocabulary (evidence.mjs isSealedFact) — never a name. +INSERT OR IGNORE INTO interest_link_evidence + (link_key, evidence_kind, registry_role, matched_fact, entry_number, entry_date, lookup_date, rules_version, live_status) +VALUES + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'document', 'owner', 'role:owner:CR_F_19_L', + '20220314150210', '2022-03-14', '2026-08-05', 'tr-rules-1', 'live'), + ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'confirmed', NULL, 'seat:СОФИЯ', + '20210902110455', '2021-09-02', '2026-08-05', 'tr-rules-1', 'live'), + -- The held link is sealed too — seals exist for held links so the review queue can explain itself, + -- and this one doubles as the fixture's negative case for the seal gate. + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|204556676', 'bar_joint_stock', NULL, NULL, + NULL, NULL, '2026-08-05', 'tr-rules-1', 'live'); From 26333d841ba50967fa8051820100207480290f49 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 14:00:55 +0300 Subject: [PATCH 25/50] fix(tr): bound both the response size and the erasure regex's backtracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two halves of the same exposure, on a paced crawl where a wasted request budget cannot be re-spent cheaply. The request timeout bounded how long a response may STALL, but nothing bounded how much it may SEND: the reader buffered whatever arrived, so the endpoint decided how much memory this process held. Cap a body at 8 MB — ~240× the 34 KB a measured deed occupies, so it bounds abuse and can never refuse a large-but-legitimate company — and destroy the request rather than reject and leave the socket draining. Split out as collectBody so it is testable without TLS or a live socket. The erasure-notice strip used an unbounded lazy `.*?`, which backtracks quadratically when the opening div is never closed: 34K→3.3ms, 68K→13.6ms, 136K→53.8ms, 272K→240ms, 1M→4.0s, ×4 per doubling. Under an 8 MB cap alone that is still minutes of remote-controlled CPU per deed, so the cap is not on its own a fix. Bounding the lazy run to 2000 characters — ~80× a real one-sentence notice — makes it linear: the same inputs measure 5.3 / 10.3 / 24.8 / 41.6 / 187ms. The bound cannot resurrect an owner: `erased` is decided independently by ERASED_MARKER, so an over-long notice still marks the block erased and liveFields still drops it. --- scripts/tr/client.mjs | 60 ++++++++++++++++++++++++++++++++------ scripts/tr/client.test.mjs | 45 ++++++++++++++++++++++++++++ scripts/tr/deed.mjs | 20 ++++++++++--- scripts/tr/deed.test.mjs | 30 +++++++++++++++++++ 4 files changed, 142 insertions(+), 13 deletions(-) diff --git a/scripts/tr/client.mjs b/scripts/tr/client.mjs index 23d3769ec..473f5e309 100644 --- a/scripts/tr/client.mjs +++ b/scripts/tr/client.mjs @@ -70,19 +70,61 @@ const wait = (ms) => new Promise((r) => setTimeout(r, ms)); * module deliberately does NOT verify against system roots (it pins a leaf instead, ADR-0011), so a * reader comparing the two needs to see that this leg takes the ordinary, stronger path. */ -export function httpsGet(url, { timeoutMs = 20_000 } = {}) { +/** + * Ceiling on a single response body. A measured deed is ~34 KB, so 8 MB is ~240× the real thing — + * this bounds abuse, not the register, and can never refuse a large-but-legitimate company. + */ +export const MAX_BODY_BYTES = 8 * 1024 * 1024; + +/** + * Read one response into a Buffer, refusing past `maxBytes`. + * + * The timeout on the request bounds how long a response may STALL; nothing bounded how much it may + * SEND. Without a counter the crawler buffers whatever arrives, so a wedged or hostile endpoint decides + * how much memory this process holds. It also composes badly with the parser: the erasure regex in + * deed.mjs backtracks quadratically on unclosed markup (measured 34K→3.7ms, 68K→13.5ms, 136K→53.2ms, + * 272K→215.4ms — ×4 per doubling), and that path is reachable only through a body large enough to make + * it matter. One byte counter bounds both, and destroying the request is the load-bearing half: merely + * rejecting the promise would leave the socket draining the rest of the response. + * + * Separated from `httpsGet` so it is testable without TLS or a live socket — `res` needs only to be an + * emitter of 'data'/'end'/'error', which is the same injection posture as politeTrGet's `httpGet`. + */ +export function collectBody(res, { req, maxBytes = MAX_BODY_BYTES } = {}) { + return new Promise((resolve, reject) => { + const chunks = []; + let size = 0; + let done = false; + const fail = (err) => { + if (done) return; + done = true; + req?.destroy(err); // tear the socket down; do not sit and drain what we already refused + reject(err); + }; + res.on('data', (c) => { + if (done) return; + size += c.length; + if (size > maxBytes) { + fail(new Error(`response too large: over ${maxBytes} bytes — refusing to buffer it`)); + return; + } + chunks.push(c); + }); + res.on('end', () => { + if (done) return; + done = true; + resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) }); + }); + res.on('error', fail); + }); +} + +export function httpsGet(url, { timeoutMs = 20_000, maxBytes = MAX_BODY_BYTES } = {}) { return new Promise((resolve, reject) => { const req = https.get( url, { headers: { accept: 'application/json', 'user-agent': TR_USER_AGENT } }, - (res) => { - const chunks = []; - res.on('data', (c) => chunks.push(c)); - res.on('end', () => - resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) }), - ); - res.on('error', reject); - }, + (res) => collectBody(res, { req, maxBytes }).then(resolve, reject), ); req.setTimeout(timeoutMs, () => req.destroy(new Error(`timeout after ${timeoutMs}ms: ${url}`))); req.on('error', reject); diff --git a/scripts/tr/client.test.mjs b/scripts/tr/client.test.mjs index 6694076fc..201426027 100644 --- a/scripts/tr/client.test.mjs +++ b/scripts/tr/client.test.mjs @@ -6,6 +6,7 @@ // So a 429 is an instruction to stop, not a transient to retry through. ADR-0033 decision 7. import { test } from 'node:test'; import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; import { TR_HOST, TR_USER_AGENT, @@ -14,6 +15,8 @@ import { politeTrGet, assertTrHost, httpsGet, + collectBody, + MAX_BODY_BYTES, } from './client.mjs'; const okRes = (body = '{}') => ({ status: 200, headers: {}, body: Buffer.from(body) }); @@ -150,3 +153,45 @@ test('RateLimitError carries the url and is distinguishable from a generic failu assert.ok(err instanceof Error); assert.match(err.message, /115536179/); }); + +// ── response size ───────────────────────────────────────────────────────────── +// The timeout bounds how long a response may STALL; nothing bounded how large it may GROW. A deed is +// ~34 KB measured; an unbounded reader buffers whatever arrives, so a wedged or hostile endpoint could +// have the crawler hold an arbitrary amount of memory. It also composes badly with the parser: the +// erasure regex in deed.mjs backtracks quadratically on unclosed markup (measured 34K→3.7ms, +// 68K→13.5ms, 136K→53.2ms, 272K→215.4ms — ×4 per doubling), and that path is only reachable through a +// body large enough to make it matter. One byte counter bounds both. +test('a response past the cap is refused and the request destroyed, not buffered', async () => { + const res = fakeRes(200); + const destroyed = []; + const p = collectBody(res, { req: { destroy: (e) => destroyed.push(e) }, maxBytes: 1024 }); + res.emit('data', Buffer.alloc(700)); + res.emit('data', Buffer.alloc(700)); // 1400 > 1024 + await assert.rejects(p, /too large|1024/i); + assert.equal(destroyed.length, 1, 'the socket must be torn down, not left draining'); +}); + +test('a response under the cap still resolves with the WHOLE body', async () => { + const res = fakeRes(200); + const p = collectBody(res, { req: { destroy: () => {} }, maxBytes: 1024 }); + res.emit('data', Buffer.from('{"uic":')); + res.emit('data', Buffer.from('"115536179"}')); + res.emit('end'); + const out = await p; + assert.equal(out.status, 200); + assert.equal(out.body.toString(), '{"uic":"115536179"}'); +}); + +test('the cap leaves real deeds far under it — it bounds abuse, not the register', () => { + // A measured deed is ~34 KB. The cap must sit well above that or it becomes a correctness bug that + // silently refuses large-but-legitimate companies. + assert.ok(MAX_BODY_BYTES >= 1_000_000, `cap ${MAX_BODY_BYTES} is too tight for a real deed`); +}); + +// A stand-in for an http.IncomingMessage: an emitter carrying a status code. +function fakeRes(status) { + const e = new EventEmitter(); + e.statusCode = status; + e.headers = {}; + return e; +} diff --git a/scripts/tr/deed.mjs b/scripts/tr/deed.mjs index 71686ed8e..1bb4a9be6 100644 --- a/scripts/tr/deed.mjs +++ b/scripts/tr/deed.mjs @@ -63,6 +63,21 @@ function stripTags(html) { // and honouring both costs nothing while assuming one costs a wrong publish. const ERASED_MARKER = /erasure-text-inline|ui-icon-erased|field-text--erased/; +// The erasure notice, stripped so „Заличено обстоятелство." never reads as content. +// +// The `{0,2000}` bound is not cosmetic. With a plain `.*?` this backtracks QUADRATICALLY on markup where +// the opening div is never closed: every opening restarts a scan to end-of-input, measured at 34K→3.3ms, +// 68K→13.6ms, 136K→53.8ms, 272K→240ms, 1M→4.0s — ×4 per doubling. Bounding the lazy run makes each start +// position scan a fixed window instead, which is linear: the same inputs measure 5.3 / 10.3 / 24.8 / 41.6 +// / 187ms. A real notice is one short sentence, so 2000 characters is ~80× the live shape. +// +// Failing to match is safe by construction: `erased` is decided independently by ERASED_MARKER above, so +// an over-long notice still marks the block erased and liveFields still drops it. The only visible effect +// is that its text survives into the block — which in strict mode raises the drift alarm, loudly, rather +// than passing anything through silently. +const ERASURE_NOTICE = + /]*class=['"][^'"]*erasure-text-inline[^'"]*['"][^>]*>.{0,2000}?<\/div>/gis; + /** * Split one field's htmlData into the separate registered entities it holds. * @@ -84,10 +99,7 @@ export function entityBlocks(html, { strict = false } = {}) { const erased = ERASED_MARKER.test(chunk); // Read the visible text WITHOUT the erasure notice, so „Заличено обстоятелство." never counts as // content and an erased block reads as empty. - const withoutNotice = chunk.replace( - /]*class=['"][^'"]*erasure-text-inline[^'"]*['"][^>]*>.*?<\/div>/gis, - ' ', - ); + const withoutNotice = chunk.replace(ERASURE_NOTICE, ' '); const text = stripTags(withoutNotice); if (erased && text !== '' && strict) { throw new Error( diff --git a/scripts/tr/deed.test.mjs b/scripts/tr/deed.test.mjs index c45611c87..4a31459f2 100644 --- a/scripts/tr/deed.test.mjs +++ b/scripts/tr/deed.test.mjs @@ -336,3 +336,33 @@ test('latestOwnershipEntryDate is null when no live ownership field survives', ( const d = deedOf([field('CR_F_23_L', F23_ERASED, { fieldOperation: 2 })]); assert.equal(latestOwnershipEntryDate(d), null); }); + +// The erasure-notice strip used an unbounded lazy `.*?`, which backtracks quadratically when the opening +// div is never closed — each opening restarts a scan to end-of-input. Measured on that shape: 34K→3.3ms, +// 68K→13.6ms, 136K→53.8ms, 272K→240ms, 1M→4.0s (×4 per doubling). The parser runs against whatever the +// register returns, so that is remote-controlled CPU on a paced crawl with a per-request budget. +test('adversarial unclosed markup parses in linear time, not quadratically', () => { + // ~1 MB of unclosed erasure openings — the exact shape that triggers the backtracking. + const doc = '
z'.repeat(32_000); + const t = process.hrtime.bigint(); + entityBlocks(doc); + const ms = Number(process.hrtime.bigint() - t) / 1e6; + // Bounded measures ~190ms here and unbounded ~4000ms, so 1500ms separates them with ~8× headroom + // over the bounded path — wide enough not to flake on a loaded runner, tight enough to catch a + // reintroduced `.*?`. + assert.ok(ms < 1500, `entityBlocks took ${ms.toFixed(0)}ms on 1MB of unclosed markup`); +}); + +test('an over-long erasure notice still marks the block erased — the bound cannot leak a live owner', () => { + // If the notice exceeds the bound the regex simply does not strip it. `erased` is decided separately + // by ERASED_MARKER, so the block is still erased and liveFields still drops it: the failure mode of + // the bound is a noisier block, never a resurrected owner. + const long = `
${'Заличено. '.repeat(400)}
`; + const [block] = entityBlocks(long); + assert.equal(block.erased, true); + assert.deepEqual( + liveFields(deedOf([field('CR_F_19_L', long)]), ['CR_F_19_L']), + [], + 'an erased block contributes no live entity regardless of its notice length', + ); +}); From 436dbc230878823cc6108c7eec941b8c88cab368 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 14:02:53 +0300 Subject: [PATCH 26/50] fix(cacbg): re-derive the deed path, and pin the duplicated joint-stock rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit load.mjs joined the cache's stored raw_path straight onto the raw directory — the one read that did not go through the safeEik rail purgeExpired already uses. The cache index travels between runs, and once it does a stored path is attacker-influenced input on a filesystem root. Re-derive from the ЕИК through deedPath, which also throws on a malformed code rather than quietly reading another company's deed. JOINT_SUFFIX in the TR parser and JOINT_STOCK in the classifier are the same rule held in two places, because the TR parser cannot import out of scripts/cacbg/ without closing a cacbg↔tr cycle. That duplication has already drifted once — 5f64f5c added КДА to classify.mjs while deed.mjs's comment still asserted it was absent there, so the comment was simply false. Correct the comment and pin the two byte-identical, textually and behaviourally: an unenforced „keep these in step" note is how the drift happened in the first place. Verified by mutation — removing КДА from either side fails the test. --- scripts/cacbg/classify.mjs | 5 ++++- scripts/cacbg/load.mjs | 9 +++++++-- scripts/tr/deed.mjs | 11 +++++++++-- scripts/tr/deed.test.mjs | 23 +++++++++++++++++++++++ 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/scripts/cacbg/classify.mjs b/scripts/cacbg/classify.mjs index 8bc07bceb..a5f9de25f 100644 --- a/scripts/cacbg/classify.mjs +++ b/scripts/cacbg/classify.mjs @@ -77,7 +77,10 @@ const norm = (s) => // token bounded on the left by string edge, whitespace or quotes — NOT hyphens/dots, so „АД-ХОК ЕООД" (a // hyphenated ООД name) is not misread. Anchoring to the suffix is what stops „АД ГРУП ООД" (an ООД whose // NAME begins with the token „АД") being wrongly excluded as joint-stock — the form there is ООД. -const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; +// Twinned byte-for-byte by JOINT_SUFFIX in scripts/tr/deed.mjs — the TR parser cannot import from this +// directory without closing a cacbg↔tr cycle. A test in deed.test.mjs pins the two identical; change one +// and that test fails rather than the two silently diverging on a legal form. +export const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; /** * Materiality by legal form. The public ownership surface is CLOSELY-HELD companies only (ООД/ЕООД/ЕТ/ * КД/СД/ДЗЗД or a form-unspecified name from the closely-held table). Joint-stock forms (АД/ЕАД/АДСИЦ/КДА, diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index df5845dc9..7e0c46c1a 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -18,7 +18,7 @@ import { fileURLToPath } from 'node:url'; // path (ADR-0033). It survives in classify.mjs for the review queue, not for a publishing decision. import { temporalStatus, localityToken, closelyHeldForm } from './classify.mjs'; import { openCache, readDeed, coverage } from '../tr/cache.mjs'; -import { TR_DB, TR_RAW } from '../tr/paths.mjs'; +import { TR_DB, TR_RAW, deedPath } from '../tr/paths.mjs'; import { evidenceVerdict, isSealedFact, @@ -535,7 +535,12 @@ function deedFor(eik) { if (!row) entry = { deed: null, outsideTr: false, missing: true }; else if (row.status === 'outside_tr') entry = { deed: null, outsideTr: true, missing: false }; else { - const file = path.join(TR_RAW_DIR, row.rawPath ?? `${eik}.json`); + // RE-DERIVED from the ЕИК through safeEik, never the stored raw_path. The cache index is written by + // the crawler but travels between runs — and once it does, a stored path is attacker-influenced + // input joined straight onto a filesystem root, which is a traversal read. purgeExpired already + // re-derives for exactly this reason; this was the one read that did not. deedPath also throws on a + // malformed ЕИК rather than quietly reading some other company's deed (R8). + const file = deedPath(eik, TR_RAW_DIR); entry = fs.existsSync(file) ? { deed: JSON.parse(fs.readFileSync(file, 'utf8')), outsideTr: false, missing: false } : { deed: null, outsideTr: false, missing: true }; diff --git a/scripts/tr/deed.mjs b/scripts/tr/deed.mjs index 1bb4a9be6..13837d7ae 100644 --- a/scripts/tr/deed.mjs +++ b/scripts/tr/deed.mjs @@ -241,8 +241,15 @@ const FORM_CODES = new Map([ // The фирма's legal form is its SUFFIX under ЗТРРЮЛНЦ, and the deed envelope's `fullName` carries it // („ПИМК" ООД) while CR_F_2_L is bare („ПИМК"). So the bar has a second, independent signal at zero -// extra cost. КДА is included here and is absent from classify.mjs's JOINT_STOCK/FORM_TOKENS today. -const JOINT_SUFFIX = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; +// extra cost. +// +// DELIBERATELY a twin of classify.mjs's JOINT_STOCK rather than an import, for the same reason +// personTokens twins parse.mjs's holderTokens: the dependency direction here is cacbg → tr (load.mjs +// imports this module), so importing back out of scripts/cacbg/ would close a cycle across the two +// directories. The two are pinned byte-identical by a test in deed.test.mjs — this comment used to +// claim КДА was missing from classify.mjs, which stopped being true in 5f64f5c, and an unenforced +// „keep these in step" note is exactly how that happens. +export const JOINT_SUFFIX = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; const CLOSELY_SUFFIX = /(?:^|[\s"„“”«»])(ООД|ЕООД|ЕТ|ДЗЗД|КД|СД|КООПЕРАЦИЯ)[\s"„“”«»]*$/u; /** diff --git a/scripts/tr/deed.test.mjs b/scripts/tr/deed.test.mjs index 4a31459f2..9c539c874 100644 --- a/scripts/tr/deed.test.mjs +++ b/scripts/tr/deed.test.mjs @@ -20,6 +20,7 @@ import { registryLegalForm, latestOwnershipEntryDate, assertUicEcho, + JOINT_SUFFIX, } from './deed.mjs'; // ── real markup ─────────────────────────────────────────────────────────────── @@ -366,3 +367,25 @@ test('an over-long erasure notice still marks the block erased — the bound can 'an erased block contributes no live entity regardless of its notice length', ); }); + +// JOINT_SUFFIX here and JOINT_STOCK in scripts/cacbg/classify.mjs are the SAME rule — which legal-form +// suffixes mark a share-issuing company — held in two places because the TR parser cannot import out of +// scripts/cacbg/ without closing a cacbg↔tr cycle. The drift this risks has already happened once: 5f64f5c +// added КДА to classify.mjs while deed.mjs's comment still asserted it was absent there. A prose „keep +// these in step" note does not keep anything in step; this does. +test('the joint-stock suffix rule is identical in the TR parser and the classifier', async () => { + const { JOINT_STOCK } = await import('../cacbg/classify.mjs'); + assert.equal(JOINT_SUFFIX.source, JOINT_STOCK.source, 'the two patterns have diverged'); + assert.equal(JOINT_SUFFIX.flags, JOINT_STOCK.flags, 'the two patterns have diverged in flags'); + // Behavioural pin as well as textual: identical sources with different behaviour is impossible, but a + // future refactor could legitimately change BOTH sources while breaking one. These are the forms the + // bar exists for — every one must be caught by both, or a joint-stock parcel publishes as ownership. + for (const name of ['ТРЕЙС ГРУП ХОЛД АД', 'НЕЩО ЕАД', 'ФОНД АДСИЦ', 'НЕЩО КДА']) { + assert.equal(JOINT_SUFFIX.test(name), true, name); + assert.equal(JOINT_STOCK.test(name), true, name); + } + for (const name of ['АЛФА СТРОЙ ООД', 'БЕТА ЕООД', 'АД ГРУП ООД']) { + assert.equal(JOINT_SUFFIX.test(name), false, name); + assert.equal(JOINT_STOCK.test(name), false, name); + } +}); From 024c26f0e791aff0161c84cbc10a9abf76a4bc4c Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 14:07:54 +0300 Subject: [PATCH 27/50] feat(cacbg): add --emit-candidates so one job can bootstrap its own crawl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The register crawl and the decision run have to share a job: the raw deeds hold third-party names and cannot travel between runners. But the crawl's input list is derived from the resolved corpus, so only load.mjs can produce it — and load.mjs refuses to run without the very cache that list is used to fill. --emit-candidates writes the list and exits 0. Deriving it by running the full load and ignoring a non-zero exit was the alternative, and it erases the difference between „no cache yet" and „this run is broken". It runs on a throwaway copy of the work DB, and that is correctness rather than tidiness. Reaching the candidate list means rebuilding the corpus tables, which drops interest_links, and the pass never publishes — so against the real DB it would leave the table empty. The next real run would then export an EMPTY prior-published set as its monotonicity snapshot, and the gate whose only job is to notice a published claim disappearing would pass unconditionally from then on. Verified by mutation: pointing the pass at the real DB fails both tests. --- scripts/cacbg/load.mjs | 55 ++++++++++++++++++++--- scripts/cacbg/load.test.mjs | 87 +++++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 5 deletions(-) diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 7e0c46c1a..bcceeb882 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -50,6 +50,19 @@ const TR_RAW_DIR = process.env.TR_RAW_DIR || TR_RAW; // A deliberate, logged override for the coverage gate below. Without it a single permanently // unreachable ЕИК would deadlock the pipeline forever; with it, the operator states that they know. const ALLOW_PARTIAL_TR = process.argv.includes('--allow-partial-tr'); +// Bootstrap mode: write the crawl's input list and stop, successfully. The decision run and the register +// crawl now share one job (they must — the raw deeds hold third-party names and cannot travel between +// runners), and that job has to be able to start from nothing: the list is derived from the resolved +// corpus, so only this script can produce it, but the full run refuses without the very cache the list +// is used to fill. Ignoring the refusal's exit code instead would erase the difference between „no cache +// yet" and „this run is broken". +// +// POINT THIS AT A SCRATCH COPY OF THE WORK DB. It is not a read-only pass: reaching the candidate list +// means rebuilding the corpus tables, so it drops and repopulates persons/declarations/declared_interests +// and leaves interest_links EMPTY. Empty is the safe end state (the ship floor refuses it, and no link +// can be published without evidence it never gathered), but it is not the state a subsequent real run +// should inherit. The one thing it must never touch either way is the monotonicity snapshot — see below. +const EMIT_CANDIDATES_ONLY = process.argv.includes('--emit-candidates'); const { companyNameKey, isMatchableKey } = await import('../../packages/shared/src/company-name-key.ts'); @@ -73,7 +86,24 @@ const readJsonl = (f) => .map((l) => JSON.parse(l)) : []; -const db = new DatabaseSync(DB); +// Bootstrap mode works on a THROWAWAY COPY, and that is not a convenience — it is the correctness of the +// monotonicity gate. Reaching the candidate list means rebuilding the corpus tables, which drops +// interest_links; the pass itself never publishes, so it would leave the table EMPTY. The next real run +// would then read that empty table as its prior-published set, write an empty snapshot, and the gate — +// whose only job is to notice a published claim disappearing — would pass unconditionally, for ever. +// Copying here rather than asking the caller to do it keeps the flag safe wherever it is invoked from. +const WORK_DB = EMIT_CANDIDATES_ONLY ? `${DB}.bootstrap` : DB; +if (EMIT_CANDIDATES_ONLY) { + for (const suffix of ['', '-wal', '-shm']) { + // -wal/-shm may legitimately be absent (a cleanly closed DB has neither); anything else must surface. + try { + fs.copyFileSync(`${DB}${suffix}`, `${WORK_DB}${suffix}`); + } catch (e) { + if (e.code !== 'ENOENT') throw e; + } + } +} +const db = new DatabaseSync(WORK_DB); db.exec('PRAGMA foreign_keys=ON'); // Suppressions live in a VERSION-CONTROLLED, HMAC-fingerprinted list (ADR-0031), NOT a DB table — so a // takedown survives a fresh-CI-runner rebuild and never ships the „who was taken down" signal to prod. @@ -115,10 +145,13 @@ const priorPublished = (() => { return []; } })(); -fs.writeFileSync( - path.join(STAGING, 'published-snapshot.json'), - JSON.stringify(priorPublished, null, 2) + '\n', -); +// Not written by the bootstrap pass, which works on a throwaway copy and has no business restating what +// the real run is about to record. +if (!EMIT_CANDIDATES_ONLY) + fs.writeFileSync( + path.join(STAGING, 'published-snapshot.json'), + JSON.stringify(priorPublished, null, 2) + '\n', + ); // Full idempotent rebuild that also picks up schema changes: drop the CACBG tables (children first — // FK-safe) and re-apply the migration. Nothing to preserve — suppressions are external now. @@ -491,6 +524,18 @@ function authOwn(authorityName, instNorms, instNormsLong, locTokens) { const candidateEiks = [...new Set([...agg.values()].map((r) => r.eik))].sort(); fs.writeFileSync(path.join(STAGING, 'candidate-eiks.txt'), candidateEiks.join('\n') + '\n'); +if (EMIT_CANDIDATES_ONLY) { + // Stop BEFORE the TR gate and before anything is written to the domain. A bootstrap pass that built + // links would leave a surface resting on no evidence at all, and a failure between this pass and the + // real one would leave that surface sitting in the work DB, shippable. + console.log( + `${candidateEiks.length} candidate ЕИК written for the crawler; stopping (--emit-candidates)`, + ); + db.close(); + for (const suffix of ['', '-wal', '-shm']) fs.rmSync(`${WORK_DB}${suffix}`, { force: true }); // the copy has served its purpose + process.exit(0); +} + if (!fs.existsSync(TR_CACHE_DB)) { db.close(); throw new Error( diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index bd5e5bc8c..b2a9dbe7f 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -1220,3 +1220,90 @@ test('the published surface is exported BEFORE the wipe, so the audit can gate m // next run is not a regression to gate on. assert.equal(snap.length, published.length); }); + +// The crawl input and the crawl's consumer are the same script, which used to make the two workflows +// deadlock: the decision run refused without a deed cache, the refresh run refused without a candidate +// list, and each produced only what the other needed. Merged into one job, the job still has to be able +// to BOOTSTRAP — produce the candidate list on a runner that has no cache yet — and it cannot do that by +// running the full load and ignoring a non-zero exit, because that exit is also how a genuinely broken +// run reports itself. Hence an explicit mode that stops at the list and succeeds. +test('--emit-candidates writes the crawl list and exits 0 with NO Trade Register cache', () => { + const gone = path.join(dir, 'bootstrap-absent.sqlite'); + const listFile = path.join(STAGING, 'candidate-eiks.txt'); + fs.rmSync(listFile, { force: true }); + assert.doesNotThrow(() => + execFileSync( + 'node', + [ + '--import', + path.join(HERE, 'register-ts.mjs'), + path.join(HERE, 'load.mjs'), + '--emit-candidates', + ], + { + cwd: ROOT, + env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING, TR_CACHE_DB: gone }, + stdio: 'pipe', + }, + ), + ); + const listed = fs.readFileSync(listFile, 'utf8').split('\n').filter(Boolean); + assert.ok(listed.length > 0, 'the bootstrap must actually produce candidates'); + assert.equal(new Set(listed).size, listed.length); + runLoad(); // restore the full built state for any later reader +}); + +function emitCandidates(trCacheDb) { + execFileSync( + 'node', + [ + '--import', + path.join(HERE, 'register-ts.mjs'), + path.join(HERE, 'load.mjs'), + '--emit-candidates', + ], + { + cwd: ROOT, + env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING, TR_CACHE_DB: trCacheDb }, + stdio: 'pipe', + }, + ); +} + +test('a bootstrap pass leaves the REAL work DB untouched — it runs on a throwaway copy', () => { + // Reaching the candidate list means rebuilding the corpus tables, and the pass never publishes, so + // against the real DB it would leave interest_links empty. That is the damage: not the pass itself, + // but what the NEXT run then reads. + runLoad(); + const db = open(); + const before = db.prepare('SELECT COUNT(*) AS n FROM interest_links').get().n; + db.close(); + assert.ok(before > 0, 'the fixture must actually publish something'); + + emitCandidates(path.join(dir, 'bootstrap-absent-2.sqlite')); + + const after = open(); + assert.equal( + after.prepare('SELECT COUNT(*) AS n FROM interest_links').get().n, + before, + 'the bootstrap pass must not empty the domain it was pointed at', + ); + after.close(); + assert.equal(fs.existsSync(`${DB}.bootstrap`), false, 'the throwaway copy must be cleaned up'); +}); + +test('the monotonicity gate still sees a prior surface AFTER a bootstrap pass', () => { + // The end-to-end shape of the merged workflow: real run → bootstrap (to produce the crawl list) → + // real run. If the bootstrap emptied interest_links, the second real run would export an EMPTY + // prior-published set, and the gate — whose only job is to notice a published claim disappearing — + // would pass unconditionally, for ever. + const snapshot = path.join(STAGING, 'published-snapshot.json'); + runLoad(); + emitCandidates(path.join(dir, 'bootstrap-absent-3.sqlite')); + runLoad(); + const prior = JSON.parse(fs.readFileSync(snapshot, 'utf8')); + assert.ok( + prior.length > 0, + 'the snapshot went empty — the gate is now vacuous and would never fire again', + ); +}); From ae661ac3838b67459db7023302257269094e1e9b Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 14:10:17 +0300 Subject: [PATCH 28/50] ci(related-persons): run the register crawl inside the decision job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two workflows deadlocked and both were permanently green no-ops. The crawl job gated on scratch/cacbg/staging/candidate-eiks.txt, which only the decision job writes and nothing persisted, so its check was false on every run forever. The decision job restored only scratch/cacbg/raw, never scratch/tr, so it hit `REFUSE TO LOAD: no Trade Register cache` on every run. Each waited on the other half of a handoff neither performed. They cannot be joined by persisting the deeds. The crawl's `path: scratch/tr` already cached deeds/*.json — the co-owner and manager names ADR-0033 decision 5 promises a 35-day retention for. purgeExpired runs on the runner, but older cache entries live on GitHub's storage under the restore-keys chain, where eviction is a capacity policy and not retention at all. So the deeds stay on one runner: crawl and decide in one job, and delete them explicitly at the end. The cadence follows from that. A decision needs the raw deeds — evidenceVerdict's strongest rung matches the declarant's name against the register's own text — so every run that decides must also crawl, and a daily decision would mean ~400 daily requests against somebody else's register. The schedule is therefore monthly, not daily as §9 and ADR-0033 describe. Restoring a daily decision needs the crawl to emit per-(link, ЕИК) verdicts so only booleans cross a run boundary; that is a design change and is deliberately not folded in here. Also export interest_links + interest_link_evidence in the hydrate step. Not for the resolver — load.mjs rebuilds them — but for the monotonicity gate: the work DB had no interest_links at all, so the pre-wipe export read "no such table", the snapshot was [] every run, and the gate added in 2327e62 protected nothing. --- .github/workflows/related-persons-data.yml | 111 ++++++++++++++-- .../workflows/related-persons-tr-refresh.yml | 119 ------------------ 2 files changed, 104 insertions(+), 126 deletions(-) delete mode 100644 .github/workflows/related-persons-tr-refresh.yml diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index c09bed4d9..7058fd320 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -16,19 +16,38 @@ name: Related-persons data foundation (build + ship) # XML (fetch.mjs skips files already on disk) — it fills gaps, not a from-scratch rebuild unless the cache is cleared. on: - # DAILY, and deliberately on the EXISTING workflow rather than a second one. #279 §9 asks for the + # MONTHLY, and deliberately on the EXISTING workflow rather than a second one. #279 §9 asks for the # publishing decisions to run on a cadence; duplicating the job would create a second ship path with # its own copy of the credential guards, the D1-target guard and the ship floor — and the copy is the - # one nobody exercises. A scheduled run takes the same path as a manual one, with two differences: + # one nobody exercises. + # + # WHY MONTHLY AND NOT DAILY, which is what §9 and ADR-0033 describe. A decision cannot be recomputed + # without the raw deeds — evidenceVerdict's strongest rung matches the declarant's name against the + # register's own text — and the raw deeds must not be persisted between runs: they carry the names and + # addresses of co-owners and managers who hold no public office, and an Actions cache entry lives on + # GitHub's storage under a restore-keys chain where ADR-0033 decision 5's 35-day retention cannot reach + # it. So the deeds live and die with one runner, and every run that decides must also crawl. A daily + # decision would therefore mean ~400 daily requests against somebody else's register, which is what + # spec §3.3 exists to prevent — so the cadence follows the deeds, not the other way round. + # + # Restoring a daily decision needs the crawl to emit the per-(link, ЕИК) match verdict, so only + # booleans cross a run boundary and nothing has to hold a name. That is a design change, not a + # schedule change, and it is deliberately not folded in here. + # + # A scheduled run takes the same path as a manual one, with two differences: # • it targets STAGING (the environment default below). Production stays manual, so the # assertD1TargetAuthorized guard keeps a prod write a deliberate act. - # • `full_crawl` is empty and therefore false, so it never crawls the register — it re-runs the + # • `full_crawl` is empty and therefore false, so it never crawls the CACBG corpus — it re-runs the # decisions over the cached corpus, which is exactly what #279 §9 means by a pure, zero-network - # decision. Registry lookups have their own workflow (related-persons-tr-refresh.yml). + # decision. Registry lookups are the one exception and run from inside this job on the 1st of the + # month (see „Decide whether to hit the register"): they cannot be a separate workflow, because the + # raw deeds they fetch carry third-party names and must not be persisted between runners. # If the corpus cache has been evicted, extract yields nothing, the ship floor refuses, and the run # fails loudly rather than wiping the surface. schedule: - - cron: '0 3 * * *' + # 03:00 UTC on the 1st. See the note above: the decision and the crawl are one pass, so the decision + # cadence is the crawl cadence — #279 §9's monthly registry rhythm. + - cron: '0 3 1 * *' workflow_dispatch: inputs: environment: @@ -40,6 +59,14 @@ on: description: Crawl the CACBG corpus (slow). RESUMES from the cached raw XML — fetch.mjs skips files already on disk, so this fills gaps rather than re-fetching from scratch (clear the cache for a true rebuild). Off = extract from the existing raw cache without crawling. type: boolean default: false + tr_max_age_days: + description: Re-fetch deeds older than this many days (default 30). Use a large value to force a full refresh. + type: string + default: '30' + tr_limit: + description: Stop after this many registry lookups (blank = all pending). Bound a first run while watching for a 429. + type: string + default: '' # Least-privilege: this workflow only reads the repo; it authenticates to Cloudflare with a scoped token. permissions: @@ -54,8 +81,9 @@ jobs: build-and-ship: runs-on: ubuntu-latest # A from-scratch corpus crawl of the whole register is the long pole (all year-folders, politely - # throttled) and must fit in ONE job with extract/resolve/audit/ship after it — 120 was too tight and - # timed out mid-crawl. 300 leaves headroom under GitHub's 6-hour hard cap. A run past this is wedged + # throttled) and must fit in ONE job with the registry lookups, extract/resolve/audit/ship after it — + # 120 was too tight and timed out mid-crawl. The Търговски регистър pass adds ~20 minutes on top + # (~400 candidates at 1 request / 3 s). 300 leaves headroom under GitHub's 6-hour hard cap. A run past this is wedged # (stalled gov-server I/O) — fail rather than burn the slot. The raw corpus is cached (below) so a # re-run resumes instead of re-crawling from scratch. timeout-minutes: 300 @@ -164,11 +192,29 @@ jobs: work="$GITHUB_WORKSPACE/data/work" mkdir -p "$work" rm -f "$work/eop.sql" "$work/backfill.sqlite" + # interest_links + interest_link_evidence come along too, and NOT for the resolver's benefit — + # load.mjs drops and rebuilds them. They are the input to the monotonicity gate: load.mjs + # exports the currently-published set BEFORE the wipe, and audit.mjs fails the run on a link + # that was published last time, under an unchanged rules_version, and is not published now. + # Without them the work DB starts with no interest_links at all, the pre-wipe export reads + # „no such table", the snapshot is [] on every run, and the gate protects nothing. + # They are absent on a first run (0003/0006 not yet applied), which is why they are exported + # separately and tolerated as missing rather than folded into the parity loop below. pnpm --filter @sigma/web exec wrangler d1 export "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote -y \ --table bidders --table contracts --table tenders --table authorities \ --output "$work/eop.sql" sqlite3 "$work/backfill.sqlite" < "$work/eop.sql" + if pnpm --filter @sigma/web exec wrangler d1 export "$SIGMA_D1_NAME" \ + --config wrangler.deploy.jsonc --remote -y \ + --table interest_links --table interest_link_evidence \ + --output "$work/prior-links.sql"; then + sqlite3 "$work/backfill.sqlite" < "$work/prior-links.sql" + echo " prior surface: $(sqlite3 "$work/backfill.sqlite" \ + "SELECT COUNT(*) FROM interest_links WHERE status='published';") published link(s)" + else + echo "::notice::No prior interest_links on this D1 (first run) — the monotonicity gate has no baseline yet." + fi for t in bidders contracts tenders authorities; do local_n=$(sqlite3 "$work/backfill.sqlite" "SELECT COUNT(*) FROM $t;") remote_json=$(pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ @@ -182,9 +228,60 @@ jobs: fi done + # ── Trade Register lookups ──────────────────────────────────────────────────────────────────── + # In THIS job, deliberately, and not in a workflow of their own. The decision run needs the raw + # deeds (evidenceVerdict matches declarant names against the register's own text, which is exactly + # the data the index refuses to store), and those deeds carry the names and addresses of co-owners + # and managers who hold no public office. Handing them between runners means persisting them — + # an Actions cache entry survives on GitHub's storage under a restore-keys chain, where the 35-day + # retention ADR-0033 decision 5 promises cannot reach it. Same runner, same job, deleted with it. + # + # The previous split also could not work on its own terms: the crawl job gated on a candidate list + # only the decision job produced, the decision job refused without a cache only the crawl job + # produced, and neither persisted its half. Both were permanently green no-ops. + - name: Emit the candidate ЕИК list for the crawler + run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/load.mjs --emit-candidates + + # ALWAYS, on every run that reaches here — not conditionally. The deeds do not survive the runner, + # so „decide without crawling" is not a state this job can be in: load.mjs's coverage gate below + # would refuse the empty cache and the run would fail. Skipping the crawl is only meaningful once + # the crawl emits verdicts rather than deeds (see the note on the schedule above). + # + # Pace and refusals live in scripts/tr/fetch-deeds.mjs and are not configurable here on purpose: + # 1 request / 3 s, sequential, a closed candidate set, and a 429 ends the run without marking + # anything. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the limiter is + # the operator's only way to state a rate preference, so we do not tune around it. + - name: Refresh the Trade Register deed cache + env: + MAX_AGE: ${{ inputs.tr_max_age_days || '30' }} + LIMIT: ${{ inputs.tr_limit }} + run: | + set -euo pipefail + ARGS=(--eiks-file scratch/cacbg/staging/candidate-eiks.txt --max-age-days "$MAX_AGE") + [ -n "$LIMIT" ] && ARGS+=(--limit "$LIMIT") + set +e + node scripts/tr/fetch-deeds.mjs "${ARGS[@]}" + code=$? + set -e + # Exit 2 is the rate limiter, and it is NOT a build failure: the run stopped politely, marked + # nothing, and the partial cache is valid. It is not a licence to publish either — load.mjs's + # coverage gate below refuses a partial cache on its own, which is where that decision belongs. + if [ "$code" -eq 2 ]; then + echo "::warning::The register rate-limited us; the crawl stopped and the cache is resumable." + exit 0 + fi + exit "$code" + - name: Resolve → build domain in a work sqlite (libel gate must pass) run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/load.mjs + # The raw deeds are third-party personal data and exist only to produce booleans. They are deleted + # with the runner regardless; deleting them explicitly means a failure AFTER this point cannot + # leave them sitting in a job artifact or a debug upload someone adds later. + - name: Delete the raw deeds + if: always() + run: rm -rf scratch/tr/deeds + - name: Audit published links (fails the run on any libel-invariant violation) run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/audit.mjs diff --git a/.github/workflows/related-persons-tr-refresh.yml b/.github/workflows/related-persons-tr-refresh.yml deleted file mode 100644 index 9cc09add0..000000000 --- a/.github/workflows/related-persons-tr-refresh.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: Related-persons — Trade Register cache refresh (monthly) - -# Refreshes the Търговски регистър deed cache the publishing decision rests on (#279, ADR-0033). -# -# WHY THIS IS ITS OWN WORKFLOW, separate from the decision run: this is the only step in the project -# that touches a public register at volume, and it is the only one with a network budget. The decisions -# themselves are a pure function of declarations, cached deeds and contracts — they must be able to run -# daily without ever calling the register. Splitting them keeps that true by construction. -# -# It SHIPS NOTHING. It writes only the cache under scratch/tr/, which is git-ignored and never leaves -# the runner except as an artifact. A run that fails leaves the previous cache intact. -# -# It also PURGES: raw deeds past the 35-day retention (ADR-0033 decision 5) are deleted at the end of -# every run, including one a 429 cut short. Retention is not the same knob as freshness — --max-age-days -# decides what gets re-requested, --retention-days decides what stops being kept — and the default is -# the ADR's 35 days, so the rail holds even when the workflow passes neither. -# -# Pace and refusals live in scripts/tr/fetch-deeds.mjs and are not configurable here on purpose: -# 1 request / 3 s, sequential, a closed candidate set, and a 429 ends the run (exit 2) without marking -# anything. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the limiter is the -# operator's only way to state a rate preference, so we do not tune around it. - -on: - schedule: - # 02:00 UTC on the 1st. Monthly, per #279 §9 — a deed changes rarely, and every extra pass is - # ~400 requests against somebody else's service for almost no new information. - - cron: '0 2 1 * *' - workflow_dispatch: - inputs: - max_age_days: - description: Re-fetch deeds older than this many days (default 30). Use a large value to force a full refresh. - type: string - default: '30' - limit: - description: Stop after this many lookups (blank = all pending). Bound a first run while watching for a 429. - type: string - default: '' - -concurrency: - # Never two crawls at once: they would double the request rate against the register, which is the one - # thing the pacing exists to prevent. Queue rather than cancel — a cancelled crawl wastes its progress. - group: tr-cache-refresh - cancel-in-progress: false - -permissions: - contents: read - -jobs: - refresh: - runs-on: ubuntu-latest - # ~400 candidates at 1 per 3 s is ~20 minutes; the ceiling allows retries and a slower day without - # letting a hung run sit for the 6-hour default. - timeout-minutes: 60 - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: 24 - cache: pnpm - - # The cache is the whole point of the workflow: without restoring it, every run would re-fetch - # every deed and turn a 400-request refresh into a 400-request full crawl each month. - - name: Restore the deed cache - id: cache - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.2 - with: - path: scratch/tr - key: tr-deed-cache-${{ github.run_id }} - restore-keys: | - tr-deed-cache- - - - run: pnpm install --frozen-lockfile - - # The closed candidate set. It is produced by the decision run (load.mjs writes it), so a cache - # refresh can never invent an ЕИК to look up — it can only refresh what the corpus already - # resolved. Absent on a first run, in which case there is nothing to refresh yet. - - name: Check for a candidate list - id: candidates - run: | - if [ -s scratch/cacbg/staging/candidate-eiks.txt ]; then - echo "count=$(wc -l < scratch/cacbg/staging/candidate-eiks.txt)" >> "$GITHUB_OUTPUT" - echo "ok=true" >> "$GITHUB_OUTPUT" - else - echo "No candidate-eiks.txt — run the decision workflow first. Nothing to refresh." - echo "ok=false" >> "$GITHUB_OUTPUT" - fi - - - name: Refresh deeds - if: steps.candidates.outputs.ok == 'true' - env: - MAX_AGE: ${{ inputs.max_age_days || '30' }} - LIMIT: ${{ inputs.limit }} - run: | - set -euo pipefail - ARGS=(--eiks-file scratch/cacbg/staging/candidate-eiks.txt --max-age-days "$MAX_AGE") - [ -n "$LIMIT" ] && ARGS+=(--limit "$LIMIT") - set +e - node scripts/tr/fetch-deeds.mjs "${ARGS[@]}" - code=$? - set -e - # Exit 2 is the rate limiter, and it is NOT a build failure: the run stopped politely, marked - # nothing, and the partial cache is valid and resumable. Failing the workflow here would train - # everyone to ignore a red X on the one signal that means "the register asked us to stop". - if [ "$code" -eq 2 ]; then - echo "::warning::The register rate-limited us; the run stopped and the cache is resumable." - exit 0 - fi - exit "$code" - - - name: Publish the cache index as an artifact (no raw deeds — they carry third-party data) - if: always() && steps.candidates.outputs.ok == 'true' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: tr-cache-index - # The SQLite index only. scratch/tr/deeds/*.json holds owner and manager names and company - # addresses (ADR-0033 decision 5) and must not leave the runner. - path: scratch/tr/tr-cache.sqlite - retention-days: 7 From 9555fbfd066127b35a152280529c7780edf6413b Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 7 Aug 2026 14:14:08 +0300 Subject: [PATCH 29/50] test(db): type the seal-gate fixture arrays explicitly `as const` narrowed the WITHHELD tuples to literal unions, so appending the no-seal and positive-control rows failed typecheck. --- packages/db/src/related-persons-sql.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index 380949a0a..512da590e 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -551,9 +551,12 @@ describe('свързани-лица SQL (real SQLite)', () => { ] as const; function seedRungs(dbPath: string): void { - const people = [...WITHHELD.map(([, id, name]) => [id, name] as const)]; - people.push(['person:none', 'Безпечатен Тестов'], ['person:ok', 'Потвърден Тестов']); - const links = [...WITHHELD.map(([, id]) => id), 'person:none', 'person:ok']; + const people: [string, string][] = [ + ...WITHHELD.map(([, id, name]) => [id, name] as [string, string]), + ['person:none', 'Безпечатен Тестов'], + ['person:ok', 'Потвърден Тестов'], + ]; + const links: string[] = [...WITHHELD.map(([, id]) => id), 'person:none', 'person:ok']; sqlite( dbPath, `INSERT INTO persons (id, name) VALUES ${people.map(([id, n]) => `('${id}','${n}')`).join(',')}; From bf4fa9769972f822f0bd0262b81a716568bd3b3b Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Tue, 11 Aug 2026 17:26:06 +0300 Subject: [PATCH 30/50] fix(tr): refuse a seat confirmation when the declared period is unknown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The R10 temporal guard read `firstDeclaredYear == null` as "covers the period" and short-circuited the whole disjunction. `load.mjs` passes null whenever no history row carried a parseable year, so every such link reached the seat rung with no temporal check at all — publishing „Потвърдено" on a settlement match alone, which is precisely the "company relocated INTO the declared settlement afterwards" case the guard exists to block. That made the weakest rung the only one without a temporal test, on exactly the links where we know least. Rung 4's refutation leg right below already refuses to run without a year. An unknown year now FAILS the guard instead of skipping it. The undated-seat leg stays: a seat with no entry date is the ordinary shape for a company that never moved, and it is still checkable because a known year sits on the other side. Three tests: both null-year shapes (dated and undated seat) held, plus a positive control proving a known year with an undated seat still confirms — the fix is a bound, not a blanket. The methodology page asserted this rule already; it now states the precondition it relied on. --- apps/web/app/routes/conflict.methodology.tsx | 6 ++- scripts/tr/evidence.mjs | 13 +++-- scripts/tr/evidence.test.mjs | 53 ++++++++++++++++++++ 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/apps/web/app/routes/conflict.methodology.tsx b/apps/web/app/routes/conflict.methodology.tsx index c9a432b06..b1fc692c8 100644 --- a/apps/web/app/routes/conflict.methodology.tsx +++ b/apps/web/app/routes/conflict.methodology.tsx @@ -231,8 +231,10 @@ export default function ConflictMethodology() { изисква пълно съвпадение и на трите имена в{' '} един и същ запис на акта (две от три не се приема; имена с инициали или с латински букви не се приемат). Седалищата се променят, затова регистрираното - седалище се приема само ако е вписано преди декларирания период. Заличените записи - се пропускат навсякъде. + седалище се приема само ако е вписано преди декларирания период —{' '} + и само когато този период е известен. Ако от декларациите не може да + се извлече година, проверката няма с какво да се направи и връзката остава скрита. + Заличените записи се пропускат навсякъде.

Справките се опресняват месечно, а датата на справката и номерът на diff --git a/scripts/tr/evidence.mjs b/scripts/tr/evidence.mjs index 01fce0e89..51c4c7708 100644 --- a/scripts/tr/evidence.mjs +++ b/scripts/tr/evidence.mjs @@ -197,10 +197,17 @@ export function evidenceVerdict(input) { const declared = declaredSeats.map(normalizeSettlement).filter((s) => s !== ''); // R10: the registered seat must have been in force for the declared period. Seats move, and a // company that relocated INTO the declared settlement afterwards would otherwise confirm falsely. + // + // A null `firstDeclaredYear` FAILS the guard rather than skipping it. `load.mjs` passes null + // whenever no history row carried a parseable year, and an unknown year is not a satisfied + // temporal test — it is the absence of one. Reading it as „covers the period" made the weakest + // rung the only one with no temporal check, on exactly the links where we know least, and rung 4 + // below already refuses to run without a year (`firstDeclaredYear != null`) on the same ground. + // The undated-seat leg is different and stays: a seat with no entry date is the ordinary shape + // for a company that never moved, and it is checkable — a known year is still on the other side. const seatCoversPeriod = - firstDeclaredYear == null || - seat.entryDate == null || - seat.entryDate <= `${firstDeclaredYear}-12-31`; + firstDeclaredYear != null && + (seat.entryDate == null || seat.entryDate <= `${firstDeclaredYear}-12-31`); if (seatCoversPeriod && declared.includes(seat.settlement)) { return verdict('confirmed', true, { matchedFact: `seat:${seat.settlement}`, diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs index 858db9194..4de42a931 100644 --- a/scripts/tr/evidence.test.mjs +++ b/scripts/tr/evidence.test.mjs @@ -162,6 +162,59 @@ test('rung 3 — a seat registered AFTER the declared period does not confirm', assert.notEqual(v.kind, 'confirmed'); }); +test('rung 3 — an UNKNOWN first declared year cannot confirm on a seat', () => { + // R10 again, from the other side. `load.mjs` passes `firstDeclaredYear: null` whenever no history row + // carried a parseable year, and a null year means the temporal check has NOTHING to compare against — + // not that the seat covers the period. The same relocated company as the test above, with the year + // unknown instead of 2021, must reach the same held outcome: an unknown guard is a failed guard. + const moved = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив'), '2024-06-01T00:00:00'), + ]); + const v = evidenceVerdict({ + ...base, + deed: moved, + declaredSeats: ['Пловдив'], + firstDeclaredYear: null, + }); + assert.notEqual(v.kind, 'confirmed'); +}); + +test('rung 3 — an unknown year holds a seat match even when the seat has NO entry date', () => { + // The nastier half: with no entry date on the register side AND no year on the declaration side there + // are two unknowns and zero evidence about the period, yet both legs of the old disjunction read TRUE. + // Rung 4's refutation leg already refuses to run without a year (`firstDeclaredYear != null`); the seat + // leg must refuse on the same ground, or the weakest rung is the one with no temporal check at all. + const undated = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: undated, + declaredSeats: ['Пловдив'], + firstDeclaredYear: null, + }); + assert.notEqual(v.kind, 'confirmed', 'two unknowns must not multiply into a public claim'); +}); + +test('rung 3 — a KNOWN year with an undated seat still confirms (the guard is not a blanket)', () => { + // Positive control. Bounding the null case must not quietly kill the rung: a seat with no entry date + // is the ordinary shape for a company that never moved, and it still confirms under a known year. + const undated = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: undated, + declaredSeats: ['Пловдив'], + firstDeclaredYear: 2021, + }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'seat:ПЛОВДИВ'); +}); + test('rung 3 — the weakest rung ALSO requires global name uniqueness (ADR-0017 carried forward)', () => { const other = deed([ fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), From 466d002a4c82b8086a098078061b70a626528c7f Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Tue, 11 Aug 2026 17:32:07 +0300 Subject: [PATCH 31/50] fix(cacbg): give the monotonicity gate a path for the removals it sanctions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0033 decision 6 licenses removal by a rules bump "or a correction of wrong input", and names the court-annulled entry as wired to the ADR-0031 suppression path. Neither could get through the gate, and they failed in opposite directions: - A suppression flips status published → suppressed with rules_version untouched, so the link leaves the published set and the gate hard-fails the one takedown the ADR points at. - Correcting the input UNBUILDS the link, so a suppression on it matches nothing and load.mjs's B3 unused-entry rail fails the build instead. So the only escape was bumping rules_version, which relabels every other link. The audit now reads each vanished link's current status: `suppressed` is a declared removal. For a corrected input there is a second fingerprinted list, link-corrections.jsonl, mirroring ADR-0031 for the same reason — `pid|eik` in git records which named official was tied to which company, permanently. load.mjs flags matching keys in the pre-wipe snapshot; the audit reads the flag. Both grounds are still PRINTED, now with the ground that licensed them, since "removed" alone flattens an annulment, a corrected parse and a rules bump into one line. A correction that matches nothing fails the build, mirroring B3 and for a sharper reason: a stale acknowledgement would pre-clear a future disappearance of that same link — the exact regression the gate exists to catch — with nobody having decided it. Acknowledgements are one-shot; the runbook says so and says not to use the list on a removal you cannot explain. Six tests, including the mutation control: an unacknowledged, unsuppressed drop still hard-fails, so a gate that accepted everything would not pass the two new cases. Also lands the snapshot's atomic write (tmp + rename, as the raw deeds already do). The audit deliberately does not swallow a parse failure there, so a torn write wedged every subsequent audit until a human cleared the file by hand. --- ...-evidence-replaces-name-distinctiveness.md | 13 +++- docs/runbooks/related-persons-suppression.md | 32 ++++++++ scripts/cacbg/audit.mjs | 54 +++++++++++-- scripts/cacbg/audit.test.mjs | 56 ++++++++++++++ scripts/cacbg/link-corrections.jsonl | 0 scripts/cacbg/load.mjs | 60 +++++++++++++-- scripts/cacbg/load.test.mjs | 76 +++++++++++++++++++ scripts/cacbg/suppressions.mjs | 23 +++++- 8 files changed, 297 insertions(+), 17 deletions(-) create mode 100644 scripts/cacbg/link-corrections.jsonl diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md index e75c28c38..cfa579ed7 100644 --- a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -252,7 +252,18 @@ Therefore: seals are **re-derived deterministically** on every run — a seal is including held ones, so the review queue explains itself — and monotonicity is enforced as a **gate**. The audit compares against the pre-wipe export and raises a hard finding when a previously published link disappears under an **unchanged** `rules_version`; under a changed version it degrades to a printed diff. -Removal remains an intentional event: a rules-version bump, or a correction of wrong input. +Removal remains an intentional event, and each ground has an expressible mechanism — a gate that hard-fails +the only removals it sanctions is a deadlock, not a rail: + +| Ground | Mechanism | Why not one of the others | +|---|---|---| +| The rules changed | a `rules_version` bump | — | +| The evidence is void (court annulment, wrong person) | ADR-0031 suppression — `status` flips to `suppressed`, so the audit reads the current status and treats it as declared | the link is correctly built; only its publication is wrong | +| The input was wrong | `scripts/cacbg/link-corrections.jsonl`, fingerprinted like the suppression list; `load.mjs` flags the key in the pre-wipe snapshot | correcting the input *unbuilds* the link, so a suppression on it matches nothing and trips the unused-entry rail | + +Both non-rules grounds are one-shot, version-controlled, and reviewed in git; neither is silent — the audit +prints every declared removal with the ground that licensed it. An acknowledgement that matches nothing +fails the build, because a stale one would pre-clear a *future* disappearance of that same link. ### 7. Launch gates diff --git a/docs/runbooks/related-persons-suppression.md b/docs/runbooks/related-persons-suppression.md index 2aaa2d975..1fffa8c19 100644 --- a/docs/runbooks/related-persons-suppression.md +++ b/docs/runbooks/related-persons-suppression.md @@ -61,6 +61,38 @@ wrangler d1 execute --remote \ Then do the Standard takedown steps above **in the same shift**. The direct UPDATE is undone by the next ship (which rebuilds the table); only the git list makes it permanent. +## Correcting a wrong INPUT (a different list — read this before reaching for a suppression) + +Suppression keeps a *built* link out of the public surface. It is the wrong tool when the link should never +have been built at all — a misparsed declaration row, a stake attributed to the wrong filing, an entity +that was never declared. There you fix the input, and the link stops existing. + +That removal still has to get past the monotonicity gate ([ADR-0033](../adr/0033-registry-evidence-replaces-name-distinctiveness.md) +decision 6), which hard-fails on a link that was published last run and is not published now. And you +cannot use a suppression to clear it: correcting the input unbuilds the link, so the fingerprint would +match nothing and `load.mjs` fails on the unmatched-entry rail instead. The two removals fail in opposite +directions, which is why there is a second list. + +1. Compute the fingerprint exactly as above (same salt, same function, same `key_version`). +2. Append one line to `scripts/cacbg/link-corrections.jsonl`: + + ```json + {"fp":"","key_version":"1","reason":"the declaration row was misparsed — ","signal_ref":"","corrected_at":"2026-08-11"} + ``` + +3. Land it **in the same PR as the input fix**. The loader flags that key in the pre-wipe snapshot and the + audit reads it as a declared removal — printed, never silent, and attributed to this ground rather than + flattened into "removed". + +**An acknowledgement is one-shot. Delete the line in the next change.** Once the corrected link stops being +published it also stops appearing in the prior set, so the entry matches nothing and the build fails on the +same rail suppressions use. That is deliberate: a stale acknowledgement would sit in the list and silently +pre-clear a *future* disappearance of that same link — the exact regression the gate exists to catch, with +nobody having decided it. + +Do not use this list to clear a removal you do not understand. If a published link vanished and you cannot +name why, that is the gate doing its job; find the cause. + ## Fail-closed behaviour `load.mjs` aborts the build (non-zero exit) on any of three conditions — each would otherwise silently diff --git a/scripts/cacbg/audit.mjs b/scripts/cacbg/audit.mjs index 9c8355f6e..5eb889d0a 100644 --- a/scripts/cacbg/audit.mjs +++ b/scripts/cacbg/audit.mjs @@ -44,6 +44,17 @@ const published = db ) .all(); +// Every link's CURRENT status, for the monotonicity gate below. A link that left the published set +// because it was SUPPRESSED is still in this table — the takedown is a status flip, not a delete — and +// distinguishing that from a link that simply stopped being built is the difference between a declared +// removal and a silent recall regression. Read here, while the handle is open. +const statusNow = new Map( + db + .prepare('SELECT link_key, status FROM interest_links') + .all() + .map((r) => [r.link_key, r.status]), +); + // The only two evidence rungs that publish (ADR-0033 decision 1). Everything else withholds. const PUBLISHING_EVIDENCE = new Set(['document', 'confirmed']); // The closed vocabulary for a sealed matched_fact lives in evidence.mjs, next to the code that WRITES @@ -189,7 +200,28 @@ try { if (e.code !== 'ENOENT') throw e; } const vanished = (priorPublished ?? []).filter((p) => !publishedNow.has(p.link_key)); -const regressions = vanished.filter((p) => p.rules_version === RULES_VERSION); + +// The three grounds decision 6 sanctions for a removal. Anything else that vanished is a regression. +// +// Getting this set right is what stops the gate from deadlocking the mechanisms it points at. The +// rules bump is the obvious one. The other two are not optional extras — each is the ONLY expressible +// form of a removal the ADR names in prose: +// +// suppressed — the ADR-0031 takedown path, which decision 6 wires the court-annulled entry +// (чл. 29 ЗТРРЮЛНЦ) to by name. It flips status published → suppressed with rules_version +// untouched, so without this branch the one removal the ADR explicitly licenses hard-fails. +// corrected — "a correction of wrong input". Suppression cannot express it: correcting the input +// unbuilds the link, and load.mjs's B3 unused-suppression gate then fails the build for a +// fingerprint that matched nothing. So the two sanctioned removals would fail in OPPOSITE +// directions, leaving a real correction with no path at all. The flag is set by load.mjs from the +// version-controlled, fingerprinted corrections list — a human decision recorded in git, not a +// condition the audit can infer. +// +// Both are declared, both are reviewed, and both are still PRINTED below: a withdrawn public claim is +// never silent, it just is not a build failure. +const declaredRemoval = (p) => + p.rules_version !== RULES_VERSION || p.corrected === true || statusNow.get(p.link_key) === 'suppressed'; +const regressions = vanished.filter((p) => !declaredRemoval(p)); for (const p of regressions) findings.push({ axis: 'D_monotonicity', @@ -214,17 +246,23 @@ for (const [axis, fs] of Object.entries(byAxis)) { if (priorPublished === null) { console.log(`## monotonicity — no prior export at ${SNAPSHOT}; treating this as a first run\n`); } else { - const declared = vanished.filter((p) => p.rules_version !== RULES_VERSION); + const declared = vanished.filter(declaredRemoval); console.log( `## monotonicity — ${priorPublished.length} published last run, ${publishedNow.size} now; ` + `${regressions.length} regression(s), ${declared.length} declared removal(s)\n`, ); - // Declared removals are not findings, but they are never silent: a rules bump is exactly when a - // human should read which named claims it withdrew. - for (const p of declared) - console.log( - ` - ${p.link_key} removed under a rules change (${p.rules_version} → ${RULES_VERSION})`, - ); + // Declared removals are not findings, but they are never silent: withdrawing a named public claim is + // exactly when a human should read WHICH claims went and on what ground. The ground is printed too — + // "removed" alone would flatten a court annulment, a corrected parse and a rules bump into one line. + for (const p of declared) { + const ground = + p.rules_version !== RULES_VERSION + ? `a rules change (${p.rules_version} → ${RULES_VERSION})` + : statusNow.get(p.link_key) === 'suppressed' + ? 'a suppression (ADR-0031 takedown path)' + : 'an acknowledged input correction'; + console.log(` - ${p.link_key} removed under ${ground}`); + } if (declared.length) console.log(''); } diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index 712178c31..ec745bae4 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -316,6 +316,62 @@ test('D — no snapshot (a first run) neither fires nor crashes', () => { assert.equal(threw, false); }); +// The gate must also let the two removals ADR-0033 decision 6 actually SANCTIONS through, or it +// deadlocks the mechanisms it points at. Both are declared events with a reviewed paper trail in git; +// neither is a silent recall regression, and the gate exists to tell those apart. + +test('D — a link taken down through the ADR-0031 suppression path is a declared removal, not a regression', () => { + // Decision 6 names the court-annulled entry (чл. 29 ЗТРРЮЛНЦ) and wires it to ADR-0031. That path + // flips status published → suppressed, so the link leaves the published set with rules_version + // unchanged — firing the gate on the ONE removal the ADR explicitly licenses. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + links: [ + ...ONE_LINK.links, + `'il2','p9|200000002','p9','200000002','${K('ВТОРА ФИРМА ЕООД')}','exact_name_key','document','b2','owns',0,1000,'suppressed'`, + ], + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, + ], + }); + assert.equal(threw, false, 'the sanctioned takedown path must not fail the build'); + assert.match(out, /p9\|200000002/, 'but a withdrawn public claim is never silent'); +}); + +test('D — a snapshot entry acknowledged as a corrected input is a declared removal', () => { + // The other sanctioned ground: the link should never have been published because its INPUT was + // wrong. Suppression cannot express it — correcting the input unbuilds the link, and load.mjs's B3 + // gate then fails the build for a suppression that matched nothing. Without this the two sanctioned + // removals fail in opposite directions and there is no way to clear either. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1', corrected: true }, + ], + }); + assert.equal(threw, false, 'an acknowledged correction must not fail the build'); + assert.match(out, /p9\|200000002/); +}); + +test('D — neither escape hatch fires on its own: an unacknowledged, unsuppressed drop still hard-fails', () => { + // The mutation control for the two tests above. A gate that accepted every disappearance would pass + // both of them, and this is the assertion that says it did not. + const { threw } = buildAndAudit({ + ...ONE_LINK, + links: [ + ...ONE_LINK.links, + `'il2','p9|200000002','p9','200000002','${K('ВТОРА ФИРМА ЕООД')}','exact_name_key','document','b2','owns',0,1000,'held'`, + ], + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, + ], + }); + assert.equal(threw, true, 'held is not a sanctioned removal — the evidence simply stopped licensing it'); +}); + test('D positive control — an unchanged published set produces no monotonicity finding', () => { // Without this, a gate that never fires would pass both negatives above. const { threw, out } = buildAndAudit({ diff --git a/scripts/cacbg/link-corrections.jsonl b/scripts/cacbg/link-corrections.jsonl new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index bcceeb882..e3da7f8a4 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -26,7 +26,12 @@ import { RULES_VERSION, } from '../tr/evidence.mjs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; -import { fingerprint, loadSuppressions, SUPPRESSION_KEY_VERSION } from './suppressions.mjs'; +import { + fingerprint, + loadCorrections, + loadSuppressions, + SUPPRESSION_KEY_VERSION, +} from './suppressions.mjs'; import { canonicalInstitution } from './institutions.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); @@ -145,13 +150,56 @@ const priorPublished = (() => { return []; } })(); +// Decision 6's SECOND sanctioned removal: „a correction of wrong input". A link whose input was wrong +// should never have been published, but correcting the input UNBUILDS it — so a suppression on it +// would match no built link and trip the B3 gate above, while doing nothing leaves a permanent hard +// finding. The acknowledgement is therefore recorded here, against the set the gate actually compares: +// each prior-published key whose fingerprint is listed is exported flagged, and audit.mjs reads the +// flag as a declared removal. Fingerprinted for ADR-0031's reason — `pid|eik` in git would record which +// named official was tied to which company for ever. +const CORRECTIONS_LIST = + process.env.CACBG_CORRECTIONS_LIST || path.join(ROOT, 'scripts/cacbg/link-corrections.jsonl'); +const correctedFp = new Set( + loadCorrections(CORRECTIONS_LIST, SUPP_SALT, SUPPRESSION_KEY_VERSION).map((e) => e.fp), +); +const usedCorrections = new Set(); +const snapshot = priorPublished.map((p) => { + if (correctedFp.size === 0) return p; + const fp = fingerprint(p.link_key, SUPP_SALT); + if (!correctedFp.has(fp)) return p; + usedCorrections.add(fp); + return { ...p, corrected: true }; +}); +// The B3 rail, mirrored — and it matters MORE here. A stale suppression silently un-suppresses; a +// stale acknowledgement silently pre-clears a FUTURE disappearance of that same link, which is exactly +// the regression the gate exists to catch, with nobody having decided it. An acknowledgement is +// one-shot by construction: once the corrected link stops being published it also stops appearing in +// the prior set, so the entry must be deleted from the list in the same change that lands the fix. +if (!EMIT_CANDIDATES_ONLY) { + const unusedCorr = [...correctedFp].filter((fp) => !usedCorrections.has(fp)); + if (unusedCorr.length > 0) { + db.close(); + throw new Error( + `${unusedCorr.length} correction(s) matched NO previously published link — a stale acknowledgement ` + + `would clear a future disappearance of that link, which is the regression this gate exists to ` + + `catch. Delete the entry once its fix has shipped (an acknowledgement is one-shot). ` + + `Unmatched fingerprints: ${unusedCorr.map((f) => f.slice(0, 12) + '…').join(', ')}`, + ); + } +} // Not written by the bootstrap pass, which works on a throwaway copy and has no business restating what // the real run is about to record. -if (!EMIT_CANDIDATES_ONLY) - fs.writeFileSync( - path.join(STAGING, 'published-snapshot.json'), - JSON.stringify(priorPublished, null, 2) + '\n', - ); +// +// tmp + rename, not a bare write: a crash mid-write leaves a truncated file, and audit.mjs deliberately +// does NOT swallow a parse failure here (only ENOENT is a legitimate first run). A torn snapshot would +// therefore wedge every subsequent audit until a human cleared the file by hand. The raw deeds already +// land this way; the gate's own input deserves the same. +if (!EMIT_CANDIDATES_ONLY) { + const snapPath = path.join(STAGING, 'published-snapshot.json'); + const snapTmp = `${snapPath}.tmp`; + fs.writeFileSync(snapTmp, JSON.stringify(snapshot, null, 2) + '\n'); + fs.renameSync(snapTmp, snapPath); +} // Full idempotent rebuild that also picks up schema changes: drop the CACBG tables (children first — // FK-safe) and re-apply the migration. Nothing to preserve — suppressions are external now. diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index b2a9dbe7f..de7e3d80a 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -1307,3 +1307,79 @@ test('the monotonicity gate still sees a prior surface AFTER a bootstrap pass', 'the snapshot went empty — the gate is now vacuous and would never fire again', ); }); + +// ── the corrections list — ADR-0033 decision 6's second sanctioned removal ──────────────────────── +// Decision 6 licenses removal by „a rules-version bump, or a correction of wrong input". Only the +// first was expressible. Suppression cannot carry a correction: correcting the input UNBUILDS the +// link, and the B3 unused-suppression gate above then fails the build for a fingerprint that matched +// nothing — so the two sanctioned removals failed in opposite directions and a real correction had no +// path at all. The list is fingerprinted for the same reason ADR-0031's is: `pid|eik` in git would +// record which named official was linked to which company, for ever. + +test('a corrections entry marks its key in the snapshot, so the audit reads a declared removal', () => { + runLoad(); + const db = open(); + const key = db + .prepare("SELECT link_key FROM interest_links WHERE status='published' LIMIT 1") + .get().link_key; + db.close(); + + const corrFile = path.join(dir, 'corr.jsonl'); + fs.writeFileSync( + corrFile, + JSON.stringify({ + fp: fingerprint(key, SUPP_SALT), + key_version: '1', + reason: 'the declaration row was misparsed; the stake was never declared', + corrected_at: '2026-08-11', + }) + '\n', + ); + runLoad({ CACBG_CORRECTIONS_LIST: corrFile, SUPPRESSION_SALT: SUPP_SALT }); + + const snap = JSON.parse(fs.readFileSync(path.join(STAGING, 'published-snapshot.json'), 'utf8')); + const marked = snap.find((s) => s.link_key === key); + assert.ok(marked, 'the key must still be exported — the gate needs to SEE it leave'); + assert.equal(marked.corrected, true); + // Every other key stays unflagged: an acknowledgement is per-link, never a blanket amnesty. + assert.equal( + snap.filter((s) => s.corrected === true).length, + 1, + 'one acknowledgement must not clear the whole surface', + ); +}); + +test('a corrections entry matching NO previously published link fails the build', () => { + // The B3 rail, mirrored. A stale acknowledgement is worse than a missing one: it sits in the list + // and would clear a FUTURE disappearance of the same link — the exact regression the gate exists to + // catch — with nobody having decided that. + const corrFile = path.join(dir, 'corr-stale.jsonl'); + fs.writeFileSync( + corrFile, + JSON.stringify({ + fp: fingerprint('p-nobody|999999999', SUPP_SALT), + key_version: '1', + reason: 'stale', + corrected_at: '2026-08-11', + }) + '\n', + ); + runLoad(); + assert.throws( + () => runLoad({ CACBG_CORRECTIONS_LIST: corrFile, SUPPRESSION_SALT: SUPP_SALT }), + (err) => + /correction/i.test(String(err.stderr ?? '') + String(err.message ?? '')) && + /matched NO/i.test(String(err.stderr ?? '') + String(err.message ?? '')), + ); +}); + +test('corrections are fail-closed on a missing salt, exactly like suppressions', () => { + const corrFile = path.join(dir, 'corr-nosalt.jsonl'); + fs.writeFileSync( + corrFile, + JSON.stringify({ fp: 'deadbeef', key_version: '1', reason: 'x', corrected_at: '2026-08-11' }) + + '\n', + ); + assert.throws( + () => runLoad({ CACBG_CORRECTIONS_LIST: corrFile, SUPPRESSION_SALT: '' }), + (err) => /SUPPRESSION_SALT is unset/.test(String(err.stderr ?? '') + String(err.message ?? '')), + ); +}); diff --git a/scripts/cacbg/suppressions.mjs b/scripts/cacbg/suppressions.mjs index b1096c606..2c67c93d4 100644 --- a/scripts/cacbg/suppressions.mjs +++ b/scripts/cacbg/suppressions.mjs @@ -33,6 +33,25 @@ export function fingerprint(linkKey, salt) { * An absent or empty list needs no salt (nothing to fingerprint), so the common path stays friction-free. */ export function loadSuppressions(listPath, salt, keyVersion = SUPPRESSION_KEY_VERSION) { + return loadFingerprintedList(listPath, salt, keyVersion, 'suppression'); +} + +/** + * The MONOTONICITY-gate twin (ADR-0033 decision 6): links whose removal is licensed because their + * INPUT was wrong, not because the rules changed. + * + * A separate list from the suppressions, because the two are not the same act and cannot share one. + * A suppression keeps a built link OUT of the public surface; a correction says the link should never + * have been built at all — so correcting the input unbuilds it, and a suppression on it would then + * match nothing and trip the B3 unused gate. Same fingerprinting for the same reason: the raw + * `pid|eik` records which named official was tied to which company, which is precisely what must not + * live in git history. Consumed by load.mjs, which flags the matching keys in the pre-wipe snapshot. + */ +export function loadCorrections(listPath, salt, keyVersion = SUPPRESSION_KEY_VERSION) { + return loadFingerprintedList(listPath, salt, keyVersion, 'correction'); +} + +function loadFingerprintedList(listPath, salt, keyVersion, label) { const entries = existsSync(listPath) ? readFileSync(listPath, 'utf8') .split('\n') @@ -42,14 +61,14 @@ export function loadSuppressions(listPath, salt, keyVersion = SUPPRESSION_KEY_VE : []; if (entries.length > 0 && !salt) { throw new Error( - `${entries.length} suppression(s) in ${listPath} but SUPPRESSION_SALT is unset — refusing to build ` + + `${entries.length} ${label}(s) in ${listPath} but SUPPRESSION_SALT is unset — refusing to build ` + `(would silently un-suppress contested links). Set SUPPRESSION_SALT and retry.`, ); } for (const e of entries) { if (String(e.key_version ?? '') !== String(keyVersion)) { throw new Error( - `suppression ${String(e.fp ?? '').slice(0, 12)}… has key_version ${JSON.stringify(e.key_version)} ` + + `${label} ${String(e.fp ?? '').slice(0, 12)}… has key_version ${JSON.stringify(e.key_version)} ` + `but the current SUPPRESSION_KEY_VERSION is ${keyVersion} — refusing to build (a rotated salt ` + `would silently un-suppress it). Re-fingerprint every entry under the current salt + key_version.`, ); From e5e8709b3e6079197261c111b908f9e05b9e2253 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Tue, 11 Aug 2026 17:39:12 +0300 Subject: [PATCH 32/50] fix(ci,tr,db): close the remaining review findings on injection, purge and card claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four independent fixes, all from the deep-pass reviews still open at 9555fbf. related-persons-data.yml — the environment guard read `${{ inputs.environment }}` inline in two `run:` blocks. `type: choice` binds the UI only; a POST to .../dispatches can pass any string, which an inline expression pastes into the shell before `[` evaluates it. Worse, that same value decides whether the production guard fires, so a crafted one could inject AND walk past the guard meant to trip on it. Both now read $SIGMA_SHIP_ENV, already bound one block up, where the value is data rather than code. (deploy.yml:49 has the same shape and is untouched — it belongs to another change.) cache.mjs — the orphan sweep called unlinkSync unguarded, unlike the expired loop above it. It runs AFTER the DB DELETE commits, so a file that vanished between readdir and unlink aborts a half-finished purge: rows gone, files still on disk, run reported as failed, and no way to tell "already gone" from "an orphan still holding third-party names". ENOENT is now tolerated and uncounted; every other error still surfaces, because a purge that cannot delete has left PII behind and reporting success is the failure it exists to prevent. `unlink` is injectable so the race is tested rather than described. conflicts.render.test.tsx — the family fixture inherited evidenceKind 'document' + registryRole 'owner' from the self-link factory, a row the query cannot produce: findPerson searches for the OFFICIAL, and a relative's stake is registered to the relative, so only a seat/ЕИК confirmation is reachable. It now mirrors production, and a new test pins what the old shape hid — a family card must never render „вписано като съдружник/собственик", which would assert the named official is recorded as an owner, on the one card designed to keep the stakeholder anonymous. related-persons.ts — `lookup_date ?? ''` was an unreachable branch (NOT NULL in 0006, and the row only arrives through the seal filter) that would have shipped an empty string as a date if it ever ran. Narrowed like evidenceKind above it. Not taken: the orphan sweep's `known` set was flagged for using raw `r.eik` while the expired loop uses `safeEik(row.eik)`. safeEik VALIDATES rather than normalizes — it throws on a bad shape — so for any storable ЕИК the two are the same string, and a non-canonical one makes the deed unreadable through readDeed, which means the file really is unreachable PII and deleting it is what the privacy rail should do. --- .github/workflows/related-persons-data.yml | 9 ++- apps/web/app/routes/conflict.methodology.tsx | 6 +- apps/web/app/routes/conflicts.render.test.tsx | 21 +++++++ packages/db/src/queries/related-persons.ts | 6 +- scripts/cacbg/audit.mjs | 4 +- scripts/cacbg/audit.test.mjs | 6 +- scripts/tr/cache.mjs | 18 ++++-- scripts/tr/cache.test.mjs | 62 +++++++++++++++++++ 8 files changed, 120 insertions(+), 12 deletions(-) diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index 7058fd320..59db1219a 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -128,20 +128,25 @@ jobs: # live in each Environment's SIGMA_D1_NAME var; the ship step additionally requires SIGMA_SHIP_ENV to match # and PROVES the name↔SIGMA_D1_ID pair against Cloudflare before deleting — assertD1TargetAuthorized.) # The slot set is kept in lockstep with PRODUCTION_SLOTS in scripts/ship-related-persons.mjs (#226). + # $SIGMA_SHIP_ENV, never `${{ inputs.environment }}` inline. `type: choice` is enforced by the UI + # only — a POST to .../dispatches can pass an arbitrary string, which an inline expression pastes + # into the shell before `[` ever evaluates it. This block is also the one that decides whether the + # production guard fires, so a crafted value could both inject and walk past the guard it was + # meant to trip. The env binding above passes the same value as data, where it cannot be code. - name: Guard — SIGMA_D1_NAME matches the environment run: | case "${SIGMA_D1_NAME}" in sigma-blue|sigma-green) IS_PROD_SLOT=1 ;; *) IS_PROD_SLOT=0 ;; esac - if [ "${{ inputs.environment || 'staging' }}" = "production" ]; then + if [ "${SIGMA_SHIP_ENV}" = "production" ]; then if [ "$IS_PROD_SLOT" != "1" ]; then echo "::error::production requires SIGMA_D1_NAME to be a prod slot (sigma-blue|sigma-green), got '${SIGMA_D1_NAME}' — refusing to ship." exit 1 fi else if [ "$IS_PROD_SLOT" = "1" ]; then - echo "::error::SIGMA_D1_NAME='${SIGMA_D1_NAME}' (a PRODUCTION slot) on a '${{ inputs.environment || 'staging' }}' run — a rebuild would WIPE prod. Set a non-production SIGMA_D1_NAME variable for this Environment." + echo "::error::SIGMA_D1_NAME='${SIGMA_D1_NAME}' (a PRODUCTION slot) on a '${SIGMA_SHIP_ENV}' run — a rebuild would WIPE prod. Set a non-production SIGMA_D1_NAME variable for this Environment." exit 1 fi fi diff --git a/apps/web/app/routes/conflict.methodology.tsx b/apps/web/app/routes/conflict.methodology.tsx index b1fc692c8..152df8a7e 100644 --- a/apps/web/app/routes/conflict.methodology.tsx +++ b/apps/web/app/routes/conflict.methodology.tsx @@ -232,9 +232,9 @@ export default function ConflictMethodology() { един и същ запис на акта (две от три не се приема; имена с инициали или с латински букви не се приемат). Седалищата се променят, затова регистрираното седалище се приема само ако е вписано преди декларирания период —{' '} - и само когато този период е известен. Ако от декларациите не може да - се извлече година, проверката няма с какво да се направи и връзката остава скрита. - Заличените записи се пропускат навсякъде. + и само когато този период е известен. Ако от декларациите не може + да се извлече година, проверката няма с какво да се направи и връзката остава + скрита. Заличените записи се пропускат навсякъде.

Справките се опресняват месечно, а датата на справката и номерът на diff --git a/apps/web/app/routes/conflicts.render.test.tsx b/apps/web/app/routes/conflicts.render.test.tsx index 6be5ea6ee..07339f058 100644 --- a/apps/web/app/routes/conflicts.render.test.tsx +++ b/apps/web/app/routes/conflicts.render.test.tsx @@ -53,6 +53,13 @@ const familyLink = link({ company: 'ЕВРОСТРОЙ 21 ЕООД', eik: '333', relation: 'related', // a close relative's stake — anonymized + // The shape the QUERY actually yields for a family link, not the base factory's self-link default. + // `findPerson` searches the act for the OFFICIAL; a relative's stake is registered to the relative, + // so the official is never found and no `document` rung is reachable — only a seat/ЕИК confirmation. + // A fixture carrying `document`/`owner` here tested a row that cannot exist and hid the assertion + // below, which is the one that matters: no registry-role claim on an anonymized card. + evidenceKind: 'confirmed' as const, + registryRole: null, ownInstitution: false, contractCount: 1, contractValueEur: 250_000, @@ -270,6 +277,20 @@ describe('Trade Register evidence on the card (#279, ADR-0033)', () => { expect(text).not.toContain('вписано като'); }); + it('a FAMILY card never carries a registry-role claim — the relative is not in the act we read', async () => { + // The production shape for a family link, which the fixture used to contradict: `findPerson` looks + // for the OFFICIAL, and a relative's stake is registered to the relative, so the official is never + // found and the rung can only ever be `confirmed`/`registryRole: null`. A card that said „лицето е + // вписано като съдружник/собственик" here would assert that the named official is recorded in the + // register as an owner of this company — a false, named, libel-shaped claim, on the one card whose + // whole design is that the stakeholder stays anonymous (ADR-0030/0032). + await renderConflicts([familyLink]); + const familyCard = container.querySelector('.conflict-card')!; + expect(familyCard.textContent).toContain('деклариран дял на свързано лице'); + expect(familyCard.textContent).toContain('самоличност, потвърдена по декларирани данни'); + expect(familyCard.textContent).not.toContain('вписано като'); + }); + it('links out to the register so a reader can check the same act we read', async () => { await renderConflicts([link({ eik: '201122335' })]); const hrefs = [...container.querySelectorAll('a')].map((a) => a.getAttribute('href') ?? ''); diff --git a/packages/db/src/queries/related-persons.ts b/packages/db/src/queries/related-persons.ts index 8451b01a1..b149ebbd9 100644 --- a/packages/db/src/queries/related-persons.ts +++ b/packages/db/src/queries/related-persons.ts @@ -239,7 +239,11 @@ function toLink(r: LinkRow): ConflictLink { r.registry_role === 'owner' || r.registry_role === 'manager' ? r.registry_role : null, registryEntryNumber: r.entry_number, registryEntryDate: r.entry_date, - registryLookupDate: r.lookup_date ?? '', + // Narrowed for the same reason as evidenceKind, not defaulted. `lookup_date` is NOT NULL in + // migration 0006 and the row only reaches here through the seal filter, so `?? ''` was a branch + // that could not run — and if the invariant ever broke it would have shipped an empty string as a + // date, which reads as a valid-but-blank provenance rather than as the contradiction it is. + registryLookupDate: r.lookup_date as string, }; } diff --git a/scripts/cacbg/audit.mjs b/scripts/cacbg/audit.mjs index 5eb889d0a..6ca5f3a75 100644 --- a/scripts/cacbg/audit.mjs +++ b/scripts/cacbg/audit.mjs @@ -220,7 +220,9 @@ const vanished = (priorPublished ?? []).filter((p) => !publishedNow.has(p.link_k // Both are declared, both are reviewed, and both are still PRINTED below: a withdrawn public claim is // never silent, it just is not a build failure. const declaredRemoval = (p) => - p.rules_version !== RULES_VERSION || p.corrected === true || statusNow.get(p.link_key) === 'suppressed'; + p.rules_version !== RULES_VERSION || + p.corrected === true || + statusNow.get(p.link_key) === 'suppressed'; const regressions = vanished.filter((p) => !declaredRemoval(p)); for (const p of regressions) findings.push({ diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index ec745bae4..267ce4ad1 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -369,7 +369,11 @@ test('D — neither escape hatch fires on its own: an unacknowledged, unsuppress { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, ], }); - assert.equal(threw, true, 'held is not a sanctioned removal — the evidence simply stopped licensing it'); + assert.equal( + threw, + true, + 'held is not a sanctioned removal — the evidence simply stopped licensing it', + ); }); test('D positive control — an unchanged published set produces no monotonicity finding', () => { diff --git a/scripts/tr/cache.mjs b/scripts/tr/cache.mjs index 206687181..764c16f2c 100644 --- a/scripts/tr/cache.mjs +++ b/scripts/tr/cache.mjs @@ -226,7 +226,7 @@ export function coverage(db, wanted) { export function purgeExpired( db, rawDir, - { retentionDays = RETENTION_DAYS, now = new Date() } = {}, + { retentionDays = RETENTION_DAYS, now = new Date(), unlink = fs.unlinkSync } = {}, ) { const cutoff = new Date(now.getTime() - retentionDays * 86_400_000).toISOString(); const expired = db.prepare('SELECT eik, raw_path FROM deeds WHERE fetched_at < ?').all(cutoff); @@ -236,7 +236,7 @@ export function purgeExpired( if (!row.raw_path) continue; // Resolve through the same safeEik path-traversal rail the writer used, never the stored string. try { - fs.unlinkSync(path.join(rawDir, `${safeEik(row.eik)}.json`)); + unlink(path.join(rawDir, `${safeEik(row.eik)}.json`)); files++; } catch (e) { // Already gone is the goal state, not a failure. Anything else must surface — a purge that @@ -261,8 +261,18 @@ export function purgeExpired( ); for (const name of names) { if (!name.endsWith('.json') || known.has(name)) continue; - fs.unlinkSync(path.join(rawDir, name)); - orphans++; + // The same ENOENT tolerance the expired loop above has, and for a sharper reason here: this loop + // runs AFTER the DB DELETE has committed, so an unguarded throw half-purges — rows gone, files + // still on disk — and reports the whole run as failed. The listing is a snapshot, so a name can + // legitimately be gone by the time we reach it (a concurrent purge, an operator clearing scratch). + // Already gone is the goal state. Anything else still surfaces: a purge that cannot delete has + // left third-party names on disk, and reporting success would be the failure it exists to prevent. + try { + unlink(path.join(rawDir, name)); + orphans++; + } catch (e) { + if (e.code !== 'ENOENT') throw e; + } } return { rows: expired.length, files, orphans }; } diff --git a/scripts/tr/cache.test.mjs b/scripts/tr/cache.test.mjs index 8046c1ada..a7357ec88 100644 --- a/scripts/tr/cache.test.mjs +++ b/scripts/tr/cache.test.mjs @@ -229,3 +229,65 @@ test('purgeExpired defaults to the 35-day window and tolerates an already-missin assert.equal(res.files, 0); assert.ok(readDeed(db, '201122335'), 'the 21-day-old deed is inside the window'); })); + +test('purgeExpired tolerates an orphan that vanished under it, and keeps sweeping', () => { + // The benign race: readdirSync lists a name, and it is gone by the time unlink runs (a concurrent + // purge, an operator clearing scratch/). The expired loop right above has always tolerated this; + // the orphan loop did not, and it throws AFTER the DB DELETE has committed — so the run half-purges, + // reports "purge failed", and leaves the operator unable to tell "already gone" from "an orphan + // still holding third-party names". Injected rather than staged, because the race cannot be timed + // from a test; the injection seam matches the one httpGet/sleep/now already use in this codebase. + return withRaw((db, rawDir) => { + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-08-01T00:00:00Z' })); + writeRaw(rawDir, '115536179'); + writeRaw(rawDir, '204556676'); // orphan 1 — disappears under us + writeRaw(rawDir, '831391124'); // orphan 2 — must still be swept + + const seen = []; + const res = purgeExpired(db, rawDir, { + now: new Date('2026-08-05T00:00:00Z'), + unlink: (p) => { + seen.push(path.basename(p)); + if (p.endsWith('204556676.json')) { + const err = new Error('ENOENT: no such file or directory'); + err.code = 'ENOENT'; + throw err; + } + fs.unlinkSync(p); + }, + }); + + assert.deepEqual(seen.sort(), ['204556676.json', '831391124.json'], 'both orphans attempted'); + assert.equal( + res.orphans, + 1, + 'a file that was already gone was not deleted BY US — do not count it', + ); + assert.equal(fs.existsSync(path.join(rawDir, '831391124.json')), false, 'the sweep continued'); + assert.equal( + fs.existsSync(path.join(rawDir, '115536179.json')), + true, + 'the live deed is untouched', + ); + }); +}); + +test('purgeExpired still refuses loudly on an orphan it could not delete for a REAL reason', () => { + // The other half, and the reason the guard is ENOENT-only. A permission error means retained PII is + // still on disk; reporting success would be the failure mode the purge exists to prevent. + return withRaw((db, rawDir) => { + writeRaw(rawDir, '204556676'); + assert.throws( + () => + purgeExpired(db, rawDir, { + now: new Date('2026-08-05T00:00:00Z'), + unlink: () => { + const err = new Error('EACCES: permission denied'); + err.code = 'EACCES'; + throw err; + }, + }), + /EACCES/, + ); + }); +}); From aee14278fa891328e63ea70c4a58d87850be687b Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Tue, 11 Aug 2026 17:49:16 +0300 Subject: [PATCH 33/50] docs(adr): record the monthly cadence as a decision, not a YAML comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merging the crawl into the decision job forced the schedule from daily to monthly, which contradicts ADR-0033's own consequence line ("decisions daily, registry lookups monthly"). That line is now false, and the truth lived only in a workflow comment — the one place a reader looking for the decision would not go, and nowhere anyone could argue with it. ADR-0034 records it properly: the two constraints that collide (raw deeds must not outlive the runner, because an Actions cache entry sits under a restore-keys chain the 35-day retention cannot reach; and the decision cannot be made without them, because the strongest rung compares the declarant's name against the deed text the index deliberately refuses to store), and both rejected alternatives — persisting the deeds behind someone else's eviction policy, and having the crawl emit per-(link, ЕИК) verdicts so only booleans cross the boundary. The second is the right way back to a daily decision and is named as such rather than left implicit. ADR-0033 is not rewritten (repo convention: an accepted ADR is superseded, not edited) — the stale line carries a pointer to its successor and is scoped to the cadence claim alone. The workflow comment now says it is the operational summary, not the record. Also notes the consequence a monthly rebuild does NOT have: a takedown does not wait for the schedule. It goes through ADR-0031 and, on live exposure, a direct UPDATE on the served D1. --- .github/workflows/related-persons-data.yml | 5 +- ...-evidence-replaces-name-distinctiveness.md | 6 +- ...ups-and-decisions-share-one-monthly-run.md | 63 +++++++++++++++++++ docs/adr/README.md | 1 + 4 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index 59db1219a..e352a7875 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -21,7 +21,10 @@ on: # its own copy of the credential guards, the D1-target guard and the ship floor — and the copy is the # one nobody exercises. # - # WHY MONTHLY AND NOT DAILY, which is what §9 and ADR-0033 describe. A decision cannot be recomputed + # WHY MONTHLY AND NOT DAILY, which is what §9 and ADR-0033 originally describe. The decision and its + # rejected alternatives are recorded in ADR-0034, which supersedes ADR-0033's cadence line — this + # comment is the operational summary, not the record, so a reader who disagrees has somewhere to argue. + # A decision cannot be recomputed # without the raw deeds — evidenceVerdict's strongest rung matches the declarant's name against the # register's own text — and the raw deeds must not be persisted between runs: they carry the names and # addresses of co-owners and managers who hold no public office, and an Actions cache entry lives on diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md index cfa579ed7..0993a41b8 100644 --- a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -327,7 +327,11 @@ None of these are follow-ups. function of declarations, cached deeds and contracts, but they no longer run on the 6-hourly ETL cycle (#279 §9 assumes they do; they do not — the loader needs `node:sqlite`, the declaration corpus and the full winner set, none of which exist inside a Worker). Cadence becomes two scheduled workflows: decisions - daily, registry lookups monthly. + daily, registry lookups monthly. **Superseded by [ADR-0034](0034-registry-lookups-and-decisions-share-one-monthly-run.md):** + the split rests on the decision run being able to work from the cache alone, and it cannot — the strongest + rung compares the declarant's name against the deed text itself, which the index deliberately does not + store, so the raw deeds must be present; and they must not survive the runner. One monthly job, therefore. + Nothing else in this ADR is affected. - Two capabilities are deliberately left on the table and become separate changes under new rules versions: as-of evidence via `?entryDate=`, and the §7 „и към днешна дата" labels with their post-period contracts. - The seat rung is structurally capped and we accept the cap: a declared seat is captured **only** from the diff --git a/docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md b/docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md new file mode 100644 index 000000000..c9afd007e --- /dev/null +++ b/docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md @@ -0,0 +1,63 @@ +# ADR-0034 — Справките в ТР и решенията текат в едно месечно задание + +- **Статус:** Прието +- **Дата:** 2026-08-11 +- **Обхват:** конвейерът за свързани лица — `related-persons-data.yml`, `scripts/cacbg/load.mjs`, + `scripts/tr/fetch-deeds.mjs`. Заменя единствено твърдението за каденцията в последствията на + [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md); всичко останало в него стои. + +## Контекст + +ADR-0033 записа каденцията така: **„решения ежедневно, справки в регистъра месечно"**. Разделянето +почиваше на едно предположение — че решаващият ход може да работи само върху кеша, а обхождането е +отделен, по-рядък ход. Прегледът на PR-а показа, че това предположение не се държи. + +Двете страни на противоречието: + +1. **Суровите актове не бива да преживяват изпълнителя.** Кешираният акт носи имена на трети лица — + съдружници и управители без публична длъжност, и адреса на дружеството. Пренасянето му между ходове + изисква хранилище (кеш на Actions или артефакт), чието изгонване е политика за капацитет, не за + задържане: 35-дневният срок по решение 5 на ADR-0033 се изпълнява от `purgeExpired` на изпълнителя и + няма никаква власт над копие в хранилището на GitHub. Това беше находка B3 на прегледа. +2. **Решението не може да се вземе без суровите актове.** Най-силното стъпало сравнява трите имена на + декларатора със самия текст на записа. Индексът на кеша нарочно не пази нито едно име (§9), така че + „кеширай само индекса" премахва точно данните, от които `evidenceVerdict` се нуждае. + +Двете заедно значат: **всеки ход, който решава, трябва и да обхожда.** А ежедневно решение тогава значи +~400 заявки дневно към чужд публичен регистър, което спец. §3.3 („НИКОГА не обхождай ТР масово"; +КЗЛД, CJEU C-200/23) не разрешава да се приема с лека ръка — ограничителят на регистъра е единственият +начин операторът да изрази предпочитание за темп и ние не се настройваме около него емпирично. + +## Решение + +**Едно задание, месечно.** `related-persons-data.yml` изпълнява последователно: зареждане на списъка с +кандидати (`load.mjs --emit-candidates`, върху захвърляемо копие) → обхождане на регистъра → решение и +публикуване (`load.mjs`) → одит (`audit.mjs`) → изтриване на суровите актове в стъпка `if: always()`. +`scratch/tr` **не се кешира никога**; кешира се само `scratch/cacbg/raw` (публични декларации). + +`related-persons-tr-refresh.yml` е премахнат — нямаше какво да разделя. + +**Отхвърлена алтернатива: пренасяне на актовете между ходове** (кеш или артефакт с правило за +жизнен цикъл). Работи технически, но връща B3: срокът на задържане престава да е наш и става политика на +чуждо хранилище. R2 със собствено правило за изтриване остава разумен вариант, ако някога се наложи — +днес не се налага. + +**Отхвърлена алтернатива: ежедневно решение.** Единственият начин да се запази е обхождането да издава +**присъди по (връзка, ЕИК)** — булеви стойности без нито едно име — така че само те да пресичат +границата между ходовете. Това е коректната посока и остава на масата, но е промяна в дизайна на +доказателствения слой, не настройка на график, и не влиза в тази промяна. + +## Последствия + +- Публикуваната повърхност се преизчислява **месечно**, не ежедневно. Практическата цена е по-малка, + отколкото звучи: входът също се движи бавно (декларациите се подават годишно, вписванията в ТР — + рядко), а датата на справката се показва на всяка връзка, така че давността е видима за читателя, а не + подразбираща се. +- **Спешна корекция не чака графика.** Свалянето на връзка минава през ADR-0031 и ръчния ход, а при + жива експозиция — през пряк `UPDATE` върху обслужващата D1; вж. + [ръководството](../runbooks/related-persons-suppression.md). Каденцията определя кога се преоткриват + връзки, не кога може да се махне една. +- Едно задание значи един режим на отказ: ако обхождането спре при 429, гейтът за покритие на `load.mjs` + отказва частичния кеш и ходът не публикува нищо. Това е желаната посока — по-скоро без обновяване, + отколкото с орязана повърхност — и е същият гейт, който вече пазеше разделения вариант. +- #279 §9 продължава да описва ежедневни решения. Този ADR е записът, че не е така, и защо. diff --git a/docs/adr/README.md b/docs/adr/README.md index ecaa107fd..f0a8193ba 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -41,5 +41,6 @@ | [0031](0031-suppressions-version-controlled-fingerprinted.md) | Свалянията на връзки са версиониран списък с HMAC-отпечатък (salt = CI secret), прилаган при зареждане — не служебна таблица; не изтича сигнала „кой е свален" към прод | Прието | | [0032](0032-family-ownership-published-under-public-interest.md) | Дял на свързано лице се публикува на поименната повърхност наравно със собствения — на основание надделяващ обществен интерес (ЗДОИ чл. 41и; C-184/20 забранява само поименните данни); близкият никога не се назовава, връзката не се твърди; заменя ADR-0030 | Прието | | [0033](0033-registry-evidence-replaces-name-distinctiveness.md) | Връзка се публикува само срещу проверим факт от Търговския регистър — доказателственият ред заменя нивата по отличителност на името; евристиката вече обосновава твърдение (изрично изключение от ADR-0007), барът за акционерни форми е обединение от три сигнала, монотонността е гейт, не хранилище; заменя ADR-0009/0015/0017 | Прието (дизайн) | +| [0034](0034-registry-lookups-and-decisions-share-one-monthly-run.md) | Справките в ТР и решенията текат в едно месечно задание — суровите актове не преживяват изпълнителя, а решението не може да се вземе без тях; заменя твърдението за каденцията в ADR-0033 | Прието | Свързан проектен документ: [spec/related-persons-foundation.md](../spec/related-persons-foundation.md). From 36cdd9d0314124c36053c2fc4b5bf4ddbe33b95a Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Tue, 11 Aug 2026 18:17:13 +0300 Subject: [PATCH 34/50] chore(deps): raise nanoid past GHSA-2v37-7h3g-55p8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Dependency audit went red on a newly published advisory for nanoid 3.3.16 (GHSA-2v37-7h3g-55p8, CVSS 8.2), not on anything this branch changed — it touches neither package.json nor pnpm-lock.yaml, so main fails identically on its next run. Lockfile-only, and confined to nanoid: postcss@8.5.24 already declares `^3.3.16`, so 3.3.18 satisfies the existing range with no manifest edit and no override. `pnpm update nanoid` does not lift a transitive pin; `-r` does. Not suppressed in osv-scanner.toml on purpose — that file's own doctrine reserves entries for advisories whose only fix is unavailable or a major upgrade, and this one is a patch. Verified with the same checksum-pinned osv-scanner 2.4.0 the CI step runs: "No issues found". The package is build-time only (postcss → vite → vitest, all dev) and reaches no Worker bundle, but the chain is exercised anyway — full suite 6/6 packages, lint, typecheck 7/7, and a real `@sigma/web` production build through postcss. --- pnpm-lock.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 304ec6820..6cf35954e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1643,8 +1643,8 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - nanoid@3.3.16: - resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} + nanoid@3.3.18: + resolution: {integrity: sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -3369,7 +3369,7 @@ snapshots: ms@2.1.3: {} - nanoid@3.3.16: {} + nanoid@3.3.18: {} node-releases@2.0.45: {} @@ -3406,7 +3406,7 @@ snapshots: postcss@8.5.24: dependencies: - nanoid: 3.3.16 + nanoid: 3.3.18 picocolors: 1.1.1 source-map-js: 1.2.1 From 4d21ba0c6b262ab50922298ebf6ede23a050d250 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:17:16 +0300 Subject: [PATCH 35/50] =?UTF-8?q?fix(cacbg):=20bar=20a=20listed=20=D0=90?= =?UTF-8?q?=D0=94=20whose=20seat=20carries=20neither=20comma=20nor=20dot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `closelyHeldForm` asked whether the name ENDS in a joint-stock form. A declarant who types „ТРЕЙС ГРУП ХОЛД АД София" defeats that: the comma-peel needs a comma and SEAT_MARKER needs a literal dot, so neither strips the seat, the name stops ending in its form, and a listed АД reads as closely-held — the „11 акции на Trace → €88M" trap, reached from the plainest input shape there is. Decide on the LAST form TOKEN instead of an end anchor. Position-independent, so a trailing seat cannot flip the verdict, while „АД" leading („АД ГРУП ООД") or glued („АД-ХОК ЕООД") still isn't the form. JOINT_STOCK itself is untouched — deed.mjs's envelope test reads a registry-clean `fullName` that does end in its form, and deed.test.mjs pins the two patterns to each other. `stripSeatSuffix` gains the same cut for nameDistinctiveness, where the blind spot fails toward PUBLISHING: an uncounted seat token inflates the content-word count to 3 and reads as distinctive. Truncation is anchored on фирма-TERMINATING forms only — ЕТ/СД/КД precede the name, and cutting after them would eat it. Positive controls throughout: a predicate that always returned false would pass every new bar assertion, so each is paired with a name that must stay material. --- scripts/cacbg/classify.mjs | 50 ++++++++++++++++++++++++++------ scripts/cacbg/classify.test.mjs | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 8 deletions(-) diff --git a/scripts/cacbg/classify.mjs b/scripts/cacbg/classify.mjs index a5f9de25f..d066a3fa9 100644 --- a/scripts/cacbg/classify.mjs +++ b/scripts/cacbg/classify.mjs @@ -33,6 +33,24 @@ const FORM_TOKENS = new Set([ // generic/withhold — the safe side), never fabricates a joint-stock exclusion. Operates on UPPERCASE input. const SEAT_MARKER = /\s+(?:ГР|С|ОБЩ|ОБЛ|Ж\.К)\.\s*\S[^,]*$/u; const hasFormToken = (s) => s.split(/[^А-ЯЁ]+/).some((t) => FORM_TOKENS.has(t)); + +// The legal forms that TERMINATE a фирма (ЗТРРЮЛНЦ writes them as a suffix), so anything after one is a +// seat or a qualifier — never part of the name. ЕТ/СД/КД/КООПЕРАЦИЯ/ФОНДАЦИЯ/СДРУЖЕНИЕ are deliberately +// ABSENT: those PRECEDE the фирма („ЕТ Алекс Петров Димитров"), and truncating after them would eat the +// name itself — turning a distinctive ЕТ into a generic one and withholding a true link. +const SUFFIX_FORMS = new Set(['ЕООД', 'ООД', 'ЕАД', 'АД', 'АДСИЦ', 'КДА', 'ДЗЗД']); + +// Cut everything after the LAST фирма-terminating form token. This is the case the comma-peel and the +// marker strip both miss: „ТРЕЙС ГРУП ХОЛД АД София" has neither a comma nor a „гр." dot, so it survived +// both, stopped ending in its form, and defeated every end-anchored form test downstream. Token-exact +// (never a substring), so „КАДИЕВ ГЛОБАЛ ЕООД" and „АД-ХОК ЕООД" are untouched. +function stripAfterSuffixForm(s) { + const re = /[А-ЯЁ]+/gu; + let last = null; + for (let m = re.exec(s); m !== null; m = re.exec(s)) if (SUFFIX_FORMS.has(m[0])) last = m; + return last === null ? s : s.slice(0, last.index + last[0].length).trim(); +} + function stripSeatSuffix(upper) { let s = String(upper).trim(); // Peel trailing comma-clauses right-to-left while the clause bears no legal form (i.e. it's a seat, not @@ -45,7 +63,7 @@ function stripSeatSuffix(upper) { ) { s = m[1].trim(); } - return s.replace(SEAT_MARKER, '').trim(); + return stripAfterSuffixForm(s.replace(SEAT_MARKER, '').trim()); } /** @@ -81,6 +99,9 @@ const norm = (s) => // directory without closing a cacbg↔tr cycle. A test in deed.test.mjs pins the two identical; change one // and that test fails rather than the two silently diverging on a legal form. export const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; +// The same four forms as whole tokens. Kept in step with JOINT_STOCK above by classify.test.mjs, and with +// deed.mjs's JOINT_SUFFIX twin by deed.test.mjs — three spellings of one rule, all three pinned. +const JOINT_STOCK_FORMS = new Set(['АД', 'ЕАД', 'АДСИЦ', 'КДА']); /** * Materiality by legal form. The public ownership surface is CLOSELY-HELD companies only (ООД/ЕООД/ЕТ/ * КД/СД/ДЗЗД or a form-unspecified name from the closely-held table). Joint-stock forms (АД/ЕАД/АДСИЦ/КДА, @@ -90,13 +111,26 @@ export const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КД * so it withholds rather than fabricates. @returns {boolean} true ⇒ material/closely-held. */ export function closelyHeldForm(name) { - return !JOINT_STOCK.test( - stripSeatSuffix( - String(name ?? '') - .normalize('NFC') - .toUpperCase(), - ), - ); + // LAST-FORM-TOKEN-WINS, not an end anchor. The anchor asked „does the name END in a joint-stock form?", + // which a declarant-typed cell can defeat just by appending a seat — and `stripSeatSuffix` cannot be + // trusted to have removed every shape of one. Asking instead „which legal form is the name's LAST?" + // is position-independent: a trailing seat, a stray qualifier, or nothing at all leaves the verdict + // unchanged, while „АД" leading („АД ГРУП ООД") or glued („АД-ХОК ЕООД") still isn't the form. + // + // This is why the predicate no longer uses JOINT_STOCK directly while deed.mjs's twin still does: that + // twin reads the deed envelope's `fullName` — a REGISTRY-clean name that genuinely ends in its form — + // whereas this one reads a free-text cell a human typed. Same rule, different input hygiene. + const tokens = stripSeatSuffix( + String(name ?? '') + .normalize('NFC') + .toUpperCase(), + ) + .split(/[^А-ЯЁ]+/u) + .filter(Boolean); + const lastForm = tokens.filter((t) => FORM_TOKENS.has(t)).at(-1); + // No form token at all ⇒ nothing says joint-stock ⇒ material, exactly as the anchor behaved. The bar + // only ever fires on an EXPLICIT joint-stock form, so it withholds rather than fabricates. + return lastForm === undefined || !JOINT_STOCK_FORMS.has(lastForm); } // seatConfirmed() and publishTier() lived here until #279. The publish tiers they produced diff --git a/scripts/cacbg/classify.test.mjs b/scripts/cacbg/classify.test.mjs index e0e8b2fa1..f0f971ba3 100644 --- a/scripts/cacbg/classify.test.mjs +++ b/scripts/cacbg/classify.test.mjs @@ -5,6 +5,7 @@ import { temporalStatus, localityToken, closelyHeldForm, + JOINT_STOCK, } from './classify.mjs'; test('nameDistinctiveness: numbers / Latin / ≥3 words are distinctive; bare 1-2 word Cyrillic is generic', () => { @@ -90,6 +91,56 @@ test('closelyHeldForm: a trailing седалище after the form does not flip assert.equal(closelyHeldForm('СТРОЙ, ИНВЕСТ ООД'), true); // trailing clause has ООД → closely-held }); +test('the token set and the JOINT_STOCK regex name the SAME four forms', () => { + // closelyHeldForm now decides on the last form TOKEN while deed.mjs's envelope test still uses the + // end-anchored REGEX. Two spellings of one rule drift silently — the day one gains a form the other + // lacks, a joint-stock company passes one gate and is barred by the other. Pin them to each other. + // (deed.mjs's third spelling is pinned to the regex by deed.test.mjs:378.) + for (const form of ['АД', 'ЕАД', 'АДСИЦ', 'КДА']) { + assert.equal(JOINT_STOCK.test(`ФИРМА ${form}`), true, `regex misses ${form}`); + assert.equal(closelyHeldForm(`ФИРМА ${form}`), false, `token set misses ${form}`); + // …and the mirror: every form the regex accepts must be one the token set bars, seat or no seat. + assert.equal(closelyHeldForm(`ФИРМА ${form} София`), false, `token set misses ${form} + seat`); + } + // Every FORM_TOKEN that is NOT one of the four must read as closely-held. + for (const form of ['ЕООД', 'ООД', 'ЕТ', 'ДЗЗД', 'КД', 'СД']) { + assert.equal(closelyHeldForm(`ФИРМА ${form}`), true, `${form} wrongly barred`); + } +}); + +test('closelyHeldForm: a seat with NO comma and NO „гр." marker still cannot flip an АД (libel)', () => { + // The gap the marker/comma rules leave open. `SEAT_MARKER` requires a literal dot and the comma-peel + // requires a comma, so „ТРЕЙС ГРУП ХОЛД АД София" — the plainest way a declarant writes it — survives + // both, no longer ENDS in the form, and the end-anchored JOINT_STOCK test misses it. A listed АД then + // reads as closely-held: the „11 акции на Trace → €88M" trap, from the one input shape nothing strips. + assert.equal(closelyHeldForm('ТРЕЙС ГРУП ХОЛД АД София'), false); + assert.equal(closelyHeldForm('ТРЕЙС ГРУП ХОЛД АД СОФИЯ'), false); + assert.equal(closelyHeldForm('Транспроект ЕАД Пловдив'), false); + assert.equal(closelyHeldForm('НЕС АДСИЦ Варна'), false); + assert.equal(closelyHeldForm('АЛФА КДА София'), false); + // POSITIVE CONTROLS — the bar must stay a bound, not become a blanket. A predicate that always returned + // false would pass every assertion above; these are what distinguish the fix from that (ADR-0027). + assert.equal(closelyHeldForm('Вамос ООД Русе'), true); // dot-less seat on a closely-held form + assert.equal(closelyHeldForm('ЕНЕРДЖИ СЪПЛАЙ ЕООД Бургас'), true); + assert.equal(closelyHeldForm('АД ГРУП ООД'), true); // leading „АД" is not the form + assert.equal(closelyHeldForm('АД-ХОК ЕООД'), true); // „АД" glued by a hyphen is not a form token + assert.equal(closelyHeldForm('КДА ГРУП ООД'), true); + assert.equal(closelyHeldForm('КАДИЕВ ГЛОБАЛ ЕООД'), true); // „АД" inside a word + assert.equal(closelyHeldForm('ЕТ Алекс'), true); // ЕТ leads the фирма — nothing after it is a seat +}); + +test('nameDistinctiveness: a dot-less trailing city is not counted as a content word either', () => { + // Same blind spot, and here it fails toward PUBLISHING: an uncounted seat token inflates the content-word + // count to 3 ⇒ 'distinctive'. Since #279 rung 2 gates an uncorroborated „Документ" publish on exactly this + // predicate (ADR-0035), a seat read as a content word is a false company-identity claim, not just noise. + assert.equal(nameDistinctiveness('СТРОЙ ИНВЕСТ ООД София'), 'generic'); + assert.equal(nameDistinctiveness('НИКАС КОМЕРС ЕООД Пловдив'), 'generic'); + // POSITIVE CONTROLS: a genuinely ≥3-content-word фирма stays distinctive, and a leading-form ЕТ name + // keeps its content words — nothing after „ЕТ" is a seat, so the strip must not reach them. + assert.equal(nameDistinctiveness('ХИДРО СТРОЙ МОНТАЖ ЕООД София'), 'distinctive'); + assert.equal(nameDistinctiveness('ЕТ АЛЕКС ПЕТРОВ ДИМИТРОВ'), 'distinctive'); +}); + test('nameDistinctiveness: a trailing city is not counted as a content word (no premature publish)', () => { // The exact over-publish case: 2 real content words + a seat token would read as 3 → distinctive. assert.equal(nameDistinctiveness('СТРОЙ ИНВЕСТ ООД, СОФИЯ'), 'generic'); // was distinctive via +СОФИЯ From 37a3db8655c56a034694495bac0d0449dc94fcde Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:22:06 +0300 Subject: [PATCH 36/50] fix(cacbg): date a filing by its folder when the declared year is unreadable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `if (!Number.isFinite(fy)) continue` looked like the surrounding fail-safe — "never withdraw on missing evidence" — but it is the opposite. An absent filing legitimately yields no horizon; a PRESENT filing that is silently dropped never advances one, so `divested` stays false and a stake the official has since sold keeps naming them on the public surface. Stale claim, real person, no symptom in the output. filings.jsonl already carries `folder` (extract.mjs), so fall back to it. The folder is the PUBLICATION year and runs ahead of the declared year, so the horizon can advance up to a year early — that errs toward withdrawing a claim we are no longer sure of, which is the safe direction on this surface. Documented at the fallback rather than left for a reader to infer. A filing datable by NEITHER field is still ignored: this dates a filing, it does not invent one. Both counts are reported, because a horizon that failed to advance is invisible — without the counter, a corpus-wide date regression would show up only as a surface that quietly stopped withdrawing anything. The RED publishes via the SEAT rung, not rung 2: an owner-matched deed would let §7 reconciliation reverse the divestment and mask the bug. Mutation-checked — removing the fallback fails the test with published/withdrawn. Positive control: an undatable filing leaves its link published. --- scripts/cacbg/load.mjs | 31 +++++++++++- scripts/cacbg/load.test.mjs | 99 ++++++++++++++++++++++++++++++++++--- 2 files changed, 121 insertions(+), 9 deletions(-) diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index e3da7f8a4..aecd680a1 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -355,14 +355,41 @@ const familyMaterialByTemplate = new Map(); // declare a stake ONLY in the interests declaration). One record per declaration incl. empty / no-material ones, // so a same-type divest-to-ZERO still advances the horizon. An absent/typeless file just yields no match ⇒ the // link is kept (fail-safe: never withdraw on missing evidence). +// +// A filing whose is unreadable falls back to its FOLDER year (#279 §1.3). Dropping the record +// instead — the previous behaviour — is not the fail-safe it resembles: an undated filing that vanishes +// never advances the horizon, so `divested` stays false and a stake the official has since SOLD keeps +// naming them on the public surface. That is a stale claim about a real person, which is the failure this +// surface can least afford. The folder year is an APPROXIMATION — it is the publication year and runs +// ahead of the declared year (migration 0003) — so it can advance the horizon by up to a year early. That +// errs toward WITHDRAWING a claim we are no longer sure of, which is the safe direction here. +// +// A filing datable by NEITHER field is still ignored: the fallback dates a filing, it does not invent one. const filingMaxByPersonType = new Map(); +let filingFolderDated = 0, + filingUndatable = 0; for (const f of readJsonl(path.join(STAGING, 'filings.jsonl'))) { if (!isMatchableKey(companyNameKey(f.person))) continue; - const fy = yr(f.year); - if (!Number.isFinite(fy)) continue; + let fy = yr(f.year); + if (!Number.isFinite(fy)) { + fy = yr(f.folder); + if (!Number.isFinite(fy)) { + // Counted, not silently dropped. A horizon we failed to advance is invisible in the output — the + // link simply stays up — so without this the only symptom of a corpus-wide date regression would be + // a surface that quietly stopped withdrawing anything. + filingUndatable++; + continue; + } + filingFolderDated++; + } const k = `${personId(f.person, f.institution)}|${f.template ?? ''}`; filingMaxByPersonType.set(k, Math.max(filingMaxByPersonType.get(k) ?? fy, fy)); } +if (filingFolderDated > 0 || filingUndatable > 0) { + console.log( + ` filings: ${filingFolderDated} dated by FOLDER (unreadable ), ${filingUndatable} undatable (ignored — no horizon)`, + ); +} db.exec('BEGIN'); for (const h of readJsonl(path.join(STAGING, 'holdings.jsonl'))) { diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index de7e3d80a..db60a560f 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -170,6 +170,15 @@ before(() => { -- stake at all. The empty filing advances his horizon past 2019 → the 2019 stake is withdrawn. INSERT INTO bidders VALUES ('eik:101010104','ДИВЕСТ ЗЕРО 4 ЕООД','101010104',1,'София'); INSERT INTO contracts VALUES ('c11','t1','eik:101010104','2019-05-01',150000); + -- §1.3 unparseable filing YEAR: Безгодин owns ДИВЕСТ БЕЗГОД (2019), then files a later declaration whose + -- is unreadable while its FOLDER carries 2023. The folder must supply the horizon, or the filing + -- is dropped, the horizon never advances, and a sold stake stays published — a stale public claim. + INSERT INTO bidders VALUES ('eik:212121218','ДИВЕСТ БЕЗГОД 9 ЕООД','212121218',1,'София'); + INSERT INTO contracts VALUES ('c17','t1','eik:212121218','2019-05-01',120000); + -- §1.3 positive control: Дрънкан's later filing is datable by NEITHER NOR folder. It must be + -- ignored, never guessed — an undatable filing is no evidence of a sale, so this stake stays published. + INSERT INTO bidders VALUES ('eik:232323231','ДРЪНКАН ТЕХ 10 ЕООД','232323231',1,'София'); + INSERT INTO contracts VALUES ('c18','t1','eik:232323231','2019-05-01',110000); -- N10 canonicalization: Канонов owns КАНОН ТЕХ 5, filing „МВР" one year and the full ministry name the -- next → ONE identity, ONE link (not a split). Distinctive name (number) → tier B publishable. INSERT INTO bidders VALUES ('eik:131313136','КАНОН ТЕХ 5 ЕООД','131313136',1,'София'); @@ -462,6 +471,43 @@ before(() => { holderRelation: 'self', controlHash: 'H14', }, + // §1.3 unparseable filing year: Безгодин's 2019 stake. His later filing (in filings.jsonl, no holdings + // row) carries an unreadable but a 2023 FOLDER — the fallback that must withdraw this stake. + { + folder: '2020', + xmlFile: 'BG0.xml', + year: '2019', + template: 'assets', + category: '', + institution: 'T3', + person: 'Безгодин Иванов Дивестов', + position: '', + entity: 'ДИВЕСТ БЕЗГОД 9 ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: 'гр. София', + holderRelation: 'self', + controlHash: 'H20', + }, + // §1.3 positive control: Дрънкан's 2019 stake, whose only later filing is undatable (see filings.jsonl). + { + folder: '2020', + xmlFile: 'DR0.xml', + year: '2019', + template: 'assets', + category: '', + institution: 'T4', + person: 'Дрънкан Иванов Тестов', + position: '', + entity: 'ДРЪНКАН ТЕХ 10 ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: '', + holderRelation: 'self', + controlHash: 'H21', + }, // B4 UNKNOWN holder: the holder cell is neither confidently the declarant's own name nor a relative's // (an ambiguous 1-token-different cell). classifyHolder → 'unknown' → this forms NO link (counted // nowhere), so a phantom relative never enters a published family figure. @@ -630,6 +676,27 @@ before(() => { person: 'Интер Иванов Тестов', institution: 'INT', }); + // §1.3: the divesting filing whose is UNREADABLE. `folder` carries 2023 and is the only thing that + // can date it. Dropping the record leaves the horizon at 2019, `divested` false, and a sold stake on the + // public surface — the failure direction that matters here, since the claim names a real official. + filings.push({ + folder: '2023', + xmlFile: 'BG1.xml', + year: 'н/д', + template: 'assets', + person: 'Безгодин Иванов Дивестов', + institution: 'T3', + }); + // POSITIVE CONTROL for the same fallback: a filing datable by NEITHER field must still be ignored, not + // guessed at. Дрън's stake stays published — an undatable filing is no evidence of a sale. + filings.push({ + folder: 'архив', + xmlFile: 'DR1.xml', + year: '', + template: 'assets', + person: 'Дрънкан Иванов Тестов', + institution: 'T4', + }); fs.writeFileSync( path.join(STAGING, 'filings.jsonl'), filings.map((f) => JSON.stringify(f)).join('\n') + '\n', @@ -659,6 +726,11 @@ before(() => { 181818187: { owners: ['АЛФА ИВАНОВ ПАРТНЬОРОВ', 'БЕТА ИВАНОВ ПАРТНЬОРОВ'] }, 121212129: { owners: ['НУЛА ИВАНОВ ТЕСТОВ'] }, 191919199: { owners: ['РОДНИНА ВТОРА'], seat: 'гр. Русе' }, // family + declared seat → confirmed + // Безгодин is ABSENT from the deed (so §7 reconciliation cannot reverse the divestment) but his + // declared seat matches the registered one, so rung 3 says „Потвърдено" and the link PUBLISHES. + // Only the folder-dated divestment withdraws it — making the pre-fix failure the dangerous one. + 212121218: { owners: ['ДРУГ СОБСТВЕНИК СЪВСЕМ'], seat: 'гр. София' }, + 232323231: { owners: ['ДРЪНКАН ИВАНОВ ТЕСТОВ'] }, // still the owner → the undatable filing changes nothing }); }); @@ -855,6 +927,17 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // that asset-declaration silence as a sale and WITHDRAWS the stake; the per-type horizon must not — no later // INTERESTS filing omits the company. This link must stay PUBLISHED. (Guards against dropping a true link: // 13% of holders declare a stake only in the interests declaration.) + // §1.3 unparseable filing YEAR: Безгодин's later declaration has an unreadable ('н/д') but a 2023 + // FOLDER. Dropping that record — the pre-fix behaviour — leaves his horizon at 2019, so `divested` stays + // false and a stake he no longer holds keeps naming him on the public surface. The folder must date it. + const noYear = link('212121218', 'Безгодин Иванов Дивестов'); + assert.equal(noYear.interest_class, 'private_ownership'); + assert.equal(noYear.status, 'withdrawn'); + // POSITIVE CONTROL: datable by NEITHER field ⇒ ignored, not guessed. The fallback must not become a + // licence to invent a horizon — an undatable filing is no evidence of a sale, so this link stays up. + const noDate = link('232323231', 'Дрънкан Иванов Тестов'); + assert.equal(noDate.status, 'published'); + const crossType = link('161616163', 'Интер Иванов Тестов'); assert.equal(crossType.interest_class, 'private_ownership'); assert.equal(crossType.status, 'published'); // NOT withdrawn — the later asset filing is a different type @@ -917,15 +1000,17 @@ test('re-run is idempotent and honors the suppression list (contested link stays 'suppressed', ); // idempotent: still exactly the same number of links + persons after a clean rebuild. - // 17 links: 12 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero - // 'withdrawn' + Интер's per-type-kept published link) + 2 family (Кмет's, now held for want of - // registry evidence, and Кметица's seat-confirmed one) + Канонов's canonicalized single link + + // 19 links: 14 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero + // 'withdrawn' + Безгодин's folder-dated 'withdrawn' + Дрънкан's undatable-filing 'published' + + // Интер's per-type-kept published link) + 2 family (Кмет's, now held for want of registry evidence, + // and Кметица's seat-confirmed one) + Канонов's canonicalized single link + // Алфа & Бета (two officials on one winner, ПАРТНЬОРИ 5); Мария (quarantined), Акционер (securities), // Двусмислен (unknown holder) & Безинст (empty institution) none. - assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 17); - // 20 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, - // zero-contract Нула and the two ПАРТНЬОРИ co-owners; Канонов's two institution-variant filings fold to ONE. - assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 20); + assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 19); + // 22 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, + // zero-contract Нула, the two ПАРТНЬОРИ co-owners and the two §1.3 filing-date cases; Канонов's two + // institution-variant filings fold to ONE. + assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 22); db.close(); }); From 8e6f1dc876cd7219152cc3694458c4c4f3228889 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:24:00 +0300 Subject: [PATCH 37/50] fix(cacbg): do not read an unresolvable holder column as an own stake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `colNum` always returns a fallback, so a table that renumbers the holder column resolves it to a column that is not in the row. `by[cHolder]` is then undefined, the caller reads '' and `classifyHolder` maps a blank cell to 'self' — its documented meaning. A RELATIVE's stake becomes the official's own, enters the OWN-only refutation path and publishes as their private_ownership. The distinction the parser needs is column RESOLVABLE-BUT-EMPTY (self, a blank cell does mean the declarant) versus column NOT RESOLVABLE (unknown — we never read the holder, so we make no claim). `colNumOrNull` supplies it for the holder column alone; the other five call sites keep `colNum` and its fallback verbatim. Scope: parse.mjs was byte-identical to main. It is touched here because #279 §5 item 4 scheduled §1.4 for this PR, and this defect fails toward publishing a stake against a person who did not declare it as theirs. --- scripts/cacbg/parse.mjs | 21 +++++++++++++++++---- scripts/cacbg/parse.test.mjs | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/scripts/cacbg/parse.mjs b/scripts/cacbg/parse.mjs index bf95f1548..6e89e2427 100644 --- a/scripts/cacbg/parse.mjs +++ b/scripts/cacbg/parse.mjs @@ -82,10 +82,18 @@ function cellsByNum(row) { } // find the @_Num of the first column whose @_Description matches `re` (labels live on the header row) function colNum(firstRow, re, fallback) { + const found = colNumOrNull(firstRow, re); + return found ?? fallback; +} +// The same resolution WITHOUT the fallback, for the one column where "I could not find it" and "I found it +// and it was blank" must stay distinguishable. `colNum` collapses them: an unresolvable column resolves to +// a Num that is not in the row, `by[col]` is undefined, and the caller reads an empty cell it never found. +// For the HOLDER column those two readings differ by who owns the stake (§1.4) — see its call site. +function colNumOrNull(firstRow, re) { for (const c of asArray(firstRow?.Cell)) { if (c?.['@_Num'] && re.test(String(c?.['@_Description'] ?? ''))) return c['@_Num']; } - return fallback; + return null; } const year4 = (s) => String(s ?? '').match(/\b(20\d{2})\b/)?.[1] ?? null; @@ -159,15 +167,20 @@ function parseAssets(pp) { ? colNum(rows[0], /наименование.*дружеств|фирма/i, '4') : colNum(rows[0], /емитент/i, '6'); const cSeat = colNum(rows[0], /седалище/i, '5'); - const cHolder = colNum(rows[0], /собствено.*фамил/i, isOod ? '7' : '8'); + // NO fallback here, unlike every other column (§1.4). A blank holder cell MEANS something — the stake is + // the declarant's own (classifyHolder) — so a column we failed to resolve must not imitate one. With a + // fallback, a renumbered table resolves the holder to a missing column, reads '' and calls a RELATIVE's + // stake the official's own, which then publishes as their private_ownership. Unresolvable ⇒ 'unknown', + // which forms no link at all: we did not read the holder, so we make no claim about them. + const cHolder = colNumOrNull(rows[0], /собствено.*фамил/i); const cEgn = colNum(rows[0], /^егн$/i, isOod ? '8' : '9'); for (const row of rows) { const by = cellsByNum(row); const company = by[cCompany] ?? ''; if (!company) continue; if ((by[cEgn] ?? '').length > 0) egnPresent = true; - const holder = by[cHolder] ?? ''; - const holderRelation = classifyHolder(holder, declarant); + const holderRelation = + cHolder === null ? 'unknown' : classifyHolder(by[cHolder] ?? '', declarant); if (holderRelation === 'related') familyHoldingCount += 1; const seat = isOod ? (by[cSeat] ?? '') : ''; interests.push({ diff --git a/scripts/cacbg/parse.test.mjs b/scripts/cacbg/parse.test.mjs index 30495481d..6a9b835e3 100644 --- a/scripts/cacbg/parse.test.mjs +++ b/scripts/cacbg/parse.test.mjs @@ -249,3 +249,35 @@ test('parseList: only values shaped like a declaration filename are announced', assert.equal(shape('Уведомление'), 0, 'a title slotted into the filename field'); assert.equal(shape('AAAA.pdf'), 0, 'not a declaration document'); }); + +test('§1.4: an UNRESOLVABLE holder column is unknown, not an own stake', () => { + // `colNum` always returns a fallback, so a table that renumbers the holder column away resolves it to a + // column that is not there: `by[cHolder]` is undefined → '' → classifyHolder('') → 'self'. A RELATIVE's + // stake then enters the OWN-only path and publishes as the official's private_ownership — the surface's + // worst failure, since the official did not declare that stake as theirs. + // + // The required distinction is column RESOLVABLE-BUT-EMPTY (→ self, a blank cell means the declarant) + // versus column NOT RESOLVABLE (→ unknown, we have not read the holder at all). + const renumbered = ` + 1 + "РЕНОМЕР" ЕООД + Мария Спасова Роднинска`; + const d = parseDeclaration(assetDecl({ rows: renumbered }), 'REN.xml'); + const it = d.interests.find((i) => i.entity === '"РЕНОМЕР" ЕООД'); + assert.equal(it.holderRelation, 'unknown'); + + // POSITIVE CONTROL 1 — the column resolves and the cell is genuinely EMPTY: still the declarant's own + // stake. Collapsing both cases to 'unknown' would silently drop every stake declared this way. + const blankHolder = ` + 1 + "ПРАЗНА" ЕООД + `; + const blank = parseDeclaration(assetDecl({ rows: blankHolder }), 'BLK.xml'); + assert.equal(blank.interests[0].holderRelation, 'self'); + + // POSITIVE CONTROL 2 — a resolvable column still classifies a relative and the declarant correctly, so + // the change bounds one case rather than blanketing the parser. + const ok = parseDeclaration(assetDecl({ rows: selfRow + familyRow }), 'OK.xml'); + assert.equal(ok.interests.find((i) => i.entity === '"ТЕСТ АГРО" ЕООД').holderRelation, 'self'); + assert.equal(ok.interests.find((i) => i.entity === '"ФАМИЛНА" ЕООД').holderRelation, 'related'); +}); From ce5b2ada4df81c2639ba4549b8cb2e7e73ed1afc Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:32:21 +0300 Subject: [PATCH 38/50] fix(tr): require the registry evidence to establish the COMPANY, not only the person MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0033 held that rung 2 needs no gate because "the register named this person in THIS company". That holds only if THIS company is the one the official declared, and it is not guaranteed. `resolveEntity` maps a declared фирма to the sole ЕИК among PROCUREMENT WINNERS, and `nameGloballyUnique` ranges over bidders — never the register at large. So an official who owns a same-named company that never bid resolves to an unrelated winner, and a three-token homonym in that winner's deed completes a link false in both halves. Two coincidences, neither rare in Bulgaria. The ladder moved the name-collision risk into the Register instead of closing it, which is the premise #279 rests on. Rung 2 now asks for a reason to believe the company is the declared one: a declared ЕИК (the identifier resolves it outright), a declared seat matching the registered one, or a фирма distinctive enough that a national twin is improbable. Anything else returns a new withholding kind `document_uncorroborated` — never publishing, carrying neither role nor matched_fact, since asserting either would leak the claim the rung just refused to make. It is a distinct kind rather than a fall-through to `unknown` because "matched a person, could not establish the company" and "matched nothing" are different facts, and the seal exists on held links precisely to keep that queue reviewable. It is counted in the run summary: the third corroborator is a bound, not a proof, and that number is what F8 reads to decide whether this tightens to strict ЕИК/seat corroboration. Strict was the alternative; declared seats come only from the ООД/ЕООД table of asset declarations, so its recall cost cannot be known before that measurement. `companyNameDistinctive` defaults to FALSE — this gates the primary publishing rung, so a caller that forgets it must withhold. Rungs 2 and 3 now share one `matchDeclaredSeat`, R10 included, so they cannot drift about what a seat match means. Mutation-checked three ways: never-withhold fails 4 tests, always-withhold fails 7 (the positive controls hold it to a bound), and dropping R10 from the shared helper fails 2. Covered end to end in load.test.mjs, not only as a pure function. ADR-0035 records it and supersedes ADR-0033 decision 2 on this point alone; the methodology page discloses the rule verbatim per ADR-0021 E10. --- apps/web/app/routes/conflict.methodology.tsx | 12 ++- ...-evidence-replaces-name-distinctiveness.md | 7 ++ ...vidence-must-also-establish-the-company.md | 81 ++++++++++++++ docs/adr/README.md | 1 + scripts/cacbg/load-collision.test.mjs | 20 +++- scripts/cacbg/load.mjs | 28 ++++- scripts/cacbg/load.test.mjs | 60 +++++++++-- scripts/tr/evidence.mjs | 96 ++++++++++++----- scripts/tr/evidence.test.mjs | 101 ++++++++++++++++++ 9 files changed, 367 insertions(+), 39 deletions(-) create mode 100644 docs/adr/0035-registry-evidence-must-also-establish-the-company.md diff --git a/apps/web/app/routes/conflict.methodology.tsx b/apps/web/app/routes/conflict.methodology.tsx index 152df8a7e..209f11f9f 100644 --- a/apps/web/app/routes/conflict.methodology.tsx +++ b/apps/web/app/routes/conflict.methodology.tsx @@ -201,7 +201,17 @@ export default function ConflictMethodology() {

  • Документ. И трите имена на лицето се срещат в живо поле на акта — управители, съдружници, едноличен собственик или физическо лице-търговец. Пазим - ролята (съдружник/собственик или само управител), защото тя не е едно и също. + ролята (съдружник/собственик или само управител), защото тя не е едно и също.{' '} + + Съвпадението по име само по себе си не стига: изисква се и нещо извън фирменото + наименование да установи, че дружеството е декларираното + {' '} + — деклариран ЕИК, декларирано седалище, съвпадащо с вписаното, или достатъчно + отличително фирмено наименование. Иначе връзката остава скрита. Причината е, че + дружеството се разпознава сред изпълнителите по обществени поръчки: ако + лицето притежава едноименно дружество, което никога не е кандидатствало, търсенето + би посочило чуждото — и съименник в неговия акт би „потвърдил" връзка, невярна и в + двете си части.
  • Потвърдено. Лицето не е намерено в акта, но декларирано от него diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md index 0993a41b8..82633fb76 100644 --- a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -138,6 +138,13 @@ Rung 2 asserts on a name match. We accept it, on these grounds and no wider: ships a libel. - **The company identity behind it passed preventive control.** The candidate ЕИК comes from the deterministic exact-name resolution of ADR-0008/#226, not from fuzzy matching. + + > **Superseded on this point by [ADR-0035](0035-registry-evidence-must-also-establish-the-company.md).** + > Deterministic is not the same as correct: that resolution ranges over PROCUREMENT WINNERS only, so an + > official whose real company never bid resolves to a same-named winner, and a homonym in the winner's + > deed completes a link false in both halves. Rung 2 now requires a corroborator for the COMPANY — + > declared ЕИК, a matching declared seat, or a distinctive фирма — and the uncorroborated remainder is + > withheld as `document_uncorroborated` and counted. - **It is corroborated, and that is the whole difference.** A bare TR name is spec §4's *weakest* join — no ЕГН, no birthdate, not even ADR-0026's `(name, ведомство)` grain. The registry-graph spike that explored this API concluded, for its own use, that no person-derived edge may reach a user-visible diff --git a/docs/adr/0035-registry-evidence-must-also-establish-the-company.md b/docs/adr/0035-registry-evidence-must-also-establish-the-company.md new file mode 100644 index 000000000..4ba01f701 --- /dev/null +++ b/docs/adr/0035-registry-evidence-must-also-establish-the-company.md @@ -0,0 +1,81 @@ +# ADR-0035 — Регистровото доказателство трябва да установи и ДРУЖЕСТВОТО, не само лицето + +- **Статус:** Прието +- **Дата:** 2026-08-12 +- **Обхват:** доказателственото стъпало 2 („Документ") — `scripts/tr/evidence.mjs`, `scripts/cacbg/load.mjs`. + Заменя единствено твърдението в решение 2 на + [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md), че стъпало 2 не носи допълнителен + гейт; всичко останало в него стои. + +## Контекст + +ADR-0033 записа за стъпало 2: „по-силното стъпало не е гейтнато — регистърът е назовал това лице в ТОВА +дружество, което прави ключа по име без значение". Прегледът на PR-а показа, че изводът важи само ако +„ТОВА дружество" е дружеството, което лицето е декларирало. Не е гарантирано. + +Веригата, по която се стига до акта: + +1. `resolveEntity` (`load.mjs`) свежда декларираното фирмено наименование до **едно ЕИК на ПЕЧЕЛИВШ** — + участник в обществена поръчка. Дружества, които никога не са участвали, не съществуват за резолвера. +2. `nameGloballyUnique` проверява уникалност **само в множеството на участниците**, не в Търговския + регистър. „Единствено" тук значи „единственото сред печелившите". + +Оттук следва сценарият, който прегледът описа и който възпроизведохме: + +- Длъжностното лице действително притежава «АЛФА ООД» с ЕИК **A**, което никога не е участвало в поръчка. +- Друго, несвързано «АЛФА ООД» с ЕИК **B** е печеливш и единственият участник с това наименование. + Резолверът избира **B**. +- Актът на **B** съдържа **съименник** на декларатора — същите три имена, което в България не е рядкост. + Стъпало 2 съвпада и публикува връзка, **невярна и в двете си половини**. + +Регистровото доказателство доказва, че *някой с това име* е съдружник в **B** — не че *това лице* е. +Тоест стъпалото **премести** риска от съвпадение на имена в регистъра, вместо да го затвори, а точно +затварянето му е основанието на #279. Съществуващият карантинен механизъм за двусмислени ключове покрива +само сблъсък печеливш-срещу-печеливш; случаят печеливш-срещу-непечеливш не беше нито обработен, нито +записан като приет остатъчен риск. + +## Решение + +**Преди стъпало 2 да твърди, нещо извън фирменото наименование трябва да каже, че дружеството е +декларираното.** Съвпадението по име остава необходимо, но вече не е достатъчно само по себе си. + +Приема се, ако е изпълнено **поне едно** от: + +| Потвърдител | Защо стига | +|---|---| +| деклариран **ЕИК** | националният идентификатор по ЗТРРЮЛНЦ решава дружеството еднозначно, независимо от споделена фирма (ADR-0028) | +| декларирано **седалище**, съвпадащо с вписаното | близнак в друго населено място отпада; същият факт, на който публикува стъпало 3, включително темпоралната гаранция R10 | +| **отличително** фирмено наименование (`nameDistinctiveness`) | национален близнак е малко вероятен изобщо да съществува | + +Съвпадение по име, което не мине гейта, връща **нов, отделен въздържащ се резултат +`document_uncorroborated`** — никога не публикува, не носи роля и не носи `matched_fact`. + +**Защо отделен вид, а не пропадане към `unknown`.** „Съвпаднахме лице, но не установихме дружеството" и +„не съвпаднахме нищо" са различни факти за една връзка. Печатът се пише за всяка връзка, включително +задържаните, именно за да е прегледаема опашката (migration 0006); сливането им я заслепява точно там, +където решението има нужда от числото. + +**Третият потвърдител е граница, не доказателство.** Затова се **брои** (`document_uncorroborated` в +обобщението на зареждането) — това е числото, което F8 чете, за да реши дали гейтът да се стегне до +първите два. + +**Отхвърлена алтернатива: строго потвърждаване** (само ЕИК или седалище). По-висока точност, но +декларирано седалище се извлича само от таблицата ООД/ЕООД на **имуществена** декларация — никога от +декларация за интереси — така че цената в покритие е неизвестна преди F8 и можеше да свали повърхността +под прага за публикуване. Остава на масата и е точно решението, което измереният остатък ще информира. + +## Последствия + +- Стъпало 2 вече е **гейтнато**, което ADR-0033 решение 2 изрично казваше, че не е. Това е единственото + твърдение оттам, което този ADR заменя; наборът от стъпала, редът им и правилото „първото съвпадение + печели" стоят. +- Гейтът се проверява по подразбиране **затворено**: `companyNameDistinctive` е `false`, ако не бъде + подадено. За разлика от `nameGloballyUnique` — чието разрешаващо подразбиране е ограничено до + най-слабото стъпало — този гейт пази основното публикуващо стъпало. +- Стъпала 2 и 3 споделят **една** реализация на съвпадението по седалище (`matchDeclaredSeat`), заедно с + R10. Две копия на „какво брои за съвпадение по седалище" рано или късно се разминават за това кои връзки + могат да се публикуват. +- Покритието пада с неизмерена величина — по построение точно върху връзките, за които дружеството е + най-слабо установено. Това е приетата посока: по-скоро задържана вярна връзка, отколкото публикувана + невярна, назоваваща действително лице. +- Методологичната страница оповестява правилото дословно (ADR-0021 E10). diff --git a/docs/adr/README.md b/docs/adr/README.md index f0a8193ba..0d63413fc 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -42,5 +42,6 @@ | [0032](0032-family-ownership-published-under-public-interest.md) | Дял на свързано лице се публикува на поименната повърхност наравно със собствения — на основание надделяващ обществен интерес (ЗДОИ чл. 41и; C-184/20 забранява само поименните данни); близкият никога не се назовава, връзката не се твърди; заменя ADR-0030 | Прието | | [0033](0033-registry-evidence-replaces-name-distinctiveness.md) | Връзка се публикува само срещу проверим факт от Търговския регистър — доказателственият ред заменя нивата по отличителност на името; евристиката вече обосновава твърдение (изрично изключение от ADR-0007), барът за акционерни форми е обединение от три сигнала, монотонността е гейт, не хранилище; заменя ADR-0009/0015/0017 | Прието (дизайн) | | [0034](0034-registry-lookups-and-decisions-share-one-monthly-run.md) | Справките в ТР и решенията текат в едно месечно задание — суровите актове не преживяват изпълнителя, а решението не може да се вземе без тях; заменя твърдението за каденцията в ADR-0033 | Прието | +| [0035](0035-registry-evidence-must-also-establish-the-company.md) | Стъпало 2 („Документ") публикува само ако извън фирменото наименование нещо установи, че дружеството е декларираното — деклариран ЕИК, съвпадащо седалище или отличителна фирма; иначе `document_uncorroborated`, което се брои; заменя твърдението в ADR-0033, че стъпало 2 не е гейтнато | Прието | Свързан проектен документ: [spec/related-persons-foundation.md](../spec/related-persons-foundation.md). diff --git a/scripts/cacbg/load-collision.test.mjs b/scripts/cacbg/load-collision.test.mjs index 8d709ec28..43fb1df44 100644 --- a/scripts/cacbg/load-collision.test.mjs +++ b/scripts/cacbg/load-collision.test.mjs @@ -49,7 +49,8 @@ function buildTrCache(owners) { raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); - for (const [eik, name] of Object.entries(owners)) { + for (const [eik, spec] of Object.entries(owners)) { + const { name, seat } = spec; const deed = { uic: eik, fullName: '"ФИКС" ЕООД', @@ -67,6 +68,17 @@ function buildTrCache(owners) { fieldEntryNumber: '20110502101007', fieldEntryDate: '2011-05-02T00:00:00', }, + // The REGISTERED seat, matching what this official declared. Both фирми here are + // generic („КОМПАНИЯ ЕДНО/ДВЕ" — two content words), so under ADR-0035 a name match + // alone cannot establish which company was declared; the agreeing seat is what does. + // Without it this fixture would exercise the withholding path instead of the + // cross-folder attribution it exists to test. + { + nameCode: 'CR_F_5_L', + htmlData: `

    Държава: БЪЛГАРИЯ
    Населено място: гр. ${seat}

    `, + fieldEntryNumber: '20110502101008', + fieldEntryDate: '2011-05-02T00:00:00', + }, ], }, ], @@ -111,7 +123,7 @@ before(() => { CREATE TABLE contracts(id TEXT PRIMARY KEY, tender_id TEXT, bidder_id TEXT, signed_at TEXT, amount_eur REAL); INSERT INTO authorities VALUES ('auth:1','ВЕДОМСТВО ТЕСТ'); INSERT INTO tenders VALUES ('t1','auth:1'),('t2','auth:1'); - -- Two distinct seat-confirmed single-ЕИК winners → both publish (A_seat), no ambiguity. + -- Two distinct single-ЕИК winners, each named in its own deed AND seat-corroborated → both publish. INSERT INTO bidders VALUES ('eik:100000001','КОМПАНИЯ ЕДНО ЕООД','100000001',1,'София'); INSERT INTO bidders VALUES ('eik:200000002','КОМПАНИЯ ДВЕ ЕООД','200000002',1,'Пловдив'); INSERT INTO contracts VALUES ('c1','t1','eik:100000001','2021-05-01',50000); @@ -160,8 +172,8 @@ before(() => { fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); buildTrCache({ - 100000001: 'ИВАН ПЪРВИ ТЕСТОВ', - 200000002: 'ПЕТЪР ВТОРИ ПРОБЕН', + 100000001: { name: 'ИВАН ПЪРВИ ТЕСТОВ', seat: 'София' }, + 200000002: { name: 'ПЕТЪР ВТОРИ ПРОБЕН', seat: 'Пловдив' }, }); }); diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index aecd680a1..31e627c95 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -16,7 +16,12 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; // nameDistinctiveness is deliberately NOT imported: the evidence ladder replaced it in the publish // path (ADR-0033). It survives in classify.mjs for the review queue, not for a publishing decision. -import { temporalStatus, localityToken, closelyHeldForm } from './classify.mjs'; +import { + temporalStatus, + localityToken, + closelyHeldForm, + nameDistinctiveness, +} from './classify.mjs'; import { openCache, readDeed, coverage } from '../tr/cache.mjs'; import { TR_DB, TR_RAW, deedPath } from '../tr/paths.mjs'; import { @@ -769,6 +774,12 @@ for (const rec of agg.values()) { firstDeclaredYear: declYears.length ? Math.min(...declYears) : null, scope: rec.scope, nameGloballyUnique: nameUnique, + // ADR-0035. The resolver picked this winner by фирма; `nameGloballyUnique` above only says no OTHER + // WINNER shares that name, which says nothing about the register at large. So an uncorroborated + // rung 2 additionally asks whether the фирма is one a national twin is unlikely to share. + // `rec.key` is the WINNER's registered name — the canonical spelling of the company we actually + // looked up, which is the identity in question. + companyNameDistinctive: nameDistinctiveness(rec.key) === 'distinctive', }); } catch (err) { // A deed we cannot parse is a deed we cannot reason about — so this link withholds, exactly as an @@ -1039,6 +1050,21 @@ const S = { r.n, ]), ), + // Every evidence rung's count, publishing and withholding alike (ADR-0033 decision 1). `publish_tier` + // IS the verdict kind since #279. + by_evidence_kind: Object.fromEntries( + q('SELECT publish_tier, COUNT(*) n FROM interest_links GROUP BY publish_tier').map((r) => [ + r.publish_tier, + r.n, + ]), + ), + // ADR-0035's residual: links where the register named this person in the resolved company but nothing + // established that the company is the declared one. This is the number F8's hand-labelled sample reads + // to decide whether the distinctiveness gate tightens to strict ЕИК/seat corroboration. Reported + // separately from the map above because it is a DECISION input, not just a tally. + document_uncorroborated: one( + "SELECT COUNT(*) n FROM interest_links WHERE publish_tier='document_uncorroborated'", + ).n, ambiguous_name_keys: ambiguousKeys.length, ambiguous_name_key_examples: ambiguousKeys, noMatch, diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index db60a560f..468f11fa7 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -179,6 +179,13 @@ before(() => { -- ignored, never guessed — an undatable filing is no evidence of a sale, so this stake stays published. INSERT INTO bidders VALUES ('eik:232323231','ДРЪНКАН ТЕХ 10 ЕООД','232323231',1,'София'); INSERT INTO contracts VALUES ('c18','t1','eik:232323231','2019-05-01',110000); + -- ADR-0035 winner-vs-non-winner homonym: „ХОМОНИМ ТРЕЙД" is a GENERIC фирма (two content words) and the + -- sole WINNER holding it. Хомоним Иванов Тестов declared a stake in a company of that name — but the one + -- he owns never bid, so the resolver lands on this winner instead. This winner's deed happens to name a + -- HOMONYM (identical three tokens), which under a name-only rung 2 „proves" a link false in both halves. + -- Nothing corroborates the company (no declared ЕИК, no declared seat), so it must be withheld. + INSERT INTO bidders VALUES ('eik:242424248','ХОМОНИМ ТРЕЙД ЕООД','242424248',1,'София'); + INSERT INTO contracts VALUES ('c19','t1','eik:242424248','2023-05-01',95000); -- N10 canonicalization: Канонов owns КАНОН ТЕХ 5, filing „МВР" one year and the full ministry name the -- next → ONE identity, ONE link (not a split). Distinctive name (number) → tier B publishable. INSERT INTO bidders VALUES ('eik:131313136','КАНОН ТЕХ 5 ЕООД','131313136',1,'София'); @@ -508,6 +515,24 @@ before(() => { holderRelation: 'self', controlHash: 'H21', }, + // ADR-0035: Хомоним's declared stake in a generic-named company, with NO ЕИК and NO seat declared. + { + folder: '2024', + xmlFile: 'HOM.xml', + year: '2023', + template: 'assets', + category: '', + institution: 'T5', + person: 'Хомоним Иванов Тестов', + position: '', + entity: 'ХОМОНИМ ТРЕЙД ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: '', + holderRelation: 'self', + controlHash: 'H22', + }, // B4 UNKNOWN holder: the holder cell is neither confidently the declarant's own name nor a relative's // (an ambiguous 1-token-different cell). classifyHolder → 'unknown' → this forms NO link (counted // nowhere), so a phantom relative never enters a published family figure. @@ -731,6 +756,9 @@ before(() => { // Only the folder-dated divestment withdraws it — making the pre-fix failure the dangerous one. 212121218: { owners: ['ДРУГ СОБСТВЕНИК СЪВСЕМ'], seat: 'гр. София' }, 232323231: { owners: ['ДРЪНКАН ИВАНОВ ТЕСТОВ'] }, // still the owner → the undatable filing changes nothing + // The homonym: the deed names someone with Хомоним's exact three tokens. Rung 2 matches — and must + // still withhold, because nothing says this is the company he declared. + 242424248: { owners: ['ХОМОНИМ ИВАНОВ ТЕСТОВ'] }, }); }); @@ -927,6 +955,24 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // that asset-declaration silence as a sale and WITHDRAWS the stake; the per-type horizon must not — no later // INTERESTS filing omits the company. This link must stay PUBLISHED. (Guards against dropping a true link: // 13% of holders declare a stake only in the interests declaration.) + // ADR-0035 — the CRITICAL, end to end. Хомоним declared a stake in „ХОМОНИМ ТРЕЙД ЕООД"; the company he + // actually owns never bid, so `resolveEntity` resolved his declaration to the same-named WINNER, whose + // deed names a person with his exact three tokens. Rung 2 matches. It must NOT publish: the register + // proves someone of that name owns THIS company, not that this is the company he declared. `nameGlobally- + // Unique` cannot catch it — it ranges over bidders, and this winner is the only bidder with the name. + const homonym = link('242424248', 'Хомоним Иванов Тестов'); + assert.equal(homonym.publish_tier, 'document_uncorroborated'); + assert.notEqual(homonym.status, 'published'); + // The seal must record WHY it was withheld, and must not carry the role the rung refused to assert. + const homonymSeal = db + .prepare( + 'SELECT evidence_kind, registry_role, matched_fact FROM interest_link_evidence WHERE link_key=?', + ) + .get(homonym.link_key); + assert.equal(homonymSeal.evidence_kind, 'document_uncorroborated'); + assert.equal(homonymSeal.registry_role, null); + assert.equal(homonymSeal.matched_fact, null); + // §1.3 unparseable filing YEAR: Безгодин's later declaration has an unreadable ('н/д') but a 2023 // FOLDER. Dropping that record — the pre-fix behaviour — leaves his horizon at 2019, so `divested` stays // false and a stake he no longer holds keeps naming him on the public surface. The folder must date it. @@ -1000,17 +1046,17 @@ test('re-run is idempotent and honors the suppression list (contested link stays 'suppressed', ); // idempotent: still exactly the same number of links + persons after a clean rebuild. - // 19 links: 14 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero + // 20 links: 15 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero // 'withdrawn' + Безгодин's folder-dated 'withdrawn' + Дрънкан's undatable-filing 'published' + - // Интер's per-type-kept published link) + 2 family (Кмет's, now held for want of registry evidence, + // Хомоним's ADR-0035 'document_uncorroborated' hold + Интер's per-type-kept published link) + 2 family (Кмет's, now held for want of registry evidence, // and Кметица's seat-confirmed one) + Канонов's canonicalized single link + // Алфа & Бета (two officials on one winner, ПАРТНЬОРИ 5); Мария (quarantined), Акционер (securities), // Двусмислен (unknown holder) & Безинст (empty institution) none. - assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 19); - // 22 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, - // zero-contract Нула, the two ПАРТНЬОРИ co-owners and the two §1.3 filing-date cases; Канонов's two - // institution-variant filings fold to ONE. - assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 22); + assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 20); + // 23 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, + // zero-contract Нула, the two ПАРТНЬОРИ co-owners, the two §1.3 filing-date cases and Хомоним; + // Канонов's two institution-variant filings fold to ONE. + assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 23); db.close(); }); diff --git a/scripts/tr/evidence.mjs b/scripts/tr/evidence.mjs index 51c4c7708..56bc01efd 100644 --- a/scripts/tr/evidence.mjs +++ b/scripts/tr/evidence.mjs @@ -89,6 +89,32 @@ function findPerson(deed, name, nameCodes) { return null; } +/** + * The registered seat, when it matches one THIS person declared for THIS company and was in force for the + * declared period. Shared by rung 3 (which publishes on it) and rung 2's company-identity corroborator. + * + * R10: seats move. A company that relocated INTO the declared settlement afterwards would confirm falsely, + * so the registered seat must predate the period. + * + * A null `firstDeclaredYear` FAILS the guard rather than skipping it. `load.mjs` passes null whenever no + * history row carried a parseable year, and an unknown year is not a satisfied temporal test — it is the + * absence of one. Reading it as „covers the period" made the weakest rung the only one with no temporal + * check, on exactly the links where we know least, and rung 4 already refuses to run without a year on the + * same ground. The undated-SEAT leg is different and stays: a seat with no entry date is the ordinary shape + * for a company that never moved, and it is checkable — a known year is still on the other side. + * + * @returns {{settlement:string, entryDate:string|null}|null} + */ +function matchDeclaredSeat(deed, declaredSeats, firstDeclaredYear) { + const seat = registrySeat(deed); + // Empty NEVER matches — otherwise every link with no seat data on either side rubber-stamps itself. + if (seat.settlement === '') return null; + if (firstDeclaredYear == null) return null; + if (seat.entryDate != null && seat.entryDate > `${firstDeclaredYear}-12-31`) return null; + const declared = declaredSeats.map(normalizeSettlement).filter((s) => s !== ''); + return declared.includes(seat.settlement) ? seat : null; +} + /** * Decide the evidence for one link. * @@ -104,6 +130,9 @@ function findPerson(deed, name, nameCodes) { * @param {number|null} [input.firstDeclaredYear] * @param {'self'|'family'} [input.scope] * @param {boolean} [input.nameGloballyUnique] AND-gate on the WEAKEST rung only + * @param {boolean} [input.companyNameDistinctive] the declared фирма is unlikely to have a national + * twin. Gates an UNCORROBORATED rung 2 (ADR-0035). + * Defaults to FALSE: a caller that forgets it withholds. * @returns {{kind:string, publishable:boolean, registryRole:string|null, matchedFact:string|null, * entryNumber:string|null, entryDate:string|null, rulesVersion:string, * shortName:boolean, latinInName:boolean}} @@ -118,6 +147,11 @@ export function evidenceVerdict(input) { firstDeclaredYear = null, scope = 'self', nameGloballyUnique = true, + // Fail-CLOSED, unlike `nameGloballyUnique` above. That one's permissive default is bounded — it gates + // only the weakest rung. This one gates the PRIMARY publishing rung, so a caller that forgets to pass + // it must withhold rather than publish a claim naming a real person against a company we did not + // establish. There is exactly one production caller (load.mjs) and it passes it explicitly. + companyNameDistinctive = false, } = input; const tokens = personTokens(declarantName); @@ -152,12 +186,43 @@ export function evidenceVerdict(input) { if (form.verdict === 'joint_stock') return verdict('bar_joint_stock', false); if (form.verdict === 'unknown') return verdict('unknown', false); + // The registered seat, matched against what THIS person declared for THIS company, with R10's temporal + // guard applied. Computed once and consumed by two rungs: rung 3 publishes „Потвърдено" on it, and rung 2 + // uses it as a COMPANY-IDENTITY corroborator. One implementation, because two copies of "what counts as a + // seat match" would eventually disagree about which links may be published. + const matchedSeat = matchDeclaredSeat(deed, declaredSeats, firstDeclaredYear); + // ── rung 2 ────────────────────────────────────────────────────────────────── // Only a full three-token name may assert. A Latin homoglyph makes the name a non-match rather than // a false match — company-name-key.ts's posture, applied to people. + // + // The company gate (ADR-0035). A name match proves someone with these three tokens is registered in the + // company we LOOKED UP — never that this is the company the official declared. `resolveEntity` picks the + // sole WINNER holding the declared name and `nameGloballyUnique` ranges over bidders only, so an official + // whose real company never bid resolves to a same-named winner, and a homonym in that winner's deed + // completes a link false in both halves. Before rung 2 may assert, something other than the фирма must + // say the company is the declared one: + // • the declarant wrote the ЕИК — the national identifier resolves it outright (ADR-0028); or + // • the declared seat matches the registered one — a twin in another town is excluded; or + // • the фирма is distinctive enough that a national twin is improbable in the first place. + // The third is a bound, not a proof, and it is COUNTED (`documentUncorroborated`) so F8 can decide from + // the measured residual whether to tighten to the first two. Strict corroboration was the alternative; + // declared seats exist only on the ООД/ЕООД table of asset declarations, so its recall cost cannot be + // known before that measurement. + const companyCorroborated = declaredEik || matchedSeat != null; const eligibleForDocument = !telemetry.shortName && !telemetry.latinInName; if (eligibleForDocument) { const owner = findPerson(deed, declarantName, OWNERSHIP_FIELDS); + const manager = owner ? null : findPerson(deed, declarantName, [MANAGER_FIELD]); + const hit = owner ?? manager; + if (hit && !companyCorroborated && !companyNameDistinctive) { + // A DISTINCT withholding kind, not a fall-through to `unknown`. „We matched a person but could not + // establish the company" and „we matched nothing" are different facts about a link, and the review + // queue (which is sealed for held links precisely to be reviewable) has to be able to tell them + // apart. It never publishes, and it carries no role or fact — asserting either would leak the very + // claim the rung just refused to make. + return verdict('document_uncorroborated', false); + } if (owner) { return verdict('document', true, { registryRole: 'owner', @@ -166,7 +231,6 @@ export function evidenceVerdict(input) { entryDate: owner.entryDate, }); } - const manager = findPerson(deed, declarantName, [MANAGER_FIELD]); if (manager) { return verdict('document', true, { registryRole: 'manager', @@ -190,31 +254,11 @@ export function evidenceVerdict(input) { // The SEAT leg is name-gated, and only this one. ADR-0017's holding carried forward: a name shared by // two ЕИК cannot support a name-derived identity claim. The seat still rescues a GENERIC name — that // is the whole point of the rung — it just cannot rescue a NATIONALLY SHARED one. - if (nameGloballyUnique) { - const seat = registrySeat(deed); - // Empty NEVER confirms — otherwise every link with no seat data on either side rubber-stamps. - if (seat.settlement !== '') { - const declared = declaredSeats.map(normalizeSettlement).filter((s) => s !== ''); - // R10: the registered seat must have been in force for the declared period. Seats move, and a - // company that relocated INTO the declared settlement afterwards would otherwise confirm falsely. - // - // A null `firstDeclaredYear` FAILS the guard rather than skipping it. `load.mjs` passes null - // whenever no history row carried a parseable year, and an unknown year is not a satisfied - // temporal test — it is the absence of one. Reading it as „covers the period" made the weakest - // rung the only one with no temporal check, on exactly the links where we know least, and rung 4 - // below already refuses to run without a year (`firstDeclaredYear != null`) on the same ground. - // The undated-seat leg is different and stays: a seat with no entry date is the ordinary shape - // for a company that never moved, and it is checkable — a known year is still on the other side. - const seatCoversPeriod = - firstDeclaredYear != null && - (seat.entryDate == null || seat.entryDate <= `${firstDeclaredYear}-12-31`); - if (seatCoversPeriod && declared.includes(seat.settlement)) { - return verdict('confirmed', true, { - matchedFact: `seat:${seat.settlement}`, - entryDate: seat.entryDate, - }); - } - } + if (nameGloballyUnique && matchedSeat != null) { + return verdict('confirmed', true, { + matchedFact: `seat:${matchedSeat.settlement}`, + entryDate: matchedSeat.entryDate, + }); } // ── rung 4 ────────────────────────────────────────────────────────────────── diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs index 4de42a931..3010698c2 100644 --- a/scripts/tr/evidence.test.mjs +++ b/scripts/tr/evidence.test.mjs @@ -41,6 +41,9 @@ const base = { firstDeclaredYear: 2021, scope: 'self', nameGloballyUnique: true, + // The company was resolved by a name unlikely to have a national twin. The rung-2 tests below are about + // NAME matching inside a deed, so they hold this dimension fixed; the gate itself is tested separately. + companyNameDistinctive: true, }; test('RULES_VERSION is a stable, non-empty identifier — §8 hangs off it', () => { @@ -270,6 +273,104 @@ test('rung 3 — name uniqueness does NOT gate the stronger „Документ" ); }); +// ── rung 2's company gate: the winner-vs-non-winner homonym (ADR-0035) ──────── +// +// The ladder proves a person with these three tokens is registered in the company we LOOKED UP. It cannot +// prove that company is the one the official declared. `resolveEntity` maps a declared name to the sole +// WINNER holding it, and `nameGloballyUnique` ranges over procurement bidders only — never the whole +// register. So when an official owns a same-named company that never bid, we resolve to the winner, and a +// three-token homonym in the winner's deed „proves" a link that is false in both halves. +// +// Two coincidences, and neither is rare in Bulgaria: a shared фирма and a shared three-part name. The gate +// asks for a reason to believe the COMPANY is the declared one before rung 2 may assert. +test('rung 2 — a GENERIC company name with no corroboration cannot publish on a name match alone', () => { + const v = evidenceVerdict({ ...base, companyNameDistinctive: false }); + assert.equal( + v.kind, + 'document_uncorroborated', + 'the deed names SOMEONE with this name in THIS company — not that this is the declared company', + ); + assert.equal(v.publishable, false); + // It must not fall through to `unknown`: the residual is the input to F8's decision on whether this gate + // tightens, and a rung-2 match withheld for want of company identity is a different fact from no match. + assert.equal(v.registryRole, null); + assert.equal(v.matchedFact, null); +}); + +test('rung 2 — a declared ЕИК corroborates the company, so the name match publishes', () => { + // POSITIVE CONTROL. The ЕИК IS the identity (ЗТРРЮЛНЦ, ADR-0028) — it resolves the company behind any + // shared фирма, which is exactly the collision the gate is about. + const v = evidenceVerdict({ ...base, companyNameDistinctive: false, declaredEik: true }); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); + assert.equal(v.registryRole, 'owner'); +}); + +test('rung 2 — a declared seat matching the registered seat corroborates the company', () => { + // POSITIVE CONTROL. The declarant put this company in гр. Пловдив and the register agrees; a national + // twin in another town is excluded by the same fact rung 3 publishes on. + const v = evidenceVerdict({ + ...base, + companyNameDistinctive: false, + declaredSeats: ['гр. Пловдив'], + }); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); +}); + +test('rung 2 — a DISTINCTIVE company name publishes uncorroborated (the gate is not a blanket)', () => { + // POSITIVE CONTROL, and the one that distinguishes this fix from disabling rung 2. A predicate that + // always withheld would satisfy the bar above; this is what it must NOT do. + const v = evidenceVerdict({ ...base, companyNameDistinctive: true }); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); +}); + +test('rung 2 — a seat that does NOT match cannot corroborate a generic name', () => { + // The corroborator has to actually corroborate. A declared seat in another town is evidence AGAINST the + // company being the declared one, so it certainly cannot rescue the rung. + const v = evidenceVerdict({ + ...base, + companyNameDistinctive: false, + declaredSeats: ['гр. Бургас'], // the deed says Пловдив + }); + assert.equal(v.kind, 'document_uncorroborated'); + assert.equal(v.publishable, false); +}); + +test('rung 2 — the seat corroborator carries the SAME temporal guard as rung 3 (R10)', () => { + // A seat registered after the declared period cannot corroborate anything: the company may have moved + // INTO that town afterwards. Rung 3 already refuses it; rungs 2 and 3 share one implementation so they + // cannot drift into disagreeing about what a seat match means. + const moved = deed([ + fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ')), + fld( + 'CR_F_5_L', + container('Държава: БЪЛГАРИЯ
    Населено място: гр. Пловдив, п.к. 4000'), + '2023-07-01T00:00:00', + ), + ]); + const v = evidenceVerdict({ + ...base, + deed: moved, + companyNameDistinctive: false, + declaredSeats: ['гр. Пловдив'], + firstDeclaredYear: 2021, + }); + assert.equal(v.kind, 'document_uncorroborated'); +}); + +test('rung 2 — the company gate never rescues a link rung 1 has barred', () => { + // Ordering: a joint-stock bar outranks everything, corroborated or not. The gate adds a way to WITHHOLD, + // never a way to publish something a stronger rung refused. + const ad = deed([fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))], { + legalForm: 5, + fullName: '"ГАМА ИНВЕСТ" АД', + }); + const v = evidenceVerdict({ ...base, deed: ad, companyNameDistinctive: true, declaredEik: true }); + assert.equal(v.kind, 'bar_joint_stock'); +}); + // ── rung 4: „Оборена" ───────────────────────────────────────────────────────── test('rung 4 — absent from a deed whose ownership predates the declaration refutes the link', () => { const older = deed([ From e9c17b980fb95052b07b3000dc61ce94239b6b92 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:42:28 +0300 Subject: [PATCH 39/50] feat(db): constrain the publishing-gate enums and fix the declaration natural key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #279 §2 asked for CHECK constraints and a non-null control_hash. The enums are done as asked; the control_hash half is not, and the reason matters. Enums. `status` and `interest_class` (0003) and `evidence_kind`/`registry_role`/ `live_status` (0006) were plain TEXT while the surface reads them as enums. A value that only LOOKS like a gate value — 'published ' with a trailing space, the case the ticket names — passes every writer and then fails `status = 'published'` silently. A CHECK binds every writer at once, including a hand-run UPDATE during an incident, which is exactly when it gets typed. 0006 is unshipped in this PR, so its constraints go on directly. control_hash NOT NULL is REJECTED, and the underlying defect fixed differently. The field is genuinely optional at the source — the register omits and parse.mjs carries that through as null — so NOT NULL converts a missing optional field into a run-stopping loader failure, and a fabricated placeholder would assert an integrity check nobody performed. The real defect is that `UNIQUE (xml_file, control_hash)` never constrained re-import: SQLite counts NULLs as DISTINCT, so every hashless declaration was mutually unique and re-imported as a fresh row each run, double-counting its stakes. Replaced by a unique index over (xml_file, folder_year, COALESCE(control_hash,'')) — the same natural key `id` already encodes, since the register reuses basenames across folders. 0007 retrofits both onto a deployed D1. `CREATE TABLE IF NOT EXISTS` never revisits an existing table and ship-related-persons wipes ROWS, not definitions, so a live database would otherwise keep the unconstrained 0003 shape forever however often the data reloads. interest_links needs a rebuild (SQLite cannot add a CHECK in place) under defer_foreign_keys, since interest_link_evidence references it and D1 enforces FKs; declarations needs only the corrected index. Both converge on re-application, which the bare `d1 execute --file` path requires, and both workflows apply it. Tests cover the trailing-space case by name, the ADR-0035 kind, the hashless duplicate that used to slip through, and a pre-0007 database that keeps its rows through the retrofit and survives a second application. Positive controls throughout: every real status and class still inserts, and the natural key still admits the same basename in a different folder and a corrected re-filing under a new hash. --- .github/workflows/deploy.yml | 8 + .github/workflows/related-persons-data.yml | 8 + .../0003_related_persons_foundation.sql | 33 ++- .../0006_interest_link_evidence.sql | 17 +- .../0007_publishing_gate_constraints.sql | 103 ++++++++++ packages/db/src/migrations.test.ts | 188 ++++++++++++++++++ 6 files changed, 349 insertions(+), 8 deletions(-) create mode 100644 packages/db/migrations/0007_publishing_gate_constraints.sql diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f52927c9d..e05a6fd34 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -151,6 +151,14 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0006_interest_link_evidence.sql + # 0007 retrofits the publishing-gate CHECK constraints and control_hash NOT NULL onto an + # ALREADY-PROVISIONED D1 (#279 §2). 0003 declares them now, but `CREATE TABLE IF NOT EXISTS` + # never revisits an existing table and the ship step wipes ROWS, not definitions — so without + # this a live database keeps the unconstrained shape no matter how often the data reloads. + # Rebuild-based and therefore idempotent: re-applying copies an already-constrained table. + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0007_publishing_gate_constraints.sql # The base schema was created out-of-band with `d1 execute --file`, so wrangler's migration # ledger is empty and `d1 migrations apply` would collide on 0000. Probe the actual table # instead. SQLite has no `ADD COLUMN IF NOT EXISTS`; a completion-marker table is created only diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index e352a7875..8472b96f5 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -309,6 +309,14 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0006_interest_link_evidence.sql + # 0007 retrofits the publishing-gate CHECK constraints and control_hash NOT NULL onto an + # ALREADY-PROVISIONED D1 (#279 §2). 0003 declares them now, but `CREATE TABLE IF NOT EXISTS` + # never revisits an existing table and the ship step wipes ROWS, not definitions — so without + # this a live database keeps the unconstrained shape no matter how often the data reloads. + # Rebuild-based and therefore idempotent: re-applying copies an already-constrained table. + pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0007_publishing_gate_constraints.sql - name: Ship свързани-лица tables → D1 # ship-related-persons.mjs shells out to `wrangler d1 execute` with cwd=apps/web. Two things it diff --git a/packages/db/migrations/0003_related_persons_foundation.sql b/packages/db/migrations/0003_related_persons_foundation.sql index 6ad3be975..58a858788 100644 --- a/packages/db/migrations/0003_related_persons_foundation.sql +++ b/packages/db/migrations/0003_related_persons_foundation.sql @@ -29,9 +29,24 @@ CREATE TABLE IF NOT EXISTS declarations ( category TEXT, -- CACBG category (e.g. Народни представители) institution TEXT, -- the body the official serves in position TEXT, -- declared position - source_url TEXT NOT NULL, -- register.cacbg.bg// — provenance - UNIQUE (xml_file, control_hash) + source_url TEXT NOT NULL -- register.cacbg.bg// — provenance ); +-- Re-import idempotence (#279 §2). This REPLACES a table-level `UNIQUE (xml_file, control_hash)`, which +-- did not constrain anything it was meant to: +-- 1. SQLite counts NULLs as DISTINCT, and `control_hash` is genuinely optional at the source — the +-- register omits on some declarations and parse.mjs carries that through as null. So +-- every hashless declaration was mutually unique and re-imported as a fresh row each run, +-- double-counting the stakes it carries. COALESCE folds them to one value. +-- 2. `xml_file` is not unique ACROSS FOLDERS — the register splits years into suffixed folders and +-- reuses basenames — which load.mjs already knows, since it namespaces the declaration id by folder +-- for exactly this reason. Without folder_year here the two keys disagree, and the narrower one +-- would reject a second official's genuinely different declaration. +-- +-- `control_hash` deliberately stays NULLABLE. NOT NULL was the other candidate, but it converts an +-- optional source field into a run-stopping loader failure, and fabricating a hash to satisfy it would +-- assert an integrity check we never performed. +CREATE UNIQUE INDEX IF NOT EXISTS idx_declarations_natural_key + ON declarations(xml_file, folder_year, COALESCE(control_hash, '')); -- Per-person lookup: audit.mjs scans a person's declarations, and LINK_SELECT's source_url subquery -- filters declarations by person_id — without this it's a full scan of the table. CREATE INDEX IF NOT EXISTS idx_declarations_person ON declarations(person_id); @@ -75,7 +90,12 @@ CREATE TABLE IF NOT EXISTS interest_links ( -- management_role — self, relation manages, a single declarant (ambiguous: private manager or small board) -- Materiality: only CLOSELY-HELD forms (ООД/ЕООД/ЕТ/…) reach ownership classes; listed АД/ЕАД securities -- and management-only roles never become private_ownership/family_ownership (ADR-0022). - interest_class TEXT NOT NULL DEFAULT 'management_role', -- fail-closed: a NON-surfaced class, so a writer that sets status but forgets the class can't leak to the public surface (load.mjs always sets it explicitly) + -- fail-closed: a NON-surfaced class, so a writer that sets status but forgets the class can't leak to + -- the public surface (load.mjs always sets it explicitly). CHECKed for the same reason as `status`: + -- the surface reads this as an enum, so a typo'd class is a gate failure, not a cosmetic one. + interest_class TEXT NOT NULL DEFAULT 'management_role' + CHECK (interest_class IN ('private_ownership','family_ownership', + 'ex_officio_board','management_role')), contemporaneous INTEGER NOT NULL DEFAULT 0, own_institution TEXT NOT NULL DEFAULT 'none', -- exact (deterministic) | locality (heuristic) | none evidence_count INTEGER NOT NULL DEFAULT 1, -- # declared_interests supporting this link @@ -87,7 +107,12 @@ CREATE TABLE IF NOT EXISTS interest_links ( contract_value_eur REAL, -- SUM(contracts.amount_eur); NULL if none summable first_contract_year TEXT, last_contract_year TEXT, - status TEXT NOT NULL DEFAULT 'held', -- published (public surface: private/family ownership) | internal (non-surfaced class) | held | withdrawn | suppressed + -- CHECKed, not merely documented (#279 §2). The public surface is `status = 'published'`, so a value + -- that only LOOKS like one — 'published ' with a trailing space is the case the ticket names — passes + -- every writer and then fails the gate silently, or slips a LIKE. A CHECK binds every writer at once, + -- including a hand-run UPDATE during an incident, which is exactly when this is most likely typed. + status TEXT NOT NULL DEFAULT 'held' + CHECK (status IN ('published','internal','held','withdrawn','suppressed')), verified_by TEXT, verified_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')) diff --git a/packages/db/migrations/0006_interest_link_evidence.sql b/packages/db/migrations/0006_interest_link_evidence.sql index a7b83033c..e91869d1e 100644 --- a/packages/db/migrations/0006_interest_link_evidence.sql +++ b/packages/db/migrations/0006_interest_link_evidence.sql @@ -20,9 +20,16 @@ CREATE TABLE IF NOT EXISTS interest_link_evidence ( link_key TEXT PRIMARY KEY REFERENCES interest_links(link_key), - -- document | confirmed | refuted | bar_joint_stock | unknown | outside_tr (ADR-0033 decision 1) - evidence_kind TEXT NOT NULL, - registry_role TEXT, -- owner | manager | NULL — only meaningful for evidence_kind='document' + -- ADR-0033 decision 1, plus document_uncorroborated (ADR-0035: a name match whose COMPANY nothing + -- corroborated). CHECKed because the read gate filters on this column — SURFACED_OWNERSHIP admits + -- exactly 'document' and 'confirmed' — so an unlisted value is either a link that silently stops + -- surfacing or, if it collides with a publishing name, one that surfaces unproven. + evidence_kind TEXT NOT NULL + CHECK (evidence_kind IN ('document','confirmed','document_uncorroborated','refuted', + 'bar_joint_stock','unknown','outside_tr')), + -- owner | manager | NULL — only meaningful for evidence_kind='document'. The card renders this as + -- „вписан като …", so a stray value becomes a public claim about a named person's registry role. + registry_role TEXT CHECK (registry_role IS NULL OR registry_role IN ('owner','manager')), matched_fact TEXT, -- the closed vocabulary above. NEVER a name. -- TEXT, not INTEGER: a fieldEntryNumber like 20130716101007 is already 14 digits and exceeds the -- exact-integer range once it round-trips through JSON/JS. @@ -33,7 +40,9 @@ CREATE TABLE IF NOT EXISTS interest_link_evidence ( -- live | terminated_owner_still | terminated_manager_still | terminated. -- RE-DERIVED on every run and deliberately NOT part of the seal's permanence: it asserts a present -- tense about a named person, and its freshness is bounded by the cache refresh cycle (ADR-0033 R3). - live_status TEXT NOT NULL, + live_status TEXT NOT NULL + CHECK (live_status IN ('live','terminated_owner_still','terminated_manager_still', + 'terminated')), sealed_at TEXT NOT NULL DEFAULT (datetime('now')) ); diff --git a/packages/db/migrations/0007_publishing_gate_constraints.sql b/packages/db/migrations/0007_publishing_gate_constraints.sql new file mode 100644 index 000000000..ac682308c --- /dev/null +++ b/packages/db/migrations/0007_publishing_gate_constraints.sql @@ -0,0 +1,103 @@ +-- Retrofit the publishing-gate constraints onto an ALREADY-DEPLOYED database (#279 §2). +-- +-- WHY THIS EXISTS AT ALL, given 0003 and 0006 now declare the same constraints: +-- `CREATE TABLE IF NOT EXISTS` is a no-op against a table that already exists, and +-- `scripts/ship-related-persons.mjs` wipes ROWS (`DELETE FROM`, WIPE_ORDER) and never table +-- definitions. So every environment that applied 0003 before today keeps the unconstrained shape +-- forever, no matter how many times the data is reloaded. Only a rebuild can change it. +-- +-- WHY IT IS SAFE TO RE-RUN: migrations here are applied by a bare `wrangler d1 execute --file` with no +-- applied-migrations tracking (see related-persons-data.yml / deploy.yml), so re-application MUST be a +-- no-op. Both blocks below are written to converge: the index block is `IF NOT EXISTS` over a de-duplicated +-- table, and the rebuild block copies the CURRENT table into a constrained one — run a second time it +-- copies an already-constrained table into an identical one. Idempotent by construction, no version flag. +-- +-- WHY A REBUILD FOR interest_links AND NOT `ALTER TABLE`: SQLite cannot add a CHECK in place, so +-- create-copy-drop-rename is the documented route. `declarations` needs no rebuild — its defect was a +-- constraint that failed to constrain, which a correct index fixes without touching the table. +-- +-- FOREIGN KEYS: `interest_link_evidence` references interest_links(link_key), and D1 enforces foreign +-- keys, so dropping the parent mid-rebuild would be refused. `defer_foreign_keys` postpones the check to +-- the end of the transaction — the documented way to rebuild a referenced table — by which point the +-- renamed table satisfies it. D1 runs each file in one implicit transaction, so a failure rolls the whole +-- file back rather than leaving a half-swapped table. +PRAGMA defer_foreign_keys = true; + +-- ── declarations ──────────────────────────────────────────────────────────────────────────────────── +-- The table-level `UNIQUE (xml_file, control_hash)` did not constrain what it was written for. SQLite +-- counts NULLs as DISTINCT and `control_hash` is genuinely optional at the source (the register omits +-- on some declarations), so every hashless declaration was mutually unique and re-imported +-- as a NEW row on each run, double-counting the stakes it carries. `xml_file` is also not unique across +-- folders — the register reuses basenames per year — which load.mjs already handles by namespacing the +-- declaration id with the folder. +-- +-- Replaced by an expression index over COALESCE(control_hash, '') plus folder_year: NULL-proof, and the +-- same natural key `id` already encodes. The column stays NULLABLE on purpose — NOT NULL would convert an +-- optional source field into a run-stopping loader failure, and a fabricated placeholder hash would +-- assert an integrity check nobody performed. +-- +-- No table rebuild is needed for this one: dropping the old constraint means dropping the table it lives +-- on, but the same effect is had by creating the correct index — and the stale UNIQUE, being strictly +-- weaker, rejects nothing the new index accepts. Duplicates ALREADY stored under the old non-constraint +-- would block the new index, so they are collapsed first, keeping the earliest row per natural key. +DELETE FROM declarations WHERE id NOT IN ( + SELECT MIN(id) FROM declarations GROUP BY xml_file, folder_year, COALESCE(control_hash, '') +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_declarations_natural_key + ON declarations(xml_file, folder_year, COALESCE(control_hash, '')); + +-- ── interest_links ────────────────────────────────────────────────────────────────────────────────── +-- `status` and `interest_class` ARE the publishing gate. The public surface is `status = 'published'` +-- AND a surfaced `interest_class`, so a value that merely resembles one — 'published ' with a trailing +-- space, the case #279 §2 names — passes every writer and then fails the gate silently. A CHECK binds +-- every writer at once, including a hand-run UPDATE during an incident, which is when it is most likely +-- to be typed. +-- +-- Rows violating the new CHECKs are dropped by INSERT OR IGNORE rather than rewritten: a status we cannot +-- interpret is not a link we should publish or guess at, and the loader rebuilds the table on every run. +CREATE TABLE IF NOT EXISTS interest_links_0007 ( + id TEXT PRIMARY KEY, + link_key TEXT NOT NULL UNIQUE, + person_id TEXT NOT NULL REFERENCES persons(id), + bidder_id TEXT NOT NULL, + eik TEXT NOT NULL, + entity_key TEXT NOT NULL, + match_method TEXT, + matcher_version TEXT NOT NULL, + publish_tier TEXT NOT NULL, + relation TEXT NOT NULL, + interest_class TEXT NOT NULL DEFAULT 'management_role' + CHECK (interest_class IN ('private_ownership','family_ownership', + 'ex_officio_board','management_role')), + contemporaneous INTEGER NOT NULL DEFAULT 0, + own_institution TEXT NOT NULL DEFAULT 'none', + evidence_count INTEGER NOT NULL DEFAULT 1, + first_declared_year TEXT, + last_declared_year TEXT, + contract_count INTEGER NOT NULL DEFAULT 0, + contract_value_eur REAL, + first_contract_year TEXT, + last_contract_year TEXT, + status TEXT NOT NULL DEFAULT 'held' + CHECK (status IN ('published','internal','held','withdrawn','suppressed')), + verified_by TEXT, + verified_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); +INSERT OR IGNORE INTO interest_links_0007 + (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, + relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, + last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, + status, verified_by, verified_at, created_at) +SELECT id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, + relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, + last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, + status, verified_by, verified_at, created_at +FROM interest_links +WHERE status IN ('published','internal','held','withdrawn','suppressed') + AND interest_class IN ('private_ownership','family_ownership','ex_officio_board','management_role'); +DROP TABLE interest_links; +ALTER TABLE interest_links_0007 RENAME TO interest_links; +CREATE INDEX IF NOT EXISTS idx_interest_links_eik ON interest_links(eik); +CREATE INDEX IF NOT EXISTS idx_interest_links_person ON interest_links(person_id); +CREATE INDEX IF NOT EXISTS idx_interest_links_status ON interest_links(status); diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index fed25dda4..1997bbd29 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -12,6 +12,7 @@ const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_ const migration2 = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration7 = resolve(root, 'packages/db/migrations/0007_publishing_gate_constraints.sql'); const backfill = resolve(root, 'scripts/backfill-current-value-currency.sql'); const precompute = resolve(root, 'scripts/precompute.sql'); @@ -205,4 +206,191 @@ describe('served migrations', () => { rmSync(dir, { recursive: true, force: true }); } }); + + // #279 §2: the publishing gate reads these columns as enums, but the schema let them hold anything. + // A value that merely LOOKS like a gate value — the trailing space in 'published ' §2 names — passes + // every writer and then silently fails `status = 'published'` (or worse, passes a LIKE somewhere). + // A CHECK is the only place this can be enforced for every writer at once, including a hand-run UPDATE. + it('constrains the publishing-gate enums and the declaration identity key', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + for (const m of [migration0, migration3, migration6, migration7]) readScript(dbPath, m); + + const seedLink = (status: string) => + sqlite( + dbPath, + `INSERT INTO persons(id, name) VALUES ('person:a', 'А') ON CONFLICT DO NOTHING; + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:${status}', '${status}', 'person:a', 'eik:201122335', '201122335', + 'АЛФА СТРОЙ ООД', 'test', 'document', 'owns', 'private_ownership', '${status}');`, + ); + + // §2's named case: 'published ' is not 'published'. It must be rejected, not stored. + expect(() => seedLink('published ')).toThrow(/CHECK/i); + expect(() => seedLink('publushed')).toThrow(/CHECK/i); + // POSITIVE CONTROL — every real status still inserts. A CHECK that rejected everything would pass + // the assertions above while breaking the loader. + for (const s of ['published', 'held', 'withdrawn', 'suppressed', 'internal']) + expect(() => seedLink(s)).not.toThrow(); + + // interest_class gates the surface just as hard: a non-surfaced class is what keeps a management + // role off the public page, so a typo'd class is a leak in the same way. + expect(() => + sqlite( + dbPath, + `INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:c', 'c', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership ', 'held');`, + ), + ).toThrow(/CHECK/i); + + // evidence_kind decides whether a link may be read at all (SURFACED_OWNERSHIP). ADR-0035's + // document_uncorroborated must be a legal value — and a misspelling must not be. + const seal = (kind: string) => + sqlite( + dbPath, + `INSERT INTO interest_link_evidence + (link_key, evidence_kind, lookup_date, rules_version, live_status) + VALUES ('${kind}', '${kind}', '2026-08-12', 'tr-rules-1', 'live');`, + ); + expect(() => seal('document_uncorroberated')).toThrow(/CHECK|FOREIGN KEY/i); + for (const k of ['document', 'confirmed', 'document_uncorroborated', 'refuted']) + expect(() => { + sqlite( + dbPath, + `INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:${k}', '${k}', 'person:a', 'eik:1', '1', 'X', 't', '${k}', 'owns', + 'private_ownership', 'held'); + INSERT INTO interest_link_evidence + (link_key, evidence_kind, lookup_date, rules_version, live_status) + VALUES ('${k}', '${k}', '2026-08-12', 'tr-rules-1', 'live');`, + ); + }).not.toThrow(); + + // §2: `UNIQUE (xml_file, control_hash)` did not constrain re-import. SQLite counts NULLs as + // DISTINCT and control_hash is genuinely optional at the source, so two hashless imports of the + // SAME declaration both inserted and double-counted the stakes it carries. + const decl = (id: string, hash: string | null, folder = '2023') => + sqlite( + dbPath, + `INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('${id}', 'person:a', 'A.xml', ${hash === null ? 'NULL' : `'${hash}'`}, + '${folder}', 'assets', 'https://x/A.xml');`, + ); + expect(() => decl('d:n1', null)).not.toThrow(); + expect(() => decl('d:n2', null)).toThrow(/UNIQUE/i); // the case that used to slip through + expect(() => decl('d:1', 'H1')).not.toThrow(); + expect(() => decl('d:2', 'H1')).toThrow(/UNIQUE/i); + // POSITIVE CONTROLS — the key must not over-constrain. The register reuses basenames across + // FOLDERS, so the same xml_file in a different folder is a DIFFERENT declaration (load.mjs + // namespaces its id for this reason), and a genuinely different hash is a corrected re-filing. + expect(() => decl('d:f2', null, '2024')).not.toThrow(); + expect(() => decl('d:3', 'H2')).not.toThrow(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + // The retrofit has to be safe on a LIVE D1: ship-related-persons wipes rows, never table definitions, + // so a deployed database keeps its unconstrained 0003 shape until this migration rebuilds it. + it('0007 retrofits a pre-existing database, keeps its rows, and re-applies as a no-op', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + for (const m of [migration0, migration3, migration6]) readScript(dbPath, m); + // Simulate a database provisioned BEFORE today: 0003 now declares the constraints, so the legacy + // shape has to be recreated explicitly. This is the state every already-deployed environment is in + // — `CREATE TABLE IF NOT EXISTS` never revisited it and ship-related-persons only wipes rows. + sqlite( + dbPath, + `DROP TABLE declarations; + CREATE TABLE declarations ( + id TEXT PRIMARY KEY, person_id TEXT NOT NULL REFERENCES persons(id), xml_file TEXT NOT NULL, + control_hash TEXT, folder_year TEXT NOT NULL, declared_year TEXT, template TEXT NOT NULL, + category TEXT, institution TEXT, position TEXT, source_url TEXT NOT NULL, + UNIQUE (xml_file, control_hash)); + DROP TABLE interest_links; + CREATE TABLE interest_links ( + id TEXT PRIMARY KEY, link_key TEXT NOT NULL UNIQUE, + person_id TEXT NOT NULL REFERENCES persons(id), bidder_id TEXT NOT NULL, eik TEXT NOT NULL, + entity_key TEXT NOT NULL, match_method TEXT, matcher_version TEXT NOT NULL, + publish_tier TEXT NOT NULL, relation TEXT NOT NULL, + interest_class TEXT NOT NULL DEFAULT 'management_role', contemporaneous INTEGER NOT NULL DEFAULT 0, + own_institution TEXT NOT NULL DEFAULT 'none', evidence_count INTEGER NOT NULL DEFAULT 1, + first_declared_year TEXT, last_declared_year TEXT, contract_count INTEGER NOT NULL DEFAULT 0, + contract_value_eur REAL, first_contract_year TEXT, last_contract_year TEXT, + status TEXT NOT NULL DEFAULT 'held', verified_by TEXT, verified_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')));`, + ); + // The rows the retrofit must preserve — plus the two shapes it must not. + sqlite( + dbPath, + `INSERT INTO persons(id, name) VALUES ('person:a', 'А'); + INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('d:keep', 'person:a', 'K.xml', 'HK', '2023', 'assets', 'https://x/K.xml'); + INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('d:drop', 'person:a', 'D.xml', NULL, '2023', 'assets', 'https://x/D.xml'); + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:k', 'k', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership', 'published'); + -- The uninterpretable status §2 is about, already stored because nothing rejected it. + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:bad', 'bad', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership', 'published ');`, + ); + + readScript(dbPath, migration7); + // The valid rows survive the rebuild… + expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( + 'd:keep', + ); + expect(sqlite(dbPath, "SELECT link_key FROM interest_links WHERE link_key='k';").trim()).toBe( + 'k', + ); + // …while the trailing-space row cannot: a status we cannot interpret is not a link to publish or + // guess at, and the loader rebuilds the table on the next run. + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('1'); + // The hashless row SURVIVES — it is a real declaration and NOT NULL was rejected for that reason — + // but it is now covered by the natural key, so re-importing it is refused rather than duplicated. + expect(sqlite(dbPath, "SELECT COUNT(*) FROM declarations WHERE id='d:drop';").trim()).toBe( + '1', + ); + expect(() => + sqlite( + dbPath, + `INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('d:dup', 'person:a', 'D.xml', NULL, '2023', 'assets', 'https://x/D.xml');`, + ), + ).toThrow(/UNIQUE/i); + // …and the constraints are actually in force afterwards. + expect(() => + sqlite(dbPath, "UPDATE interest_links SET status='published ' WHERE link_key='k';"), + ).toThrow(/CHECK/i); + + // Migrations here are applied by a bare `d1 execute --file` with no applied-migrations tracking, + // so a second application MUST be a no-op rather than an error or a data loss. + readScript(dbPath, migration7); + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('1'); + expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( + 'd:keep', + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); From 021da5a1f37797b4ee9b26662b5cd84e98533f53 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:45:31 +0300 Subject: [PATCH 40/50] fix(db): gate the company-search badge on the evidence seal, and bind all four copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The detail-page gate is belt-and-braces — SURFACED_OWNERSHIP requires `status='published'` AND a publishing seal. The company-search badge checked status alone, so a published row whose seal was missing or withholding (a legacy row, a partial run, a loader bug) advertised a свързани-лица claim about a named official to every searcher, while the page behind the badge correctly showed nothing. Search is the wider surface of the two: it is what a reader sees before deciding to look. The table probe now requires interest_links AND interest_link_evidence. The join reads both, so an env with 0003 but not 0006 would 500 every search on every kind — the exact failure the probe was written to prevent. Requiring both also gives the right answer there: with no seal table nothing is provably sealed, so nothing should badge. The predicate now has four copies, and nothing but a test can bind SQL across three languages and two file types. related-persons-sql.test.ts asserts all four agree on both the EXISTS shape and the rung list, whitespace-insensitively. refresh-slice.sql is in that set for its own reason: it runs on the 6-hourly cron, so drift there silently keeps officials in the search index whose links no longer surface. Mutation-checked: adding a rung to refresh-slice.sql alone fails both binding tests. The badge fixture carries an unsealed published link and a withholding-sealed one, each of which must not badge, plus sealed links on both publishing rungs that must. --- packages/db/src/queries/search.ts | 45 +++++++++++----- packages/db/src/related-persons-sql.test.ts | 36 ++++++++++++- packages/db/src/search-sql.test.ts | 59 ++++++++++++++++++--- 3 files changed, 120 insertions(+), 20 deletions(-) diff --git a/packages/db/src/queries/search.ts b/packages/db/src/queries/search.ts index 7a19f01e8..c3f3f02d0 100644 --- a/packages/db/src/queries/search.ts +++ b/packages/db/src/queries/search.ts @@ -112,15 +112,23 @@ interface HitRow { // One group's ranked hits. Company rows additionally carry a свързани-лица flag: a LEFT JOIN against the // published conflict links keyed on the winner's ЕИК (= the company row's `ident`). Published, self OR family -// stake, AND the winner must have LIVE contracts (the same read-time N9 gate LINK_SELECT applies), so search -// flags exactly the companies the /conflicts surface shows and never one whose page would 404. A family-only -// winner badges — the /conflicts page already publishes that link by name, so the badge discloses nothing the -// surface doesn't. Binds: kind, match, limit. +// stake, backed by a Trade Register evidence SEAL, AND the winner must have LIVE contracts (the same +// read-time N9 gate LINK_SELECT applies), so search flags exactly the companies the /conflicts surface shows +// and never one whose page would 404. A family-only winner badges — the /conflicts page already publishes +// that link by name, so the badge discloses nothing the surface doesn't. Binds: kind, match, limit. +// +// The seal EXISTS clause is the same one SURFACED_OWNERSHIP applies (related-persons.ts), and it belongs +// here for a sharper reason than symmetry: search is the WIDER surface. A published row whose seal is +// missing or withholding — a legacy row, a partial run, a loader bug — would badge a свързани-лица claim +// about a named official to every searcher, while the page behind the badge correctly showed nothing. +// Four copies of this predicate now exist (here, SURFACED_OWNERSHIP, precompute.sql, refresh-slice.sql); +// related-persons-sql.test.ts pins them to each other. // Exported so search-sql.test runs the EXACT SQL (not a copy). -// Built with or without the interest_links conflict join. The join is on the свързани-лица migration (0003); -// on an env where it has not been applied yet the join would make the ENTIRE search 500 with „no such table: -// interest_links". search() detects the table once per request and picks the no-conflict variant when absent, -// so search degrades to has_conflict=0 rather than breaking (ADR-0031 robustness ask). +// Built with or without the conflict join. The join reads BOTH свързани-лица migrations — interest_links +// (0003) and interest_link_evidence (0006) — and on an env where either is unapplied it would make the +// ENTIRE search 500 with „no such table". search() detects both tables once per request and picks the +// no-conflict variant when either is absent, so search degrades to has_conflict=0 rather than breaking +// (ADR-0031 robustness ask). const hitsSql = (withConflict: boolean): string => `SELECT search_index.ref, search_index.title, search_index.ident, search_index.subtitle, search_index.amount, rank, ct.kind AS entity_kind, ct.ownership_kind, ct.eik_valid, @@ -133,6 +141,8 @@ const hitsSql = (withConflict: boolean): string => `SELECT search_index.ref, sea ? `LEFT JOIN ( SELECT DISTINCT il.eik FROM interest_links il WHERE il.status = 'published' AND il.interest_class IN ('private_ownership', 'family_ownership') + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed')) AND EXISTS (SELECT 1 FROM contracts cc JOIN bidders bb ON bb.id = cc.bidder_id WHERE bb.eik_normalized = il.eik) ) cf ON search_index.kind = 'company' AND cf.eik = search_index.ident` @@ -161,12 +171,21 @@ export async function search(db: D1Database, rawQuery: string): Promise(); const counts = new Map(countRows.results.map((r) => [r.kind, r.n])); - // Pick the hits SQL once: on an env where the свързани-лица migration (0003) is not applied, the conflict - // join would 500 the whole search — fall back to the no-conflict variant (has_conflict=0) instead. + // Pick the hits SQL once: on an env where the свързани-лица migrations are not applied, the conflict join + // would 500 the whole search — fall back to the no-conflict variant (has_conflict=0) instead. + // + // BOTH tables are required, not just interest_links: the join now also reads interest_link_evidence + // (0006), so an env with 0003 but not 0006 would break every search on every kind. Requiring both also + // gives the right answer on such an env — with no seal table nothing is provably sealed, so nothing + // should badge. const hasConflictTable = - (await db - .prepare("SELECT 1 AS n FROM sqlite_master WHERE type='table' AND name='interest_links'") - .first<{ n: number }>()) != null; + ( + await db + .prepare( + "SELECT COUNT(*) AS n FROM sqlite_master WHERE type='table' AND name IN ('interest_links','interest_link_evidence')", + ) + .first<{ n: number }>() + )?.n === 2; const hitsSql = hasConflictTable ? SEARCH_HITS_SQL : SEARCH_HITS_SQL_NO_CONFLICT; const built = await Promise.all( diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index 512da590e..1501f16db 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -1,6 +1,6 @@ /// import { execFileSync } from 'node:child_process'; -import { mkdtempSync, rmSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -11,7 +11,9 @@ import { LINK_CONTRACTS_LIMIT, LINK_CONTRACTS_SQL, OFFICIAL_SQL, + SURFACED_OWNERSHIP, } from './queries/related-persons'; +import { SEARCH_HITS_SQL } from './queries/search'; // Integration test for the свързани-лица SQL. The query layer's unit tests (queries/related-persons.test) // use a fake D1 and never run the aggregation; this runs the EXACT exported SQL against a real SQLite @@ -629,3 +631,35 @@ describe('свързани-лица SQL (real SQLite)', () => { }); }); }); + +// The evidence-seal gate has FOUR copies, and they must say the same thing. Reviewers found the fourth +// (the company-search badge) checking `status='published'` alone, so search advertised a свързани-лица +// claim about a named official that the detail page correctly withheld. `refresh-slice.sql` matters for a +// different reason: it runs on the 6-hourly cron, so drift there silently keeps officials in the search +// index whose links no longer surface. Nothing but a test binds SQL in three languages and two file types. +describe('the evidence-seal gate is identical in all four places it is written', () => { + const sources: [string, string][] = [ + ['related-persons.ts (SURFACED_OWNERSHIP)', SURFACED_OWNERSHIP], + ['search.ts (company badge)', SEARCH_HITS_SQL], + ['precompute.sql', readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8')], + ['refresh-slice.sql', readFileSync(resolve(root, 'scripts/refresh-slice.sql'), 'utf8')], + ]; + + it.each(sources)('%s requires a publishing evidence seal', (_label, sql) => { + // Whitespace-insensitive: the four are formatted for their own file, and a line break is not drift. + const flat = sql.replace(/\s+/g, ' '); + expect(flat).toMatch( + /EXISTS \( ?SELECT 1 FROM interest_link_evidence e WHERE e\.link_key = il\.link_key AND e\.evidence_kind IN \('document','confirmed'\) ?\)/, + ); + }); + + it.each(sources)('%s admits exactly the two publishing rungs', (_label, sql) => { + // The rung list is the gate. A future rung added to one copy and not the others either leaks an + // unproven claim or silently drops a proven one, depending on which copy gained it. + const kinds = [...sql.matchAll(/evidence_kind IN \(([^)]*)\)/g)].map((m) => + m[1]!.replace(/\s|'/g, ''), + ); + expect(kinds.length).toBeGreaterThan(0); + for (const k of kinds) expect(k).toBe('document,confirmed'); + }); +}); diff --git a/packages/db/src/search-sql.test.ts b/packages/db/src/search-sql.test.ts index 58c25205d..f371cd1df 100644 --- a/packages/db/src/search-sql.test.ts +++ b/packages/db/src/search-sql.test.ts @@ -16,6 +16,7 @@ import { SEARCH_HITS_SQL, SEARCH_HITS_SQL_NO_CONFLICT } from './queries/search'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); function readScript(dbPath: string, path: string): void { execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${path}\n`, stdio: 'pipe' }); @@ -44,17 +45,26 @@ INSERT INTO bidders (id, name, eik_normalized, eik_valid, kind) VALUES ('eik:333','ГАМА ООД','333333333',1,'company'), ('eik:444','ДЕЛТА ООД','444444444',1,'company'), ('eik:555','ЕПСИЛОН ООД','555555555',1,'company'), - ('eik:666','ЗЕТА ООД','666666666',1,'company'); + ('eik:666','ЗЕТА ООД','666666666',1,'company'), + -- #279: two winners with a PUBLISHED link that must not badge. ЙОТА's link carries NO evidence seal at + -- all (a legacy row, a partial run, or a loader bug); КАПА's seal is a WITHHOLDING rung. The detail page + -- refuses both via SURFACED_OWNERSHIP; the badge has to agree, or search advertises an unproven claim + -- about a named official on a page that then shows nothing. + ('eik:777','ЙОТА ООД','777777777',1,'company'), + ('eik:888','КАПА ООД','888888888',1,'company'); INSERT INTO company_totals (bidder_id, name, kind, eik, eik_valid, won_eur, contracts, authorities) VALUES ('eik:111','АЛФА ООД','company','111111111',1,1000000,1,1), ('eik:222','БЕТА ООД','company','222222222',1,500000,1,1), ('eik:333','ГАМА ООД','company','333333333',1,200000,1,1), ('eik:444','ДЕЛТА ООД','company','444444444',1,300000,1,1), ('eik:555','ЕПСИЛОН ООД','company','555555555',1,700000,1,1), - ('eik:666','ЗЕТА ООД','company','666666666',1,900000,1,1); + ('eik:666','ЗЕТА ООД','company','666666666',1,900000,1,1), + ('eik:777','ЙОТА ООД','company','777777777',1,400000,1,1), + ('eik:888','КАПА ООД','company','888888888',1,600000,1,1); INSERT INTO persons (id, name) VALUES ('person:ИВАН МИНЕВ','Иван Минев'),('person:ГЕОРГИ ПЕТРОВ','Георги Петров'), - ('person:ДАНА ФАМ','Дана Фам'),('person:БОРИС БОРД','Борис Борд'),('person:ДВОЕН ТЕСТ','Двоен Тест'); + ('person:ДАНА ФАМ','Дана Фам'),('person:БОРИС БОРД','Борис Борд'),('person:ДВОЕН ТЕСТ','Двоен Тест'), + ('person:БЕЗ ПЕЧАТ','Без Печат'),('person:ЗАДЪРЖАН ПЕЧАТ','Задържан Печат'); INSERT INTO declarations (id, person_id, xml_file, control_hash, folder_year, declared_year, template, category, institution, position, source_url) VALUES ('decl:i','person:ИВАН МИНЕВ','i.xml','H1','2024','2023','assets','','ОБЩИНА РУСЕ','', 'https://register.cacbg.bg/2024/i.xml'), ('decl:g','person:ГЕОРГИ ПЕТРОВ','g.xml','H2','2024','2023','assets','','МИНИСТЕРСТВО Х','', 'https://register.cacbg.bg/2024/g.xml'), @@ -75,15 +85,30 @@ INSERT INTO interest_links -- so the redundant-family collapse (ADR-0032) DROPS the family link from the index; the winner's €50k counts -- ONCE, not €100k — no de-anonymization vector, no double-count. ('il:ds','person:ДВОЕН ТЕСТ|666','person:ДВОЕН ТЕСТ','eik:666','666666666','ЗЕТА ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',0,'none',1,'2020','2023',1,50000,'2021','2021','published'), - ('il:dfam','person:ДВОЕН ТЕСТ|666|family','person:ДВОЕН ТЕСТ','eik:666','666666666','ЗЕТА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2023',1,50000,'2021','2021','published'); + ('il:dfam','person:ДВОЕН ТЕСТ|666|family','person:ДВОЕН ТЕСТ','eik:666','666666666','ЗЕТА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2023',1,50000,'2021','2021','published'), + -- published, ownership class, live contracts — and NO seal. Everything the badge used to check, passed. + ('il:no','person:БЕЗ ПЕЧАТ|777','person:БЕЗ ПЕЧАТ','eik:777','777777777','ЙОТА ООД','exact_name_key','v1','document','owns','private_ownership',0,'none',1,'2020','2023',1,400000,'2021','2021','published'), + -- published, and sealed with a rung that WITHHOLDS (ADR-0035). A seal existing is not a seal permitting. + ('il:un','person:ЗАДЪРЖАН ПЕЧАТ|888','person:ЗАДЪРЖАН ПЕЧАТ','eik:888','888888888','КАПА ООД','exact_name_key','v1','document_uncorroborated','owns','private_ownership',0,'none',1,'2020','2023',1,600000,'2021','2021','published'); +INSERT INTO interest_link_evidence (link_key, evidence_kind, lookup_date, rules_version, live_status) VALUES + ('person:ИВАН МИНЕВ|111','document','2026-08-12','tr-rules-1','live'), + ('person:ИВАН МИНЕВ|333','confirmed','2026-08-12','tr-rules-1','live'), + ('person:ДАНА ФАМ|444','confirmed','2026-08-12','tr-rules-1','live'), + ('person:БОРИС БОРД|555','document','2026-08-12','tr-rules-1','live'), + ('person:ДВОЕН ТЕСТ|666','document','2026-08-12','tr-rules-1','live'), + ('person:ДВОЕН ТЕСТ|666|family','document','2026-08-12','tr-rules-1','live'), + ('person:ЗАДЪРЖАН ПЕЧАТ|888','document_uncorroborated','2026-08-12','tr-rules-1','live'); INSERT INTO authorities (id, name) VALUES ('a:1','ВЕДОМСТВО ТЕСТ'); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES - ('t:1','s1','Т1','a:1','open'),('t:3','s3','Т3','a:1','open'),('t:4','s4','Т4','a:1','open'),('t:6','s6','Т6','a:1','open'); + ('t:1','s1','Т1','a:1','open'),('t:3','s3','Т3','a:1','open'),('t:4','s4','Т4','a:1','open'), + ('t:6','s6','Т6','a:1','open'),('t:7','s7','Т7','a:1','open'),('t:8','s8','Т8','a:1','open'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES ('c:1','t:1','eik:111',1000000,'EUR','2021-05-01','N1',1000000), ('c:3','t:3','eik:333',200000,'EUR','2021-05-01','N3',200000), ('c:4','t:4','eik:444',300000,'EUR','2021-05-01','N4',300000), - ('c:6','t:6','eik:666',50000,'EUR','2021-05-01','N6',50000); + ('c:6','t:6','eik:666',50000,'EUR','2021-05-01','N6',50000), + ('c:7','t:7','eik:777',400000,'EUR','2021-05-01','N7',400000), + ('c:8','t:8','eik:888',600000,'EUR','2021-05-01','N8',600000); `; // Search-index population — a STRUCTURAL proxy for scripts/precompute.sql's officials block: it exercises the @@ -116,6 +141,7 @@ function withDb(fn: (dbPath: string) => void): void { try { readScript(dbPath, migration0); readScript(dbPath, migration2); + readScript(dbPath, migration6); exec(dbPath, FIXTURE); exec(dbPath, POPULATE_INDEX); fn(dbPath); @@ -151,6 +177,27 @@ describe('search свързани-лица SQL', () => { }); }); + it('the badge requires an evidence SEAL, not merely status=published (#279)', () => { + // The detail page gate is belt-and-braces — status AND a publishing seal (SURFACED_OWNERSHIP). The + // badge checked status alone, so an evidence-less published row advertised a свързани-лица claim on a + // named official in search while the page it links to correctly withheld it. Search is the wider + // surface of the two: it is what a reader sees before deciding to look. + withDb((dbPath) => { + // NO seal at all — the legacy/partial-run/loader-bug shape. + const iota = rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'йота*', 10)); + expect(iota).toHaveLength(1); + expect(iota[0]!.has_conflict).toBe(0); + // Sealed, but with a rung that WITHHOLDS. A seal existing is not a seal permitting. + const kapa = rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'капа*', 10)); + expect(kapa).toHaveLength(1); + expect(kapa[0]!.has_conflict).toBe(0); + // POSITIVE CONTROL — a properly sealed link still badges, on both publishing rungs. Without this a + // gate that rejected everything would satisfy the two assertions above. + expect(rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'алфа*', 10))[0]!.has_conflict).toBe(1); + expect(rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'гама*', 10))[0]!.has_conflict).toBe(1); + }); + }); + it('indexes family_ownership (ADR-0032) but NEVER management_role/ex-officio (interest_class gate)', () => { withDb((dbPath) => { // family_ownership now reaches the index identically to self (ADR-0032): Дана (a relative's stake) IS From b8b4332442fab0dc13db107bde890a963964b942 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:48:01 +0300 Subject: [PATCH 41/50] fix(web,ci): make the page prose family-aware and default the data job to staging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §2.6 made the CARD labels tense-neutral and family-aware but left the surrounding page copy asserting „собствен дял" — an OWN stake — directly above cards that correctly read „деклариран дял на свързано лице". On a family-only page that is a false claim about the named official, and it is exactly the second source of truth the card-label fix set out to remove. `declaredStakeNoun` derives the wording from the links the page is actually rendering, so a page cannot describe a set it isn't showing. Family-AWARE rather than family-blind: an own-stake page still says „собствен дял" (a positive control covers this — deleting the claim everywhere would pass the family assertion while making the page vaguer than the data warrants), and a mixed set gets the only phrasing true of every card. The ledes also stopped claiming „точно съвпадение по фирмено име" — since #279 identity rests on a Trade Register fact, not on the name. related-persons-data.yml offered `default: dev` while no `dev` environment exists, every fallback in the file is `|| 'staging'`, and the note at the top says the job targets staging. A manual dispatch therefore went to an environment with no secrets and failed at the credential guard rather than running where intended. Dropped from the options too, so it cannot be picked deliberately either. Methodology: „премахва незабавно" now distinguishes the D1 write (immediate) from what a reader sees, which the page's own publicCache(3600) can hold for up to an hour. The promise was not wrong about the action, only about when it becomes visible. --- .github/workflows/related-persons-data.yml | 7 ++++-- apps/web/app/lib/conflicts.ts | 17 ++++++++++++++ apps/web/app/routes/conflict.company.tsx | 4 ++-- apps/web/app/routes/conflict.methodology.tsx | 12 ++++++---- apps/web/app/routes/conflict.official.tsx | 14 +++++++---- .../app/routes/conflict.pages.render.test.tsx | 23 +++++++++++++++++++ 6 files changed, 63 insertions(+), 14 deletions(-) diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index 8472b96f5..02a0fa56c 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -56,8 +56,11 @@ on: environment: description: Target D1 environment type: choice - options: [dev, staging, production] - default: dev + # No `dev` environment exists — every fallback in this file is `|| 'staging'` and the note above + # says the job targets staging. A `dev` default sent a manual dispatch to an environment with no + # secrets, so it failed at the credential guard rather than running where it was meant to. + options: [staging, production] + default: staging full_crawl: description: Crawl the CACBG corpus (slow). RESUMES from the cached raw XML — fetch.mjs skips files already on disk, so this fills gaps rather than re-fetching from scratch (clear the cache for a true rebuild). Off = extract from the existing raw cache without crawling. type: boolean diff --git a/apps/web/app/lib/conflicts.ts b/apps/web/app/lib/conflicts.ts index 5b99180ed..7e0908c16 100644 --- a/apps/web/app/lib/conflicts.ts +++ b/apps/web/app/lib/conflicts.ts @@ -46,6 +46,23 @@ export function relationLabel(relation: string): string { return RELATION_LABEL[relation] ?? relation; } +/** + * How to describe a PAGE's set of links in prose (#279 §2.6). The card labels above are already + * family-aware; the surrounding page copy was not, and asserted „собствен дял" — an OWN stake — above + * cards that correctly read „свързано лице". On a family-only page that is a false claim about the named + * official, and it is the second source of truth the card-label fix set out to remove. + * + * Derived from the links themselves rather than passed in, so a page cannot describe a set it isn't + * rendering. Mixed sets get the neutral wording: it is the only phrasing true of every card. + */ +export function declaredStakeNoun(links: { relation: string }[]): string { + const anyFamily = links.some((l) => l.relation === 'related'); + const anySelf = links.some((l) => l.relation !== 'related'); + if (anyFamily && !anySelf) return 'дял на свързано лице'; + if (anyFamily && anySelf) return 'деклариран дял — собствен или на свързано лице'; + return 'собствен дял'; +} + // Defense in depth: the slug is base64url and the ЕИК numeric today (so encoding is a no-op), but if either // assumption ever drifts, an un-escaped `/`, `?` or `#` would break routing and the cache key. Escape the // dynamic segments unconditionally (ydimitrof #226, conflicts.ts). diff --git a/apps/web/app/routes/conflict.company.tsx b/apps/web/app/routes/conflict.company.tsx index 9dd285950..5de8e12e6 100644 --- a/apps/web/app/routes/conflict.company.tsx +++ b/apps/web/app/routes/conflict.company.tsx @@ -9,7 +9,7 @@ import { ConflictCards } from '../components/ConflictCards'; import { publicCache } from '../lib/cache'; import { withDbRetry } from '../lib/retry'; import { seoMeta } from '../lib/meta'; -import { companyProfileHref } from '../lib/conflicts'; +import { companyProfileHref, declaredStakeNoun } from '../lib/conflicts'; // Officials with a published declared interest in one winner (by ЕИК). Reads interest_links only. 404 when // no official has a published link to this company — never an empty page under a company's name. @@ -59,7 +59,7 @@ export default function ConflictCompany({ loaderData }: Route.ComponentProps) { } title={company} - lede={`Длъжностни лица, декларирали собствен дял в това дружество пред КПКОНПИ. ${count(links.length)} ${plural(links.length, 'връзка', 'връзки')} — всяка е точно съвпадение по фирмено име.`} + lede={`Длъжностни лица, декларирали ${declaredStakeNoun(links)} в това дружество пред КПКОНПИ. ${count(links.length)} ${plural(links.length, 'връзка', 'връзки')} — всяка почива на проверим факт от Търговския регистър.`} /> diff --git a/apps/web/app/routes/conflict.methodology.tsx b/apps/web/app/routes/conflict.methodology.tsx index 209f11f9f..d7b1f9d4c 100644 --- a/apps/web/app/routes/conflict.methodology.tsx +++ b/apps/web/app/routes/conflict.methodology.tsx @@ -437,11 +437,13 @@ export default function ConflictMethodology() {

    Разглеждаме сигнала в срок до 7 работни дни. При основателен сигнал - връзката се премахва незабавно и се записва в списък за премахване, - който оцелява при всяко следващо обновяване — веднъж премахната - връзка не се възстановява при повторно зареждане на данните. Премахваме връзка при - доказана неточност; не заличаваме коректни записи по искане на изпълнител или - възложител — данните идват от публични източници и остават публични. + връзката се премахва веднага след решението и се записва в списък + за премахване, който оцелява при всяко следващо обновяване — веднъж + премахната връзка не се възстановява при повторно зареждане на данните. Премахването + се прилага незабавно в базата; страницата може да се вижда в кеш{' '} + до един час след това. Премахваме връзка при доказана неточност; не + заличаваме коректни записи по искане на изпълнител или възложител — данните идват от + публични източници и остават публични.

    • diff --git a/apps/web/app/routes/conflict.official.tsx b/apps/web/app/routes/conflict.official.tsx index 8ef918790..01710bf5f 100644 --- a/apps/web/app/routes/conflict.official.tsx +++ b/apps/web/app/routes/conflict.official.tsx @@ -8,6 +8,7 @@ import { ConflictCards } from '../components/ConflictCards'; import { publicCache } from '../lib/cache'; import { withDbRetry } from '../lib/retry'; import { seoMeta } from '../lib/meta'; +import { declaredStakeNoun } from '../lib/conflicts'; // One office-holder's declared ownership links. Reads private-ownership interest_links only. 404 (not an // empty page) when the person has no published link — a bare page under someone's name reads as an @@ -39,6 +40,9 @@ export async function loader({ params, context }: Route.LoaderArgs) { export default function ConflictOfficial({ loaderData }: Route.ComponentProps) { const { official, links } = loaderData; + // Family-AWARE, not family-blind: the page must not assert an own stake above cards that say „свързано + // лице", and must not go vague where the stake really is the official's own (§2.6). + const stake = declaredStakeNoun(links); return ( <>

    Данните са от собствените декларации на лицето (публичен регистър на КПКОНПИ), - съпоставени точно с регистъра на изпълнителите. Показваме само 100% съвпадения и само - собствен деклариран дял. Сигнал за неточност:{' '} - Методология → Поправки. + съпоставени точно с регистъра на изпълнителите. Показваме само 100% съвпадения, и само + когато самоличността на дружеството е потвърдена от Търговския регистър. Сигнал за + неточност: Методология → Поправки.

    { expect(card.textContent).not.toContain('Кмет Тестов'); }); + it('a family-only page never asserts the official owns the stake (§2.6)', async () => { + // The CARD labels were made tense-neutral and family-aware, but the page lede and the section hint + // still read „декларирало собствен дял" — a second source of truth on the very page that renders a + // relative's stake. On a family-only page that is a false claim about the named official, printed + // above a card that correctly says „свързано лице". + await mount(ConflictOfficial as never, { + official: 'Кмет Тестов', + links: [link({ relation: 'related', company: 'ЕВРОСТРОЙ 21 ЕООД', eik: '333' })], + }); + expect(text()).not.toContain('собствен дял'); + expect(text()).toContain('деклариран дял на свързано лице'); + }); + + it('an own-stake page still says so — the wording is family-AWARE, not family-blind', async () => { + // POSITIVE CONTROL. Removing the claim everywhere would satisfy the assertion above while making the + // page vaguer than the data warrants: a self stake IS the official's own and should read that way. + await mount(ConflictOfficial as never, { + official: 'Кмет Тестов', + links: [link({ relation: 'owns', company: 'ЕВРОСТРОЙ 21 ЕООД', eik: '333' })], + }); + expect(text()).toContain('собствен дял'); + }); + it('meta() names the person in the title and marks the page noindex', () => { const tags = officialMeta({ data: { official: 'Иван Петров', links: [] }, From 2039b9f7ee1d472248f5d212bef2d1b0cfaae1c9 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 16:50:13 +0300 Subject: [PATCH 42/50] test: cover the four unexercised audit axes, every ownership field, and the rule disclosure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three gaps were named by reviewers, and none had a bug behind it — but each guards a path where failure is silent, which is the case for testing them rather than trusting them. Audit axes A_key_missing, A_eik_mismatch, B_bidder_eik and B_eik_invalid fired in no test. The audit is the last gate before a named claim ships, so an axis that quietly stopped firing would be invisible: A_eik_mismatch is the libel case in its purest form (a name resolving to one winner, the link published against another), and B_eik_invalid is the only rail keeping a checksum-invalid ЕИК off the surface. Mutation-checked — disabling B_eik_invalid fails its test — and paired with a positive control, since four assertions that something fails prove nothing unless the clean shape passes. Every rung-2 owner test used CR_F_19_L, leaving CR_F_18_L (едноличен собственик) and CR_F_23_L (ФЛ-търговец) unexercised. A dropped entry there would withhold an entire ownership shape — a sole owner, the commonest ЕООД form — as „Неизвестна", a recall hole with no symptom. Controlled by a non-ownership field code that must NOT match. The methodology page's rule disclosure is a launch condition under ADR-0033 decision 7, not a nicety, and nothing kept it in step with the ladder. The rule could change in evidence.mjs and leave the page describing a system that no longer exists — worse than not disclosing it. Now pinned: every rung by name, the joint-stock bar, the three-names/ one-record rule, ADR-0035's company gate, both halves of R10, the no-ЕГН homonym limit, and the company-identity-not-ownership distinction the surface rests on. --- .../app/routes/conflict.pages.render.test.tsx | 29 +++++++ scripts/cacbg/audit.test.mjs | 86 +++++++++++++++++++ scripts/tr/evidence.test.mjs | 18 ++++ 3 files changed, 133 insertions(+) diff --git a/apps/web/app/routes/conflict.pages.render.test.tsx b/apps/web/app/routes/conflict.pages.render.test.tsx index d94f45673..3178fed7c 100644 --- a/apps/web/app/routes/conflict.pages.render.test.tsx +++ b/apps/web/app/routes/conflict.pages.render.test.tsx @@ -162,6 +162,35 @@ describe('/conflicts/company/:eik — render', () => { }); describe('/conflicts/methodology — render', () => { + it('discloses the matching rule verbatim — every rung, and what each may conclude', async () => { + // ADR-0021 E10 makes this page the disclosure of the rule, and ADR-0033 decision 7 makes it a LAUNCH + // CONDITION rather than a follow-up: a heuristic that asserts something about a named person is only + // defensible if the reader can see exactly what was asserted and why. Nothing but a test keeps the + // page in step with the ladder — the rule can change in evidence.mjs and leave the page describing a + // system that no longer exists, which is worse than not disclosing it at all. + await mount(ConflictMethodology as never, {}); + const t = text(); + // every rung of the ladder, by the name the seal and the card use + for (const rung of ['Документ', 'Потвърдено', 'Оборена', 'Неизвестна']) + expect(t).toContain(rung); + // rung 1 — the joint-stock bar and its reason (the „11 акции" trap) + expect(t).toContain('Акционерна форма'); + expect(t).toContain('не е публична'); + // rung 2 — all three names, one record, and the two refusals + expect(t).toContain('пълно съвпадение и на трите имена'); + expect(t).toContain('един и същ запис'); + // ADR-0035 — the company gate, the part a reader most needs to judge the claim + expect(t).toContain('Съвпадението по име само по себе си не стига'); + // R10 — the seat's temporal guard, both halves + expect(t).toContain('вписано преди декларирания период'); + expect(t).toContain('когато този период е известен'); + // the honest limit: no ЕГН, so a homonym is possible + expect(t).toContain('не съдържа ЕГН'); + expect(t).toContain('съименник'); + // what the register proves and what it does not — the distinction the whole surface rests on + expect(t).toContain('самоличността на дружеството'); + }); + it('states the three libel rails in plain language', async () => { await mount(ConflictMethodology as never, {}); const t = text(); diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index 267ce4ad1..e0f8af4bb 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -385,3 +385,89 @@ test('D positive control — an unchanged published set produces no monotonicity assert.equal(threw, false); assert.doesNotMatch(out, /D_monotonicity/); }); + +// The four axes below fire on shapes that no test exercised. Each is a hard finding — the audit is the +// last gate before a named claim ships — so an axis that silently stopped firing would be invisible. + +test('A_key_missing: a link whose entity_key is in NO live bidder → hard finding', () => { + // The key resolved when the link was built and does not now: the winner was renamed, re-keyed or + // dropped from the corpus. The link still names an official against a company we can no longer find, + // so it must stop the run rather than ship pointing at nothing. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('ИЗЧЕЗНАЛ ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'an unresolvable entity_key must fail the gate'); + assert.equal(/A_key_missing/.test(out), true); +}); + +test('A_eik_mismatch: a name-resolved key pointing at a DIFFERENT ЕИК than it resolves to → hard', () => { + // The libel case in its purest form: the name resolves to exactly one winner, and the link published a + // different company against it. Everything on the card — contracts, money, the ЕИК link — would be the + // wrong company's, under a real official's name. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`, `'b2','ДРУГ ЕООД','200000002',1`], + links: [ + `'il1','p1|200000002','p1','200000002','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b2','owns',0,1000,'published'`, + ], + seals: [ + `'p1|200000002','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true); + assert.equal(/A_eik_mismatch/.test(out), true); +}); + +test('B_bidder_eik: the stored bidder row disagrees with the link ЕИК → hard finding', () => { + // Row integrity. The card renders the BIDDER's name and money but links out on the link's ЕИК, so a + // disagreement means the reader is shown one company and sent to another. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`, `'b2','ДРУГ ЕООД','200000002',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b2','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true); + assert.equal(/B_bidder_eik/.test(out), true); +}); + +test('B_eik_invalid: publishing against a checksum-INVALID ЕИК → hard finding', () => { + // eik_valid=0 means the ЕИК failed its control digit, so it identifies no company at all. This axis is + // the rail that keeps such a row off the public surface, and nothing else re-checks it downstream. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',0`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'an invalid-ЕИК publish must fail the gate'); + assert.equal(/B_eik_invalid/.test(out), true); +}); + +test('the four axes above are a BOUND: a clean, valid, sealed link passes them all', () => { + // POSITIVE CONTROL for the whole block. Four assertions that something fails prove nothing unless the + // correct shape passes — an audit that flagged everything would satisfy every test above. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, false, `a clean link must pass: ${out}`); + for (const axis of ['A_key_missing', 'A_eik_mismatch', 'B_bidder_eik', 'B_eik_invalid']) + assert.equal(new RegExp(axis).test(out), false, `${axis} must not fire on a clean link`); +}); diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs index 3010698c2..321230706 100644 --- a/scripts/tr/evidence.test.mjs +++ b/scripts/tr/evidence.test.mjs @@ -273,6 +273,24 @@ test('rung 3 — name uniqueness does NOT gate the stronger „Документ" ); }); +test('rung 2 — every OWNERSHIP field code can carry the match, not just CR_F_19_L', () => { + // OWNERSHIP_FIELDS is ['CR_F_18_L','CR_F_19_L','CR_F_23_L'] — едноличен собственик, съдружници, and + // ФЛ-търговец. Every owner test used CR_F_19_L, so a typo or a dropped entry in the other two would + // have silently withheld an entire ownership shape: a sole owner (the commonest ЕООД form) publishing + // as „Неизвестна" is a recall hole with no symptom. + for (const code of ['CR_F_18_L', 'CR_F_19_L', 'CR_F_23_L']) { + const d = deed([fld(code, container('ИВАН ПЕТРОВ ТЕСТОВ'))]); + const v = evidenceVerdict({ ...base, deed: d }); + assert.equal(v.kind, 'document', `${code} must carry an ownership match`); + assert.equal(v.registryRole, 'owner', `${code} is an OWNERSHIP field, not management`); + assert.equal(v.matchedFact, `role:owner:${code}`); + } + // POSITIVE CONTROL — a field that is NOT an ownership or manager field must not match at all, or the + // loop above would pass for a reason other than the one it claims. + const other = deed([fld('CR_F_99_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))]); + assert.notEqual(evidenceVerdict({ ...base, deed: other }).kind, 'document'); +}); + // ── rung 2's company gate: the winner-vs-non-winner homonym (ADR-0035) ──────── // // The ladder proves a person with these three tokens is registered in the company we LOOKED UP. It cannot From da80eafbabdd537e0b2e01d33e77b7ccc5ae42b2 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Wed, 12 Aug 2026 17:00:55 +0300 Subject: [PATCH 43/50] fix(db): enforce the retrofit constraints with triggers, not a table rebuild MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `wrangler d1 migrations apply` rejected the create-copy-drop-rename rebuild: interest_link_evidence references interest_links(link_key), D1 enforces foreign keys, and `PRAGMA defer_foreign_keys` does not survive the statement-by-statement execution wrangler performs — the migration aborted with SQLITE_CONSTRAINT_FOREIGNKEY and the Durable Object rolled back. The sqlite3-CLI test passed because that CLI has foreign keys OFF by default, so only a real local apply surfaced it. Dropping the child first would work mechanically and is the wrong trade: it strips every evidence seal, and since the read gate REQUIRES a seal, that empties the public surface until the next monthly data run. BEFORE INSERT/UPDATE triggers enforce the identical invariant for every writer — including the hand-run UPDATE during an incident that no application-level validation covers — with no rebuild, no FK exposure and no seal loss. Fresh databases still get true CHECK constraints from 0003/0006; this is only the retrofit path for databases that already exist. Verified against real local D1: applies clean, and a second application is a no-op. The test now also covers UPDATE, and asserts that a pre-existing bad row SURVIVES — a trigger constrains future writes and cannot retroactively reject stored data, and deleting a real link on a guess would be worse than leaving one the read gate already refuses to show. --- .../0007_publishing_gate_constraints.sql | 128 +++++++++--------- packages/db/src/migrations.test.ts | 32 ++++- 2 files changed, 87 insertions(+), 73 deletions(-) diff --git a/packages/db/migrations/0007_publishing_gate_constraints.sql b/packages/db/migrations/0007_publishing_gate_constraints.sql index ac682308c..7720ac458 100644 --- a/packages/db/migrations/0007_publishing_gate_constraints.sql +++ b/packages/db/migrations/0007_publishing_gate_constraints.sql @@ -8,20 +8,20 @@ -- -- WHY IT IS SAFE TO RE-RUN: migrations here are applied by a bare `wrangler d1 execute --file` with no -- applied-migrations tracking (see related-persons-data.yml / deploy.yml), so re-application MUST be a --- no-op. Both blocks below are written to converge: the index block is `IF NOT EXISTS` over a de-duplicated --- table, and the rebuild block copies the CURRENT table into a constrained one — run a second time it --- copies an already-constrained table into an identical one. Idempotent by construction, no version flag. +-- no-op. Everything below is `IF NOT EXISTS` over statements that converge. -- --- WHY A REBUILD FOR interest_links AND NOT `ALTER TABLE`: SQLite cannot add a CHECK in place, so --- create-copy-drop-rename is the documented route. `declarations` needs no rebuild — its defect was a --- constraint that failed to constrain, which a correct index fixes without touching the table. +-- WHY TRIGGERS AND NOT A TABLE REBUILD: SQLite cannot add a CHECK in place, so the textbook route is +-- create-copy-drop-rename. That route is WRONG here, and a real `wrangler d1 migrations apply` proved it: +-- `interest_link_evidence` references interest_links(link_key), D1 enforces foreign keys, and +-- `PRAGMA defer_foreign_keys` does not survive the statement-by-statement execution wrangler performs — +-- the rebuild aborts with SQLITE_CONSTRAINT_FOREIGNKEY and the Durable Object rolls back. Dropping the +-- child first would work mechanically but would strip every evidence seal, and since the read gate +-- REQUIRES a seal that empties the public surface until the next monthly data run. -- --- FOREIGN KEYS: `interest_link_evidence` references interest_links(link_key), and D1 enforces foreign --- keys, so dropping the parent mid-rebuild would be refused. `defer_foreign_keys` postpones the check to --- the end of the transaction — the documented way to rebuild a referenced table — by which point the --- renamed table satisfies it. D1 runs each file in one implicit transaction, so a failure rolls the whole --- file back rather than leaving a half-swapped table. -PRAGMA defer_foreign_keys = true; +-- A BEFORE INSERT/UPDATE trigger that RAISEs enforces the identical invariant for every writer — +-- including a hand-run UPDATE during an incident, which is when a bad value is most likely typed — with +-- no rebuild, no FK exposure and no seal loss. Freshly created databases still get true CHECK constraints +-- from 0003/0006; this is the retrofit path for databases that already exist. -- ── declarations ──────────────────────────────────────────────────────────────────────────────────── -- The table-level `UNIQUE (xml_file, control_hash)` did not constrain what it was written for. SQLite @@ -46,58 +46,54 @@ DELETE FROM declarations WHERE id NOT IN ( CREATE UNIQUE INDEX IF NOT EXISTS idx_declarations_natural_key ON declarations(xml_file, folder_year, COALESCE(control_hash, '')); --- ── interest_links ────────────────────────────────────────────────────────────────────────────────── --- `status` and `interest_class` ARE the publishing gate. The public surface is `status = 'published'` --- AND a surfaced `interest_class`, so a value that merely resembles one — 'published ' with a trailing --- space, the case #279 §2 names — passes every writer and then fails the gate silently. A CHECK binds --- every writer at once, including a hand-run UPDATE during an incident, which is when it is most likely --- to be typed. +-- ── interest_links: status / interest_class ───────────────────────────────────────────────────────── +-- These two ARE the publishing gate: the surface is `status = 'published'` AND a surfaced +-- `interest_class`. A value that merely resembles one — 'published ' with a trailing space, the case +-- #279 §2 names — passes every writer and then fails the gate silently, hiding a link that should show. -- --- Rows violating the new CHECKs are dropped by INSERT OR IGNORE rather than rewritten: a status we cannot --- interpret is not a link we should publish or guess at, and the loader rebuilds the table on every run. -CREATE TABLE IF NOT EXISTS interest_links_0007 ( - id TEXT PRIMARY KEY, - link_key TEXT NOT NULL UNIQUE, - person_id TEXT NOT NULL REFERENCES persons(id), - bidder_id TEXT NOT NULL, - eik TEXT NOT NULL, - entity_key TEXT NOT NULL, - match_method TEXT, - matcher_version TEXT NOT NULL, - publish_tier TEXT NOT NULL, - relation TEXT NOT NULL, - interest_class TEXT NOT NULL DEFAULT 'management_role' - CHECK (interest_class IN ('private_ownership','family_ownership', - 'ex_officio_board','management_role')), - contemporaneous INTEGER NOT NULL DEFAULT 0, - own_institution TEXT NOT NULL DEFAULT 'none', - evidence_count INTEGER NOT NULL DEFAULT 1, - first_declared_year TEXT, - last_declared_year TEXT, - contract_count INTEGER NOT NULL DEFAULT 0, - contract_value_eur REAL, - first_contract_year TEXT, - last_contract_year TEXT, - status TEXT NOT NULL DEFAULT 'held' - CHECK (status IN ('published','internal','held','withdrawn','suppressed')), - verified_by TEXT, - verified_at TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')) -); -INSERT OR IGNORE INTO interest_links_0007 - (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, - relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, - last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, - status, verified_by, verified_at, created_at) -SELECT id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, - relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, - last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, - status, verified_by, verified_at, created_at -FROM interest_links -WHERE status IN ('published','internal','held','withdrawn','suppressed') - AND interest_class IN ('private_ownership','family_ownership','ex_officio_board','management_role'); -DROP TABLE interest_links; -ALTER TABLE interest_links_0007 RENAME TO interest_links; -CREATE INDEX IF NOT EXISTS idx_interest_links_eik ON interest_links(eik); -CREATE INDEX IF NOT EXISTS idx_interest_links_person ON interest_links(person_id); -CREATE INDEX IF NOT EXISTS idx_interest_links_status ON interest_links(status); +-- One trigger per operation, since SQLite triggers are per-event. `RAISE(ABORT)` rolls back the +-- statement, so a bad write fails loudly at its source instead of surfacing later as a missing link. +DROP TRIGGER IF EXISTS trg_interest_links_status_ins; +CREATE TRIGGER trg_interest_links_status_ins +BEFORE INSERT ON interest_links +WHEN NEW.status NOT IN ('published','internal','held','withdrawn','suppressed') + OR NEW.interest_class NOT IN ('private_ownership','family_ownership','ex_officio_board','management_role') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_links.status/interest_class outside the publishing-gate enum'); +END; + +DROP TRIGGER IF EXISTS trg_interest_links_status_upd; +CREATE TRIGGER trg_interest_links_status_upd +BEFORE UPDATE ON interest_links +WHEN NEW.status NOT IN ('published','internal','held','withdrawn','suppressed') + OR NEW.interest_class NOT IN ('private_ownership','family_ownership','ex_officio_board','management_role') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_links.status/interest_class outside the publishing-gate enum'); +END; + +-- ── interest_link_evidence: evidence_kind ─────────────────────────────────────────────────────────── +-- The read gate filters on this column — SURFACED_OWNERSHIP admits exactly 'document' and 'confirmed' — +-- so an unlisted value either silently stops a link surfacing or, if it collides with a publishing name, +-- surfaces one that was never proven. 0006 declares this as a CHECK for new databases; the trigger is +-- the same rule for those provisioned before it. +DROP TRIGGER IF EXISTS trg_ile_kind_ins; +CREATE TRIGGER trg_ile_kind_ins +BEFORE INSERT ON interest_link_evidence +WHEN NEW.evidence_kind NOT IN ('document','confirmed','document_uncorroborated','refuted', + 'bar_joint_stock','unknown','outside_tr') + OR (NEW.registry_role IS NOT NULL AND NEW.registry_role NOT IN ('owner','manager')) + OR NEW.live_status NOT IN ('live','terminated_owner_still','terminated_manager_still','terminated') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_link_evidence enum outside the ADR-0033/0035 vocabulary'); +END; + +DROP TRIGGER IF EXISTS trg_ile_kind_upd; +CREATE TRIGGER trg_ile_kind_upd +BEFORE UPDATE ON interest_link_evidence +WHEN NEW.evidence_kind NOT IN ('document','confirmed','document_uncorroborated','refuted', + 'bar_joint_stock','unknown','outside_tr') + OR (NEW.registry_role IS NOT NULL AND NEW.registry_role NOT IN ('owner','manager')) + OR NEW.live_status NOT IN ('live','terminated_owner_still','terminated_manager_still','terminated') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_link_evidence enum outside the ADR-0033/0035 vocabulary'); +END; diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index 1997bbd29..bc5d9a225 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -354,16 +354,18 @@ describe('served migrations', () => { ); readScript(dbPath, migration7); - // The valid rows survive the rebuild… + // The valid rows survive — nothing is rebuilt, so nothing can be lost… expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( 'd:keep', ); expect(sqlite(dbPath, "SELECT link_key FROM interest_links WHERE link_key='k';").trim()).toBe( 'k', ); - // …while the trailing-space row cannot: a status we cannot interpret is not a link to publish or - // guess at, and the loader rebuilds the table on the next run. - expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('1'); + // …including the pre-existing trailing-space row. A trigger constrains FUTURE writes; it cannot + // retroactively reject a row already stored, and deleting one would mean dropping a real link on a + // guess. The loader rewrites the table wholesale on the next run, which is what corrects it — and + // the read gate never showed it anyway, since 'published ' is not 'published'. + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('2'); // The hashless row SURVIVES — it is a real declaration and NOT NULL was rejected for that reason — // but it is now covered by the natural key, so re-importing it is refused rather than duplicated. expect(sqlite(dbPath, "SELECT COUNT(*) FROM declarations WHERE id='d:drop';").trim()).toBe( @@ -377,15 +379,31 @@ describe('served migrations', () => { VALUES ('d:dup', 'person:a', 'D.xml', NULL, '2023', 'assets', 'https://x/D.xml');`, ), ).toThrow(/UNIQUE/i); - // …and the constraints are actually in force afterwards. + // …and the constraint is in force for every writer afterwards, on INSERT and on UPDATE alike. + // UPDATE is the one that matters most: a hand-run status change during an incident is exactly when + // a stray character gets typed, and it is the path no application-level validation covers. expect(() => sqlite(dbPath, "UPDATE interest_links SET status='published ' WHERE link_key='k';"), - ).toThrow(/CHECK/i); + ).toThrow(/CHECK failed/i); + expect(() => + sqlite( + dbPath, + `INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:x', 'x', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership', 'publushed');`, + ), + ).toThrow(/CHECK failed/i); + // POSITIVE CONTROL: a legitimate status change still goes through — the trigger is a bound. + expect(() => + sqlite(dbPath, "UPDATE interest_links SET status='withdrawn' WHERE link_key='k';"), + ).not.toThrow(); // Migrations here are applied by a bare `d1 execute --file` with no applied-migrations tracking, // so a second application MUST be a no-op rather than an error or a data loss. readScript(dbPath, migration7); - expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('1'); + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('2'); expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( 'd:keep', ); From ac15498e73e59bb07a4f501b9eee51a9f5b35c0f Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Thu, 13 Aug 2026 19:54:35 +0300 Subject: [PATCH 44/50] test: apply migration 0006 wherever the merged suites build a schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge with upstream/main was textually clean and semantically broken — `git merge-tree` reported no conflict, and four suites then failed. Both sides had edited refresh-slice.sql and ship-related-persons.mjs, and upstream's new harnesses build their schemas from migrations that stop at 0003, while this branch made both files depend on interest_link_evidence (0006). - refresh-slice.test.ts: upstream's new OCDS-bridge site (#286) was a third setup site this branch had not seen; without 0006 sqlite3 aborts on "no such table" at refresh-slice.sql:1957, before reaching the logic under test. - value-flag-stotinki-sql.test.ts / value-flag-annex-step-sql.test.ts: new files, same cause — they died on the свързани-лица block while testing the value_flag CASE. - ship-e2e.test.ts: the evidence seal is now a shipped table, so a target built from 0003 alone makes the generated wipe abort before a row moves. Its corpus now seeds one seal per link, which is the only shape the ship should ever carry — a published link without one is what the audit fails on and what the read gate refuses to surface. One behaviour change worth naming. `a request that never landed fails the run` injected a skipped interest_links chunk; with seals shipping, that now trips the FOREIGN KEY as the orphaned rows land, before the read-back runs. Stronger, but it leaves the read-back gate — the thing the test exists for — unexercised. The injection moves to a seal chunk, which has no dependents, so the shortfall stays invisible until the counts are compared. All suites green on the merged tree: 421 db, 492 web, 61 ingest, 45 shared, 20 etl, 10 config, 91 root scripts, 248 pipeline; lint, typecheck, check:docs and the coverage ratchet clean. --- packages/db/src/refresh-slice.test.ts | 3 +++ .../db/src/value-flag-annex-step-sql.test.ts | 5 ++++ .../db/src/value-flag-stotinki-sql.test.ts | 5 ++++ scripts/ship-e2e.test.mjs | 26 +++++++++++++++---- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 58c434d24..9c76fb83f 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -662,6 +662,9 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + // 0006 too: refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279), so the + // script cannot parse against a DB that stops at 0003 — every site here applies both. + readScript(dbPath, migration6Path); readScript(dbPath, workStagingSchemaPath); // An EOP procedure (tender + base contract) with УНП UNP-SLICE / tender.id TENDER-SLICE. An diff --git a/packages/db/src/value-flag-annex-step-sql.test.ts b/packages/db/src/value-flag-annex-step-sql.test.ts index ea9f91f50..c2afb6848 100644 --- a/packages/db/src/value-flag-annex-step-sql.test.ts +++ b/packages/db/src/value-flag-annex-step-sql.test.ts @@ -26,6 +26,10 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script +// only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it +// ever reaches the value_flag CASE these tests are about. +const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const promotePath = resolve(root, 'scripts/promote-amendments.sql'); @@ -60,6 +64,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/value-flag-stotinki-sql.test.ts b/packages/db/src/value-flag-stotinki-sql.test.ts index 2fc97624e..eb41d8609 100644 --- a/packages/db/src/value-flag-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-stotinki-sql.test.ts @@ -20,6 +20,10 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script +// only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it +// ever reaches the value_flag CASE these tests are about. +const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -49,6 +53,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/scripts/ship-e2e.test.mjs b/scripts/ship-e2e.test.mjs index cf971a2e8..60d6d67ef 100644 --- a/scripts/ship-e2e.test.mjs +++ b/scripts/ship-e2e.test.mjs @@ -28,6 +28,10 @@ const HERE = dirname(fileURLToPath(import.meta.url)); const ROOT = resolve(HERE, '..'); const SCRIPT = resolve(HERE, 'ship-related-persons.mjs'); const MIG = resolve(ROOT, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// 0006 too: the evidence seal is one of the shipped tables (#279, ADR-0033), so it appears in the ship's +// TABLES and WIPE_ORDER. A target built from 0003 alone makes the generated wipe abort on „no such table" +// before a single row moves — the schema under test has to be the schema the ship writes. +const MIG_EVIDENCE = resolve(ROOT, 'packages/db/migrations/0006_interest_link_evidence.sql'); const D1_NAME = 'sigma-test-local'; // Derived from BOTH production constants, and the run below does NOT override either: one row past @@ -51,11 +55,12 @@ if (spawnSync('sqlite3', ['-version'], { stdio: 'ignore' }).error) { throw new Error('scripts/ship-e2e.test.mjs requires the sqlite3 binary on PATH'); } -/** The five served tables plus the two FK parents they reference. */ +/** The served свързани-лица tables plus the two FK parents they reference. */ const SCHEMA = `PRAGMA foreign_keys=ON; CREATE TABLE bidders(id TEXT PRIMARY KEY); CREATE TABLE authorities(id TEXT PRIMARY KEY); .read ${MIG} +.read ${MIG_EVIDENCE} INSERT INTO bidders(id) VALUES('eik:1'); INSERT INTO authorities(id) VALUES('auth:1');`; @@ -66,7 +71,11 @@ INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VAL ${Array.from( { length: links }, (_, i) => - `INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('il${i}','p1|${i}','p1','eik:1','1','e','v1','B_distinctive','owns','published');`, + `INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('il${i}','p1|${i}','p1','eik:1','1','e','v1','B_distinctive','owns','published');\n` + + // One seal per link. Since #279 a published link without one is exactly the state the audit fails + // the run on (C_no_evidence) and the read gate refuses to surface, so a corpus without seals is not + // a shape the ship should ever be asked to carry. + `INSERT INTO interest_link_evidence(link_key,evidence_kind,lookup_date,rules_version,live_status) VALUES('p1|${i}','document','2026-08-13','tr-rules-1','live');`, ).join('\n')} `; // interest_link_authorities is deliberately left EMPTY above: its expected count is 0, which is the @@ -77,13 +86,15 @@ INSERT INTO persons(id,name) VALUES('stale','Стар запис'); INSERT INTO declarations(id,person_id,xml_file,folder_year,template,source_url) VALUES('sd','stale','s.xml','2019','assets','u'); INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VALUES('sdi','sd','S','s','shares'); INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('sil','stale|0','stale','eik:1','1','s','v0','B_distinctive','owns','published'); -INSERT INTO interest_link_authorities(link_key,authority_id,authority_name) VALUES('stale|0','auth:1','A');`; +INSERT INTO interest_link_authorities(link_key,authority_id,authority_name) VALUES('stale|0','auth:1','A'); +INSERT INTO interest_link_evidence(link_key,evidence_kind,lookup_date,rules_version,live_status) VALUES('stale|0','confirmed','2019-01-01','tr-rules-0','live');`; const EXPECTED_ROWS = { persons: 1, declarations: 1, declared_interests: 1, interest_links: LINKS, + interest_link_evidence: LINKS, // one seal per link (#279) interest_link_authorities: 0, }; @@ -255,10 +266,15 @@ test('a request that never landed fails the run', (t) => { const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-short-')); t.after(() => rmSync(dir, { recursive: true, force: true })); - const { res } = runShip(dir, { env: { SHIP_FAKE_SKIP: 'interest_links.2.sql' } }); + // The skip targets a LEAF table on purpose. Since #279 the ship also carries interest_link_evidence, + // which has an FK to interest_links — so dropping an interest_links chunk now fails on the FOREIGN KEY + // as the orphaned seals land, before the read-back ever runs. That is a stronger guard, but it would + // leave the read-back gate itself unexercised, which is what this test is for. A seal chunk has no + // dependents, so its loss is invisible until the counts are compared. + const { res } = runShip(dir, { env: { SHIP_FAKE_SKIP: 'interest_link_evidence.2.sql' } }); assert.notEqual(res.status, 0, 'a short target must fail the run'); assert.match(res.stderr, /ship verification FAILED/); - assert.match(res.stderr, /interest_links: shipped \d+, target has \d+/); + assert.match(res.stderr, /interest_link_evidence: shipped \d+, target has \d+/); }); test('a read-back that answers with a non-number fails closed', (t) => { From 24c80016550e90a8c25475c825e194dc1cd35c93 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 14 Aug 2026 09:49:41 +0300 Subject: [PATCH 45/50] fix(db): leave migration 0003 untouched and let 0007 enforce for every database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0003 is already applied everywhere, so adding CHECKs to it was the wrong move even though that is where the table is declared. `CREATE TABLE IF NOT EXISTS` is a no-op against an existing table and ship-related-persons wipes ROWS, never definitions — so the constraints would have existed only on databases built after the edit. Two schemas under one name, with the gate absent precisely on the served database, which is where a hand-run `UPDATE … SET status='published '` during an incident actually lands. The comment promised the opposite of what the deployed base would have had. 0003 is restored byte-for-byte and enforcement moves entirely into 0007, which both paths reach: a fresh chain runs it after 0003, a deployed database gets it as a retrofit. 0007 already carried the triggers, the natural-key index and the dedup, so nothing is lost — enforcement simply becomes uniform. A fresh build keeps 0003's table-level UNIQUE too; it treats NULLs as distinct, so it is strictly weaker than the index and rejects nothing the index accepts. 0006 keeps its own CHECKs: it is new here and has never been applied to a served environment, so declaring them there edits no applied history. The parity test is the point: a fresh 0000..0007 build and a legacy database retrofitted by 0007 must reject the SAME values and accept the SAME ones, including the UPDATE path no CHECK on a legacy table would ever have covered. Mutation-checked — dropping the UPDATE trigger fails it. Verified on a real local D1 built from the reverted 0003, not only in tests: the bad UPDATE returns SQLITE_CONSTRAINT_TRIGGER and the legitimate one succeeds. The dedup DELETE now announces itself before changing live data, so a deployment that drops declarations says how many instead of leaving a row count nobody recorded. Its subtraction is parenthesised because `||` binds tighter than `-` in SQLite — the first real apply printed "notice: -2" and proved it. No explicit transaction: `d1 execute --file` already runs the file as one, so a later failure rolls the DELETE back with it. --- .../0003_related_persons_foundation.sql | 33 +------ .../0007_publishing_gate_constraints.sql | 58 ++++++++---- packages/db/src/migrations.test.ts | 88 +++++++++++++++++++ 3 files changed, 133 insertions(+), 46 deletions(-) diff --git a/packages/db/migrations/0003_related_persons_foundation.sql b/packages/db/migrations/0003_related_persons_foundation.sql index 58a858788..6ad3be975 100644 --- a/packages/db/migrations/0003_related_persons_foundation.sql +++ b/packages/db/migrations/0003_related_persons_foundation.sql @@ -29,24 +29,9 @@ CREATE TABLE IF NOT EXISTS declarations ( category TEXT, -- CACBG category (e.g. Народни представители) institution TEXT, -- the body the official serves in position TEXT, -- declared position - source_url TEXT NOT NULL -- register.cacbg.bg// — provenance + source_url TEXT NOT NULL, -- register.cacbg.bg// — provenance + UNIQUE (xml_file, control_hash) ); --- Re-import idempotence (#279 §2). This REPLACES a table-level `UNIQUE (xml_file, control_hash)`, which --- did not constrain anything it was meant to: --- 1. SQLite counts NULLs as DISTINCT, and `control_hash` is genuinely optional at the source — the --- register omits on some declarations and parse.mjs carries that through as null. So --- every hashless declaration was mutually unique and re-imported as a fresh row each run, --- double-counting the stakes it carries. COALESCE folds them to one value. --- 2. `xml_file` is not unique ACROSS FOLDERS — the register splits years into suffixed folders and --- reuses basenames — which load.mjs already knows, since it namespaces the declaration id by folder --- for exactly this reason. Without folder_year here the two keys disagree, and the narrower one --- would reject a second official's genuinely different declaration. --- --- `control_hash` deliberately stays NULLABLE. NOT NULL was the other candidate, but it converts an --- optional source field into a run-stopping loader failure, and fabricating a hash to satisfy it would --- assert an integrity check we never performed. -CREATE UNIQUE INDEX IF NOT EXISTS idx_declarations_natural_key - ON declarations(xml_file, folder_year, COALESCE(control_hash, '')); -- Per-person lookup: audit.mjs scans a person's declarations, and LINK_SELECT's source_url subquery -- filters declarations by person_id — without this it's a full scan of the table. CREATE INDEX IF NOT EXISTS idx_declarations_person ON declarations(person_id); @@ -90,12 +75,7 @@ CREATE TABLE IF NOT EXISTS interest_links ( -- management_role — self, relation manages, a single declarant (ambiguous: private manager or small board) -- Materiality: only CLOSELY-HELD forms (ООД/ЕООД/ЕТ/…) reach ownership classes; listed АД/ЕАД securities -- and management-only roles never become private_ownership/family_ownership (ADR-0022). - -- fail-closed: a NON-surfaced class, so a writer that sets status but forgets the class can't leak to - -- the public surface (load.mjs always sets it explicitly). CHECKed for the same reason as `status`: - -- the surface reads this as an enum, so a typo'd class is a gate failure, not a cosmetic one. - interest_class TEXT NOT NULL DEFAULT 'management_role' - CHECK (interest_class IN ('private_ownership','family_ownership', - 'ex_officio_board','management_role')), + interest_class TEXT NOT NULL DEFAULT 'management_role', -- fail-closed: a NON-surfaced class, so a writer that sets status but forgets the class can't leak to the public surface (load.mjs always sets it explicitly) contemporaneous INTEGER NOT NULL DEFAULT 0, own_institution TEXT NOT NULL DEFAULT 'none', -- exact (deterministic) | locality (heuristic) | none evidence_count INTEGER NOT NULL DEFAULT 1, -- # declared_interests supporting this link @@ -107,12 +87,7 @@ CREATE TABLE IF NOT EXISTS interest_links ( contract_value_eur REAL, -- SUM(contracts.amount_eur); NULL if none summable first_contract_year TEXT, last_contract_year TEXT, - -- CHECKed, not merely documented (#279 §2). The public surface is `status = 'published'`, so a value - -- that only LOOKS like one — 'published ' with a trailing space is the case the ticket names — passes - -- every writer and then fails the gate silently, or slips a LIKE. A CHECK binds every writer at once, - -- including a hand-run UPDATE during an incident, which is exactly when this is most likely typed. - status TEXT NOT NULL DEFAULT 'held' - CHECK (status IN ('published','internal','held','withdrawn','suppressed')), + status TEXT NOT NULL DEFAULT 'held', -- published (public surface: private/family ownership) | internal (non-surfaced class) | held | withdrawn | suppressed verified_by TEXT, verified_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')) diff --git a/packages/db/migrations/0007_publishing_gate_constraints.sql b/packages/db/migrations/0007_publishing_gate_constraints.sql index 7720ac458..b3e9d4cf0 100644 --- a/packages/db/migrations/0007_publishing_gate_constraints.sql +++ b/packages/db/migrations/0007_publishing_gate_constraints.sql @@ -1,14 +1,17 @@ --- Retrofit the publishing-gate constraints onto an ALREADY-DEPLOYED database (#279 §2). +-- The publishing-gate constraints, for EVERY database — freshly built and already-deployed alike (#279 §2). -- --- WHY THIS EXISTS AT ALL, given 0003 and 0006 now declare the same constraints: --- `CREATE TABLE IF NOT EXISTS` is a no-op against a table that already exists, and --- `scripts/ship-related-persons.mjs` wipes ROWS (`DELETE FROM`, WIPE_ORDER) and never table --- definitions. So every environment that applied 0003 before today keeps the unconstrained shape --- forever, no matter how many times the data is reloaded. Only a rebuild can change it. +-- WHY THIS MIGRATION OWNS THEM RATHER THAN 0003 (todorkolev, #309). The obvious move is to add the CHECKs +-- to `interest_links` in 0003, where the table is declared. That does not work and is worse than not +-- trying: 0003 is ALREADY APPLIED everywhere, `CREATE TABLE IF NOT EXISTS` is a no-op against an existing +-- table, and `ship-related-persons.mjs` wipes ROWS (`DELETE FROM`, WIPE_ORDER), never definitions. So an +-- in-place CHECK would exist only on databases built after the edit — putting two different schemas under +-- one name, with the constraint absent precisely on the served database. That is where a hand-run +-- `UPDATE … SET status='published '` during an incident actually lands. -- --- WHY IT IS SAFE TO RE-RUN: migrations here are applied by a bare `wrangler d1 execute --file` with no --- applied-migrations tracking (see related-persons-data.yml / deploy.yml), so re-application MUST be a --- no-op. Everything below is `IF NOT EXISTS` over statements that converge. +-- So 0003 stays byte-identical to what was applied, and enforcement lives here, reached by both paths: +-- a fresh chain runs 0007 after 0003, and a deployed database gets it as a retrofit. One shape, one +-- mechanism. `packages/db/src/migrations.test.ts` holds the two shapes to the same rejections and the same +-- acceptances, so this stays true rather than merely intended. -- -- WHY TRIGGERS AND NOT A TABLE REBUILD: SQLite cannot add a CHECK in place, so the textbook route is -- create-copy-drop-rename. That route is WRONG here, and a real `wrangler d1 migrations apply` proved it: @@ -18,10 +21,14 @@ -- child first would work mechanically but would strip every evidence seal, and since the read gate -- REQUIRES a seal that empties the public surface until the next monthly data run. -- --- A BEFORE INSERT/UPDATE trigger that RAISEs enforces the identical invariant for every writer — --- including a hand-run UPDATE during an incident, which is when a bad value is most likely typed — with --- no rebuild, no FK exposure and no seal loss. Freshly created databases still get true CHECK constraints --- from 0003/0006; this is the retrofit path for databases that already exist. +-- A BEFORE INSERT/UPDATE trigger that RAISEs enforces the identical invariant for every writer — including +-- the hand-run UPDATE above, which no CHECK on a legacy table would ever have covered — with no rebuild, +-- no FK exposure and no seal loss. 0006 keeps its own CHECKs: it is NEW in this change, has never been +-- applied to a served environment, so declaring them there edits no applied history. +-- +-- WHY IT IS SAFE TO RE-RUN: migrations here are applied by a bare `wrangler d1 execute --file` with no +-- applied-migrations tracking (see related-persons-data.yml / deploy.yml), so re-application MUST be a +-- no-op. Everything below is `IF NOT EXISTS` over statements that converge. -- ── declarations ──────────────────────────────────────────────────────────────────────────────────── -- The table-level `UNIQUE (xml_file, control_hash)` did not constrain what it was written for. SQLite @@ -36,10 +43,27 @@ -- optional source field into a run-stopping loader failure, and a fabricated placeholder hash would -- assert an integrity check nobody performed. -- --- No table rebuild is needed for this one: dropping the old constraint means dropping the table it lives --- on, but the same effect is had by creating the correct index — and the stale UNIQUE, being strictly --- weaker, rejects nothing the new index accepts. Duplicates ALREADY stored under the old non-constraint --- would block the new index, so they are collapsed first, keeping the earliest row per natural key. +-- No table rebuild is needed for this one: the table-level `UNIQUE (xml_file, control_hash)` declared in +-- 0003 STAYS (0003 is not edited — see the header), and the correct index is simply added alongside it. +-- The stale constraint is strictly WEAKER than the index — it treats NULLs as distinct, so it accepts a +-- superset — and therefore rejects nothing the index accepts. The index governs. +-- +-- Duplicates ALREADY stored under the old non-constraint would block the index, so they are collapsed +-- first, keeping the earliest row per natural key. +-- +-- The DELETE below changes live data, so it ANNOUNCES itself first (cefothe, #309): the count is emitted +-- before the rows go, and a run that removes nothing says so. Without it the only evidence a deployment +-- silently dropped declarations would be a row count nobody recorded beforehand. It needs no explicit +-- transaction — `wrangler d1 execute --file` runs the file as one implicit transaction, so a failure +-- anywhere below rolls the DELETE back with it rather than leaving a half-collapsed table. +-- The subtraction is parenthesised deliberately: `||` binds TIGHTER than `-` in SQLite, so without it +-- this reads as ('…' || countA) - (countB || '…') — two strings coerced to numbers — and reports a +-- meaningless figure instead of the row count. It did, before a real apply showed „notice: -2". +SELECT 'migration 0007: collapsing ' || + ((SELECT COUNT(*) FROM declarations) - + (SELECT COUNT(*) FROM (SELECT 1 FROM declarations + GROUP BY xml_file, folder_year, COALESCE(control_hash, '')))) || + ' duplicate declaration row(s) before the natural-key index' AS notice; DELETE FROM declarations WHERE id NOT IN ( SELECT MIN(id) FROM declarations GROUP BY xml_file, folder_year, COALESCE(control_hash, '') ); diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index bc5d9a225..25e5ef128 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -299,6 +299,94 @@ describe('served migrations', () => { } }); + // THE POINT OF 0007, stated as an executable claim rather than a comment (todorkolev, #309). + // + // 0003 is an ALREADY-APPLIED migration and is therefore never edited: `CREATE TABLE IF NOT EXISTS` is a + // no-op on a live database, so an in-place CHECK would exist only on freshly built ones. That would put + // two different schemas under one name — and the divergence would land precisely on the served database, + // which is where a hand-run `UPDATE status='published '` during an incident actually happens. + // + // So 0007 owns the enforcement for BOTH shapes, and this test is what keeps them one shape: a fresh + // 0000..0007 build and a legacy database retrofitted by 0007 must reject the SAME values and accept the + // SAME values. If either drifts, the parity assertions below fail rather than a reviewer noticing. + it('a fresh build and a retrofitted legacy database enforce the SAME gate', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-parity-')); + const fresh = resolve(dir, 'fresh.sqlite'); + const legacy = resolve(dir, 'legacy.sqlite'); + try { + // (1) fresh: the migration chain exactly as a new environment applies it. + for (const m of [migration0, migration3, migration6, migration7]) readScript(fresh, m); + + // (2) legacy: 0003 + 0006 as they were BEFORE this PR, then 0007 retrofits. `interest_links` is + // recreated without constraints because that is the shape every already-deployed database holds. + for (const m of [migration0, migration3, migration6]) readScript(legacy, m); + sqlite( + legacy, + `DROP TABLE interest_link_evidence; + DROP TABLE interest_links; + CREATE TABLE interest_links ( + id TEXT PRIMARY KEY, link_key TEXT NOT NULL UNIQUE, + person_id TEXT NOT NULL REFERENCES persons(id), bidder_id TEXT NOT NULL, eik TEXT NOT NULL, + entity_key TEXT NOT NULL, match_method TEXT, matcher_version TEXT NOT NULL, + publish_tier TEXT NOT NULL, relation TEXT NOT NULL, + interest_class TEXT NOT NULL DEFAULT 'management_role', + contemporaneous INTEGER NOT NULL DEFAULT 0, own_institution TEXT NOT NULL DEFAULT 'none', + evidence_count INTEGER NOT NULL DEFAULT 1, first_declared_year TEXT, last_declared_year TEXT, + contract_count INTEGER NOT NULL DEFAULT 0, contract_value_eur REAL, first_contract_year TEXT, + last_contract_year TEXT, status TEXT NOT NULL DEFAULT 'held', verified_by TEXT, + verified_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now'))); + CREATE TABLE interest_link_evidence ( + link_key TEXT PRIMARY KEY REFERENCES interest_links(link_key), evidence_kind TEXT NOT NULL, + registry_role TEXT, matched_fact TEXT, entry_number TEXT, entry_date TEXT, + lookup_date TEXT NOT NULL, rules_version TEXT NOT NULL, live_status TEXT NOT NULL, + sealed_at TEXT NOT NULL DEFAULT (datetime('now')));`, + ); + readScript(legacy, migration7); + + const link = (db: string, id: string, status: string, cls = 'private_ownership') => + sqlite( + db, + `INSERT INTO persons(id, name) VALUES ('person:a', 'А') ON CONFLICT DO NOTHING; + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:${id}', '${id}', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + '${cls}', '${status}');`, + ); + const seal = (db: string, key: string, kind: string) => + sqlite( + db, + `INSERT INTO interest_link_evidence + (link_key, evidence_kind, lookup_date, rules_version, live_status) + VALUES ('${key}', '${kind}', '2026-08-14', 'tr-rules-1', 'live');`, + ); + + for (const [label, db] of [ + ['fresh', fresh], + ['legacy', legacy], + ] as const) { + // REJECTED identically — 'published ' is #279 §2's named case, and it is the one a human types. + expect(() => link(db, `bad1-${label}`, 'published ')).toThrow(/CHECK/i); + expect(() => link(db, `bad2-${label}`, 'publushed')).toThrow(/CHECK/i); + expect(() => link(db, `bad3-${label}`, 'held', 'private_ownership ')).toThrow(/CHECK/i); + // ACCEPTED identically — the gate is a bound, not a blanket, on both shapes. + expect(() => link(db, `ok-${label}`, 'published')).not.toThrow(); + expect(() => seal(db, `ok-${label}`, 'document')).not.toThrow(); + expect(() => seal(db, `ok-${label}`, 'document_uncorroborated')).toThrow(/UNIQUE|CHECK/i); + // …and the evidence enum is enforced on both too. + expect(() => link(db, `k2-${label}`, 'held')).not.toThrow(); + expect(() => seal(db, `k2-${label}`, 'document_uncorroberated')).toThrow(/CHECK/i); + // The UPDATE path is the one an incident actually uses, and no CHECK would ever cover it on a + // legacy table — only the trigger does. + expect(() => + sqlite(db, `UPDATE interest_links SET status='published ' WHERE link_key='ok-${label}';`), + ).toThrow(/CHECK/i); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + // The retrofit has to be safe on a LIVE D1: ship-related-persons wipes rows, never table definitions, // so a deployed database keeps its unconstrained 0003 shape until this migration rebuilds it. it('0007 retrofits a pre-existing database, keeps its rows, and re-applies as a no-op', () => { From 23a1939a50bf330082b5f0e9813cf2a67d936727 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 14 Aug 2026 09:51:38 +0300 Subject: [PATCH 46/50] test: assert the seal vocabulary with the production predicate, not a looser copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both seal tests restated MATCHED_FACT_RE locally, and both got it wrong in the permissive direction: `seat:` followed by unlimited uppercase tokens, where production bounds a settlement to two. So `seat:ИВАН ПЕТРОВ ГЕОРГИЕВ` — a three-part Bulgarian name (ЗГР чл. 9) wearing an allowed prefix, which is exactly what a mis-split seat field produces — passed both tests and fails the real rail. They could not fail on the regression they exist to catch, which is the one shape this rail keeps off a served column. Both now import `isSealedFact` from the module that WRITES the vocabulary, so the assertion and the enforcement cannot drift apart again. Adds the direct unit test neither file had: MATCHED_FACT_RE accepts real one- and two-token settlements including a hyphenated one, accepts every role/field-code shape and `eik`, and rejects the three-token seat, a bare name, a name where a field code belongs, a role outside the vocabulary, an empty settlement and a seat carrying the ЕИК itself. Mutation-checked: widening the production bound to the old local pattern now fails with "wrongly accepted: seat:ИВАН ПЕТРОВ ГЕОРГИЕВ" — under the previous tests that same change was silent. --- scripts/cacbg/load.test.mjs | 7 ++++-- scripts/tr/evidence.test.mjs | 48 +++++++++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index 468f11fa7..987cbdd8a 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -10,6 +10,8 @@ import path from 'node:path'; import os from 'node:os'; import { fileURLToPath } from 'node:url'; import { fingerprint } from './suppressions.mjs'; +// The seal vocabulary is asserted with the PRODUCTION predicate, imported from the module that writes it. +import { isSealedFact } from '../tr/evidence.mjs'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); @@ -1241,9 +1243,10 @@ test('every link carries an evidence seal, and no seal carries a name', () => { const seals = db.prepare('SELECT * FROM interest_link_evidence').all(); assert.equal(seals.length, links.length, 'a seal for EVERY link, held and withdrawn included'); - const CLOSED = /^(?:seat:[\p{Lu} -]+|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + // The PRODUCTION predicate, imported from the module that WRITES the vocabulary — a restated regex + // here was looser than the real one and would have passed a seat token carrying a full name. for (const s of seals) { - if (s.matched_fact != null) assert.match(s.matched_fact, CLOSED, s.matched_fact); + assert.ok(isSealedFact(s.matched_fact), s.matched_fact); assert.ok(s.rules_version.length > 0); assert.match(s.lookup_date, /^\d{4}-\d{2}-\d{2}$/); } diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs index 321230706..d3f71085c 100644 --- a/scripts/tr/evidence.test.mjs +++ b/scripts/tr/evidence.test.mjs @@ -6,7 +6,13 @@ // ownership claim, it attaches a real official to the wrong company's ЕИК, contracts and money. import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { RULES_VERSION, evidenceVerdict, reconcileTermination } from './evidence.mjs'; +import { + RULES_VERSION, + evidenceVerdict, + reconcileTermination, + MATCHED_FACT_RE, + isSealedFact, +} from './evidence.mjs'; const container = (t) => `

    ${t}

    `; @@ -452,8 +458,44 @@ test('a missing deed that is NOT marked outside-ТР is an error, not a silent h }); // ── the seal ────────────────────────────────────────────────────────────────── +test('MATCHED_FACT_RE bounds a settlement to two tokens — a NAME cannot wear the seat: prefix', () => { + // The rail tested DIRECTLY, not just through whatever verdicts the ladder happens to produce. Both + // seal tests previously restated this regex locally and got it WRONG in the permissive direction — + // `seat:` followed by unlimited uppercase tokens — so a three-part Bulgarian name (ЗГР чл. 9) wearing + // an allowed prefix passed them. That value is exactly what a mis-split of the seat field produces, + // and it is the one shape this rail exists to keep off a served column. + for (const ok of [ + 'seat:СОФИЯ', + 'seat:ВЕЛИКО ТЪРНОВО', // a real two-token settlement must still pass + 'seat:ГЕНЕРАЛ ТОШЕВО', + 'seat:ЦАР-КАЛОЯН', // hyphenated is one token + 'role:owner:CR_F_19_L', + 'role:manager:CR_F_7_L', + 'role:owner:CR_F_23_L', + 'eik', + ]) + assert.equal(MATCHED_FACT_RE.test(ok), true, `wrongly rejected: ${ok}`); + + for (const bad of [ + 'seat:ИВАН ПЕТРОВ ГЕОРГИЕВ', // THE case: three tokens is a name, not a settlement + 'seat:ИВАН ПЕТРОВ ГЕОРГИЕВ ДРУГ', + 'ИВАН ПЕТРОВ ГЕОРГИЕВ', // a bare name with no prefix at all + 'role:owner:ИВАН ПЕТРОВ', // a name where a field code belongs + 'role:cashier:CR_F_19_L', // a role outside the vocabulary + 'seat:', // an empty settlement asserts nothing + 'eik:201122335', // the ЕИК itself is never stored, only the fact that one matched + ]) + assert.equal(MATCHED_FACT_RE.test(bad), false, `wrongly accepted: ${bad}`); + + // null is legal — a rung may match no fact — and that is isSealedFact's job, not the regex's. + assert.equal(isSealedFact(null), true); + assert.equal(isSealedFact('seat:ИВАН ПЕТРОВ ГЕОРГИЕВ'), false); +}); + test('matched_fact stays inside the closed vocabulary — it can never carry a name', () => { - const CLOSED = /^(?:seat:[\p{Lu} -]+|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + // The PRODUCTION predicate, imported — never a local copy of it. A re-stated regex here was looser + // than `MATCHED_FACT_RE` (it allowed `seat:` + unlimited tokens), so this loop certified values the + // real rail rejects and could not fail on the regression it exists to catch (cefothe, #309). for (const v of [ evidenceVerdict(base), evidenceVerdict({ ...base, deed: deed([fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))]) }), @@ -468,7 +510,7 @@ test('matched_fact stays inside the closed vocabulary — it can never carry a n }), ]) { if (v.matchedFact == null) continue; - assert.match(v.matchedFact, CLOSED, `matched_fact escaped the vocabulary: ${v.matchedFact}`); + assert.ok(isSealedFact(v.matchedFact), `matched_fact escaped the vocabulary: ${v.matchedFact}`); assert.ok(!/ИВАН|ПЕТРОВ|ТЕСТОВ/.test(v.matchedFact), 'a NAME reached matched_fact'); } }); From 28137261004dcb0bcd8c75f3f2bd7ef8a5992d02 Mon Sep 17 00:00:00 2001 From: Yoan Dimitrov Date: Fri, 14 Aug 2026 09:58:47 +0300 Subject: [PATCH 47/50] fix(web,db,docs): render the entry number, bind the join invariant, correct the docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit registryEntryNumber was in the DTO, mapped in the query and present in every fixture, and rendered nowhere — carried to every client for nothing. It is also what makes the claim findable: a date does not identify a record in the register, and a reader checking whether we read the act correctly needs the number. Shown only when present, since a seat/ЕИК confirmation cites no act entry and an empty „№" would read as missing data rather than an inapplicable field. (cefothe cited ADR-0033 as requiring this. It does not — no such text exists there. The dead payload stands on its own.) The contract join is now pinned writer-to-reader. On the read side `tenders`/`authorities` are never projected, so both joins look dead and invite deletion — a reviewer proposed exactly that. They are not dead, they are the SHAPE: the writer at load.mjs stores contract_count/contract_value_eur through the same four tables, so deleting them on the read side alone would let the read exceed the stored aggregate and surface links the zero-contract gate excluded. Deleting them on BOTH sides is defensible but changes published figures and needs re-baselining against ADR-0033 §10, so it stays #226 §1.6. A comment cannot survive a refactor; the test compares both by relation and join condition. Mutation-checked in both directions. Docs: SUPPRESSION_KEY_VERSION is a repository VARIABLE, not a secret — looking for it under Secrets during an incident is a dead end. The suppression runbook gains the verification it lacked: a takedown that silently failed looks exactly like one that worked, so it now says to check the served D1 and the page, allow for the 1h public cache, and re-run the next load to prove the removal survives a rebuild. deploy.md no longer claims deploys do not migrate — deploy.yml applies 0003/0006/0007 explicitly, and why. review-testing.md records scripts/ as a reviewed carve-out from the ratchet, naming the three runners and the mutation discipline that stands in for a percentage. --- apps/web/app/components/ConflictCards.tsx | 5 +++ apps/web/app/routes/conflicts.render.test.tsx | 16 +++++++++ docs/deploy.md | 11 ++++-- docs/review-testing.md | 20 +++++++++++ docs/runbooks/related-persons-suppression.md | 30 ++++++++++++++-- packages/db/src/queries/related-persons.ts | 12 +++++++ packages/db/src/related-persons-sql.test.ts | 34 +++++++++++++++++++ scripts/cacbg/load.mjs | 6 ++++ 8 files changed, 129 insertions(+), 5 deletions(-) diff --git a/apps/web/app/components/ConflictCards.tsx b/apps/web/app/components/ConflictCards.tsx index 28783efd3..a716c425e 100644 --- a/apps/web/app/components/ConflictCards.tsx +++ b/apps/web/app/components/ConflictCards.tsx @@ -193,6 +193,11 @@ function ConflictCard({ {registryEvidenceLabel(l)} {l.registryEntryDate ? ` · вписване ${l.registryEntryDate}` : ''} + {/* The entry NUMBER, not just its date: a date does not identify a record, and this is + what a reader types to find the same act we read. Rendered only when present — + a seat/ЕИК confirmation cites no act entry, and an empty „№" would read as missing + data rather than as an inapplicable field. */} + {l.registryEntryNumber ? ` · № ${l.registryEntryNumber}` : ''} {l.registryLookupDate ? ` · справка ${l.registryLookupDate}` : ''} diff --git a/apps/web/app/routes/conflicts.render.test.tsx b/apps/web/app/routes/conflicts.render.test.tsx index 07339f058..814c36708 100644 --- a/apps/web/app/routes/conflicts.render.test.tsx +++ b/apps/web/app/routes/conflicts.render.test.tsx @@ -268,6 +268,22 @@ describe('Trade Register evidence on the card (#279, ADR-0033)', () => { expect(text).toContain('лицето е вписано като съдружник/собственик'); expect(text).toContain('вписване 2011-05-02'); // WHICH entry expect(text).toContain('справка 2026-08-05'); // and HOW FRESH it is + // The entry NUMBER is what makes the claim findable in the register — a date alone does not + // identify a record. It was carried to every client in the DTO and never rendered, which is the + // one payload that costs bytes and answers nothing (cefothe, #309). + expect(text).toContain('20110502101007'); + }); + + it('omits the entry number rather than printing an empty label when there is none', async () => { + // POSITIVE CONTROL for the row's shape: a confirmed link (seat/ЕИК) has no act entry to cite, so + // the label must be absent entirely — not „· №" with nothing after it, which reads as missing data + // rather than as an inapplicable field. Scoped to the evidence label: „№" alone is the card RANK. + await renderConflicts([ + link({ evidenceKind: 'confirmed', registryRole: null, registryEntryNumber: null }), + ]); + const text = container.textContent ?? ''; + expect(text).toContain('Регистър'); + expect(text).not.toContain('· №'); }); it('a seat/ЕИК confirmation never implies somebody was found in the act', async () => { diff --git a/docs/deploy.md b/docs/deploy.md index ac3e845ce..eb25a430f 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -170,8 +170,15 @@ domain таблиците и преизчислява rollup-ите + FTS. > [scripts/precompute.sql](../scripts/precompute.sql) — не изпращайте FTS съдържание през dump. > Промени в схемата след първото зареждане се прилагат out-of-band, за всяка среда: -> `SIGMA_D1_NAME=sigma-stage wrangler d1 migrations apply sigma-stage --remote`. Деплоите не мигрират -> и не презареждат данни. +> `SIGMA_D1_NAME=sigma-stage wrangler d1 migrations apply sigma-stage --remote`. Деплоите **не презареждат +> данни**. +> +> **Уточнение (беше неточно):** тук пишеше и „деплоите не мигрират". Вече не е вярно — +> [deploy.yml](../.github/workflows/deploy.yml) прилага изрично `0003`, `0006` и `0007` с +> `d1 execute --file` при всеки деплой. Причината е, че ledger-ът на wrangler на тези бази е празен +> (базовата схема е създадена out-of-band), тъй че `d1 migrations apply` би се сблъскал с `0000`. И трите +> са безопасни за повторно прилагане: `CREATE … IF NOT EXISTS` и, при `0007`, тригери и индекси, които +> се пресъздават до същото състояние. Останалите схемни промени наистина си остават out-of-band. ## 3. Конфигуриране на GitHub Environments diff --git a/docs/review-testing.md b/docs/review-testing.md index 3ab4b106a..0cbd4b0af 100644 --- a/docs/review-testing.md +++ b/docs/review-testing.md @@ -43,6 +43,26 @@ - Нов workspace с тестове се добавя и в `coverage-baseline.json` (и получава `vitest.config.ts` с `sharedCoverage(...)`); `packages/api-contract` е освободен, докато няма тестове. +### `scripts/` е извън ratchet-а — прегледано изключение, не пропуск + +Ratchet-ът покрива само `apps/*` и `packages/*`. `scripts/` (~7 хил. реда, сред които клеветнически +критичната логика на свързаните лица — доказателственият ред, съвпадението по имена, гейтовете при +публикуване) не се измерва от него. Записано е тук, за да е решение, а не дупка, която някой открива +по-късно. + +**Защо:** тези файлове са `.mjs`, пускат се от три отделни runner-а извън `turbo run test` и голяма част +от тях са I/O обвивки около мрежа и SQLite, за които процент покритие би измервал колко от обвивката е +докосната, а не дали правилото е вярно. + +**Какво пази качеството вместо процент:** и трите набора са в CI и са задължителни — +`node --test scripts/*.test.mjs`, `node --import ./scripts/cacbg/register-ts.mjs --test +scripts/cacbg/*.test.mjs scripts/tr/*.test.mjs` — а гейтовете, при които грешка означава поименно +твърдение за реален човек, се проверяват **мутационно**: тестът трябва да пада при обърнато условие, +иначе не е гейт. Виж ADR-0027 за защо тест, който минава и преди реализацията, е фалшив гейт. + +**Кога това трябва да се преразгледа:** ако `scripts/` започне да носи логика, която не е нито +чиста функция с мутационно проверен тест, нито тънка I/O обвивка. + ## Integrity gate - Пуска се върху обслужвания D1 след `precompute` в `ship-domain.mjs` и след `runSliceDerive()` в diff --git a/docs/runbooks/related-persons-suppression.md b/docs/runbooks/related-persons-suppression.md index 1fffa8c19..7d407cd37 100644 --- a/docs/runbooks/related-persons-suppression.md +++ b/docs/runbooks/related-persons-suppression.md @@ -112,6 +112,30 @@ If a ship fails on any of these, fix the cause; do **not** empty the list or dro A rotated salt invalidates every existing fingerprint, so it is a coordinated change guarded by `key_version`: re-derive `fp` for each entry from its original `link_key` under the new salt (you need the plaintext keys, kept out-of-band, e.g. in the incident tickets), **bump every entry's `key_version`** to the -new value, replace the file in one commit, and update both CI secrets (`SUPPRESSION_SALT` and -`SUPPRESSION_KEY_VERSION`). Because the loader refuses any entry on a non-current `key_version`, a -half-finished rotation fails the build loudly instead of silently un-suppressing. +new value, replace the file in one commit, and update **both** CI settings — but note they are different +kinds: `SUPPRESSION_SALT` is a repository **secret**, while `SUPPRESSION_KEY_VERSION` is a repository +**variable** (`${{ vars.SUPPRESSION_KEY_VERSION || '1' }}` in `related-persons-data.yml`). The version is +not sensitive — it is a counter — and looking for it under Secrets is a dead end during an incident. +Because the loader refuses any entry on a non-current `key_version`, a half-finished rotation fails the +build loudly instead of silently un-suppressing. + +## Verifying a takedown actually worked + +A takedown that silently failed looks exactly like one that succeeded: the entry sits in the list, the +build is green, and the link is still public. The list is applied at LOAD time, so nothing changes on the +served surface until the next data run ships — check the surface, not the commit. + +1. **The loader saw it.** The run refuses an entry matching no built link (the B3 unused-entry rail), so a + green run already proves the fingerprint matched something. A run that fails with „suppression matched + NO link" means the `link_key` was wrong — a family link needs its `|family` suffix. +2. **The row is gone from the served D1**, which is the only copy a reader can reach: + ``` + wrangler d1 execute "$SIGMA_D1_NAME" --remote \ + --command "SELECT status FROM interest_links WHERE link_key = ''" + ``` + Expect `suppressed`, or no row at all. Anything else means the ship did not carry the decision. +3. **The page is gone**, allowing for cache: the link's page must 404 and the official's page must not + list it. `publicCache(3600)` means a reader can still see it for up to an hour after the write — if it + is still there beyond that, the takedown did not land. +4. **It stays gone.** Re-run the next scheduled load and repeat step 2: the list is what makes a takedown + survive a rebuild, and this is the only step that proves the survival rather than assuming it. diff --git a/packages/db/src/queries/related-persons.ts b/packages/db/src/queries/related-persons.ts index b149ebbd9..25f2a7524 100644 --- a/packages/db/src/queries/related-persons.ts +++ b/packages/db/src/queries/related-persons.ts @@ -92,6 +92,18 @@ interface LinkRow { // The winner's contracts, joined exactly as the ETL aggregate does (contracts→tenders→authorities→bidders, // matched by eik_normalized) so any read-time subset is a true subset of the stored contract_count/value. // Alias-distinct (cc/tt/aa/bb) so it composes as a correlated subquery under the LINK_SELECT `il`/`b` scope. +// +// `tt` and `aa` are NOT projected here, which makes both joins look dead and invites deleting them. They +// are not dead — they are the SHAPE, and its counterpart is the WRITER at scripts/cacbg/load.mjs (the +// per-winner contract query that fills contract_count / contract_value_eur). The two must stay identical. +// +// Removing them on the read side ALONE would widen the read to contracts the stored aggregate never +// counted: `contemporaneous_contract_count` could then exceed `contract_count`, and the EXISTS gate below +// (the one that decides whether a link surfaces at all) would publish links the I5 zero-contract gate had +// excluded. Removing them on BOTH sides is defensible — an unresolvable authority currently drops a +// contract from the money everywhere, consistently — but it changes published figures and so must be +// re-baselined against ADR-0033 §10's control totals, not slipped in as a cleanup. Tracked as #226 §1.6. +// `related-persons-sql.test.ts` pins this string to the writer's so neither can be "optimised" alone. const CONTRACT_JOIN = `FROM contracts cc JOIN tenders tt ON tt.id = cc.tender_id JOIN authorities aa ON aa.id = tt.authority_id diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index 1501f16db..f389d5252 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -10,6 +10,7 @@ import { LEADERBOARD_SQL, LINK_CONTRACTS_LIMIT, LINK_CONTRACTS_SQL, + LINK_SELECT, OFFICIAL_SQL, SURFACED_OWNERSHIP, } from './queries/related-persons'; @@ -632,6 +633,39 @@ describe('свързани-лица SQL (real SQLite)', () => { }); }); +// The read-time contract join and the WRITER that fills contract_count/contract_value_eur must use the +// same join shape. On the read side `tenders`/`authorities` are never projected, so both joins read as +// dead and a reviewer will eventually propose deleting them (one did — cefothe on #309). Deleting them +// there alone widens the read past the stored aggregate: contemporaneous counts could exceed +// contract_count, and the EXISTS gate would surface links the zero-contract gate excluded. Nothing but a +// test can hold a TypeScript template string and a .mjs prepared statement to the same shape. +describe('the read-time contract join matches the writer that stored the aggregate', () => { + const writer = readFileSync(resolve(root, 'scripts/cacbg/load.mjs'), 'utf8'); + // The writer's per-winner contract query, located by its projection rather than by line number. + const writerJoin = /FROM contracts c JOIN tenders t ON[^"]*?WHERE b\.eik_normalized/.exec( + writer, + )?.[0]; + + it('the writer still joins contracts→tenders→authorities→bidders', () => { + expect(writerJoin, 'load.mjs per-winner contract query not found').toBeTruthy(); + for (const rel of ['tenders', 'authorities', 'bidders']) expect(writerJoin).toContain(rel); + }); + + it('CONTRACT_JOIN joins the same four relations, in the same direction', () => { + // Compared by RELATION and join condition, not by text: the two are written in different languages + // and alias differently (cc/tt/aa/bb vs c/t/a/b), so only the shape is comparable. + const flat = LINK_SELECT.replace(/\s+/g, ' '); + expect(flat).toContain('FROM contracts cc'); + expect(flat).toContain('JOIN tenders tt ON tt.id = cc.tender_id'); + expect(flat).toContain('JOIN authorities aa ON aa.id = tt.authority_id'); + expect(flat).toContain('JOIN bidders bb ON bb.id = cc.bidder_id'); + // …and the writer's equivalents, so a change to either side fails here. + expect(writerJoin).toContain('JOIN tenders t ON t.id=c.tender_id'); + expect(writerJoin).toContain('JOIN authorities a ON a.id=t.authority_id'); + expect(writerJoin).toContain('JOIN bidders b ON b.id=c.bidder_id'); + }); +}); + // The evidence-seal gate has FOUR copies, and they must say the same thing. Reviewers found the fourth // (the company-search badge) checking `status='published'` alone, so search advertised a свързани-лица // claim about a named official that the detail page correctly withheld. `refresh-slice.sql` matters for a diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 31e627c95..18a806f12 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -552,6 +552,12 @@ for (const r of readJsonl(path.join(STAGING, 'related.jsonl'))) { db.exec('COMMIT'); // --- enrich each (person,eik) → interest_links (+ per-authority breakdown) ----------------------- +// THE WRITER of contract_count / contract_value_eur. Its join shape (contracts→tenders→authorities→ +// bidders) is mirrored by CONTRACT_JOIN in packages/db/src/queries/related-persons.ts, so that the +// read-time subset can never exceed what was stored. `authorities` here IS projected (the per-authority +// breakdown needs the name), unlike on the read side where it looks dead — that asymmetry is why the two +// have to be pinned to each other rather than reasoned about separately. related-persons-sql.test.ts +// asserts they agree; change one only by changing both, and re-baseline ADR-0033 §10 when you do. const contractStmt = db.prepare( "SELECT strftime('%Y', c.signed_at) yr, a.id auth_id, a.name authority, c.amount_eur eur FROM contracts c JOIN tenders t ON t.id=c.tender_id JOIN authorities a ON a.id=t.authority_id JOIN bidders b ON b.id=c.bidder_id WHERE b.eik_normalized=?", ); From 0a97059bf290c0629466f00e964e9d4456694180 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 14 Aug 2026 12:34:30 +0000 Subject: [PATCH 48/50] =?UTF-8?q?fix(db,ci,docs):=20=D0=BF=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BE=D0=BC=D0=B5=D1=80=D0=B8=D1=80=D0=B0=D0=B9=20=D0=BC?= =?UTF-8?q?=D0=B8=D0=B3=D1=80=D0=B0=D1=86=D0=B8=D0=B8=D1=82=D0=B5=20=D0=BD?= =?UTF-8?q?=D0=B0=200009/0010=20=D0=B8=20=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=20=D0=BA=D0=BE=D0=BC=D0=B5=D0=BD=D1=82=D0=B0=D1=80=D0=B0?= =?UTF-8?q?=20=D0=B7=D0=B0=200010?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Преномериране: #307 взе 0006/0007, #308 взе 0008, тъй че тези две се местят на 0009 и 0010. Обновени са всички препратки - тестове, deploy.yml, related-persons-data.yml, scripts/cacbg/load.mjs, docs/deploy.md. Коментарът над прилагането на 0010 в deploy.yml беше останал от предишния замисъл и твърдеше три неверни неща: че 0003 обявявала CHECK-овете (тя е върната непокътната и не обявява нищо), че се слагало control_hash NOT NULL (остава NULL-ируема нарочно - регистърът я пропуска за част от декларациите, а индексът по естествен ключ ги сгъва с COALESCE) и че миграцията е „rebuild-based" (вече е тригерна; заглавието ѝ обяснява защо пресъздаването е опасно - оголва външните ключове и обира доказателствените печати). Последното беше най-неприятно: обещаваше на следващия четец точно операцията, която самата миграция отхвърля, а „0003 declares them now" щеше да го прати обратно да добавя CHECK-ове в приложена миграция. docs/deploy.md вече изброява и стъпките, които сондират таблицата и добавят само липсващите колони, за да не изглежда, че --file е единственият механизъм. db 424 зелени, typecheck и prettier чисти, scripts/tr 134/134. --- .github/workflows/deploy.yml | 22 ++++++++++++------- .github/workflows/related-persons-data.yml | 4 ++-- docs/deploy.md | 8 ++++--- ...ce.sql => 0009_interest_link_evidence.sql} | 0 ...l => 0010_publishing_gate_constraints.sql} | 0 .../db/src/contractor-identity-sql.test.ts | 2 +- .../etl-entity-canonicalization-sql.test.ts | 2 +- packages/db/src/integrity-checks.test.ts | 2 +- packages/db/src/migrations.test.ts | 4 ++-- packages/db/src/refresh-slice.test.ts | 2 +- packages/db/src/related-persons-sql.test.ts | 2 +- packages/db/src/search-sql.test.ts | 2 +- packages/db/src/seed-surface.test.ts | 2 +- .../db/src/value-flag-annex-step-sql.test.ts | 2 +- .../db/src/value-flag-stotinki-sql.test.ts | 2 +- packages/ingest/src/refresh-officials.test.ts | 2 +- scripts/cacbg/load.mjs | 2 +- scripts/ship-e2e.test.mjs | 2 +- scripts/ship-reseed.test.mjs | 2 +- 19 files changed, 36 insertions(+), 28 deletions(-) rename packages/db/migrations/{0006_interest_link_evidence.sql => 0009_interest_link_evidence.sql} (100%) rename packages/db/migrations/{0007_publishing_gate_constraints.sql => 0010_publishing_gate_constraints.sql} (100%) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 86567ef8d..30ffb5bba 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -145,20 +145,26 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0003_related_persons_foundation.sql - # 0006 attaches the Trade Register evidence seal (#279, ADR-0033). Both migrations are + # 0009 attaches the Trade Register evidence seal (#279, ADR-0033). Both migrations are # applied by name here because wrangler's migration ledger is empty on this D1 (the base # schema was created out-of-band), so `d1 migrations apply` would collide on 0000. pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ --config wrangler.deploy.jsonc --remote --yes \ - --file ../../packages/db/migrations/0006_interest_link_evidence.sql - # 0007 retrofits the publishing-gate CHECK constraints and control_hash NOT NULL onto an - # ALREADY-PROVISIONED D1 (#279 §2). 0003 declares them now, but `CREATE TABLE IF NOT EXISTS` - # never revisits an existing table and the ship step wipes ROWS, not definitions — so without - # this a live database keeps the unconstrained shape no matter how often the data reloads. - # Rebuild-based and therefore idempotent: re-applying copies an already-constrained table. + --file ../../packages/db/migrations/0009_interest_link_evidence.sql + # 0010 owns the publishing-gate enforcement for EVERY database (#279 §2) — fresh and already + # deployed alike. It is deliberately NOT declared in 0003: that migration is already applied + # everywhere, `CREATE TABLE IF NOT EXISTS` never revisits an existing table, and the ship step + # wipes ROWS, not definitions — so an in-place CHECK would exist only on databases built after + # the edit, and be absent on exactly the database that serves the site. Enforced with BEFORE + # INSERT/UPDATE triggers, NOT a table rebuild: the create-copy-drop-rename route would expose + # the foreign keys and strip every evidence seal, emptying the public surface until the next + # monthly run (see the migration header). `control_hash` stays NULLABLE by design — the register + # omits it on some declarations; the natural-key index folds those with COALESCE instead. + # Idempotent: every statement is `IF NOT EXISTS` over a converging definition, so re-applying it + # on each deploy is a no-op. Verified by three consecutive applications, gate still enforcing. pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ --config wrangler.deploy.jsonc --remote --yes \ - --file ../../packages/db/migrations/0007_publishing_gate_constraints.sql + --file ../../packages/db/migrations/0010_publishing_gate_constraints.sql # The base schema was created out-of-band with `d1 execute --file`, so wrangler's migration # ledger is empty and `d1 migrations apply` would collide on 0000. Probe the actual table # instead. SQLite has no `ADD COLUMN IF NOT EXISTS`; a completion-marker table is created only diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index 40dfb5ac9..2beb27062 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -311,7 +311,7 @@ jobs: # schema was created out-of-band), so `d1 migrations apply` would collide on 0000. pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote --yes \ - --file ../../packages/db/migrations/0006_interest_link_evidence.sql + --file ../../packages/db/migrations/0009_interest_link_evidence.sql # 0007 retrofits the publishing-gate CHECK constraints and control_hash NOT NULL onto an # ALREADY-PROVISIONED D1 (#279 §2). 0003 declares them now, but `CREATE TABLE IF NOT EXISTS` # never revisits an existing table and the ship step wipes ROWS, not definitions — so without @@ -319,7 +319,7 @@ jobs: # Rebuild-based and therefore idempotent: re-applying copies an already-constrained table. pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote --yes \ - --file ../../packages/db/migrations/0007_publishing_gate_constraints.sql + --file ../../packages/db/migrations/0010_publishing_gate_constraints.sql - name: Ship свързани-лица tables → D1 # ship-related-persons.mjs shells out to `wrangler d1 execute` with cwd=apps/web. Two things it diff --git a/docs/deploy.md b/docs/deploy.md index eb25a430f..e296eda86 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -174,11 +174,13 @@ domain таблиците и преизчислява rollup-ите + FTS. > данни**. > > **Уточнение (беше неточно):** тук пишеше и „деплоите не мигрират". Вече не е вярно — -> [deploy.yml](../.github/workflows/deploy.yml) прилага изрично `0003`, `0006` и `0007` с +> [deploy.yml](../.github/workflows/deploy.yml) прилага изрично `0003`, `0009` и `0010` с > `d1 execute --file` при всеки деплой. Причината е, че ledger-ът на wrangler на тези бази е празен > (базовата схема е създадена out-of-band), тъй че `d1 migrations apply` би се сблъскал с `0000`. И трите -> са безопасни за повторно прилагане: `CREATE … IF NOT EXISTS` и, при `0007`, тригери и индекси, които -> се пресъздават до същото състояние. Останалите схемни промени наистина си остават out-of-band. +> са безопасни за повторно прилагане: `CREATE … IF NOT EXISTS` и, при `0010`, тригери и индекси, които +> се пресъздават до същото състояние. Отделно от тях има и стъпки, които сондират таблицата и добавят +> само липсващите колони — за `0002` и за колоните на анексите. Останалите схемни промени наистина си +> остават out-of-band. ## 3. Конфигуриране на GitHub Environments diff --git a/packages/db/migrations/0006_interest_link_evidence.sql b/packages/db/migrations/0009_interest_link_evidence.sql similarity index 100% rename from packages/db/migrations/0006_interest_link_evidence.sql rename to packages/db/migrations/0009_interest_link_evidence.sql diff --git a/packages/db/migrations/0007_publishing_gate_constraints.sql b/packages/db/migrations/0010_publishing_gate_constraints.sql similarity index 100% rename from packages/db/migrations/0007_publishing_gate_constraints.sql rename to packages/db/migrations/0010_publishing_gate_constraints.sql diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index ea4de1cef..ce1961acb 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -18,7 +18,7 @@ const migration3 = readFileSync( ); // …and 0006, which those same blocks now join for the Trade Register evidence gate (#279, ADR-0033). const migration6 = readFileSync( - resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'), + resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'), 'utf8', ); const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index 9ffdea5b0..da3ed90ff 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -12,7 +12,7 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ // refresh-slice.sql's officials block reads interest_links (0003); build it so the script doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). -const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index 645af634a..415fe7510 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -30,7 +30,7 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ // precompute.sql's officials block reads interest_links (0003); build it so precompute doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). -const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const precomputePath = resolve(root, 'scripts/precompute.sql'); function sqlite(dbPath: string, sql: string): void { diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index 25e5ef128..a879754a0 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -11,8 +11,8 @@ const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); const migration2 = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); -const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); -const migration7 = resolve(root, 'packages/db/migrations/0007_publishing_gate_constraints.sql'); +const migration6 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration7 = resolve(root, 'packages/db/migrations/0010_publishing_gate_constraints.sql'); const backfill = resolve(root, 'scripts/backfill-current-value-currency.sql'); const precompute = resolve(root, 'scripts/precompute.sql'); diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 9c76fb83f..232307c90 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -14,7 +14,7 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ // refresh-slice.sql / precompute.sql officials block reads interest_links (0003) — build it in every chain. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). -const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const deriveAmendmentsPath = resolve(root, 'scripts/derive-amendments.sql'); diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index f389d5252..66cee0ebb 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -26,7 +26,7 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006: SURFACED_OWNERSHIP now requires a Trade Register evidence seal (#279, ADR-0033). -const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }).trim(); diff --git a/packages/db/src/search-sql.test.ts b/packages/db/src/search-sql.test.ts index f371cd1df..4b412d217 100644 --- a/packages/db/src/search-sql.test.ts +++ b/packages/db/src/search-sql.test.ts @@ -16,7 +16,7 @@ import { SEARCH_HITS_SQL, SEARCH_HITS_SQL_NO_CONFLICT } from './queries/search'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); -const migration6 = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function readScript(dbPath: string, path: string): void { execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${path}\n`, stdio: 'pipe' }); diff --git a/packages/db/src/seed-surface.test.ts b/packages/db/src/seed-surface.test.ts index e9484d25a..78a70aaa9 100644 --- a/packages/db/src/seed-surface.test.ts +++ b/packages/db/src/seed-surface.test.ts @@ -21,7 +21,7 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migrations = [ 'packages/db/migrations/0000_init.sql', 'packages/db/migrations/0003_related_persons_foundation.sql', - 'packages/db/migrations/0006_interest_link_evidence.sql', + 'packages/db/migrations/0009_interest_link_evidence.sql', ].map((m) => resolve(root, m)); const seed = resolve(root, 'scripts/seed.sql'); diff --git a/packages/db/src/value-flag-annex-step-sql.test.ts b/packages/db/src/value-flag-annex-step-sql.test.ts index c2afb6848..c5cbb2a19 100644 --- a/packages/db/src/value-flag-annex-step-sql.test.ts +++ b/packages/db/src/value-flag-annex-step-sql.test.ts @@ -29,7 +29,7 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script // only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it // ever reaches the value_flag CASE these tests are about. -const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const promotePath = resolve(root, 'scripts/promote-amendments.sql'); diff --git a/packages/db/src/value-flag-stotinki-sql.test.ts b/packages/db/src/value-flag-stotinki-sql.test.ts index eb41d8609..615ffc3aa 100644 --- a/packages/db/src/value-flag-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-stotinki-sql.test.ts @@ -23,7 +23,7 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script // only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it // ever reaches the value_flag CASE these tests are about. -const migration6Path = resolve(root, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], diff --git a/packages/ingest/src/refresh-officials.test.ts b/packages/ingest/src/refresh-officials.test.ts index 6f3eb4c60..2c0ebb8f8 100644 --- a/packages/ingest/src/refresh-officials.test.ts +++ b/packages/ingest/src/refresh-officials.test.ts @@ -22,7 +22,7 @@ const migrations = [ '0003_related_persons_foundation.sql', // The officials batch now joins the Trade Register evidence seal (#279, ADR-0033) — without 0006 the // real refresh-slice.sql this test executes cannot parse. - '0006_interest_link_evidence.sql', + '0009_interest_link_evidence.sql', ].map((f) => resolve(root, 'packages/db/migrations', f)); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 18a806f12..ca7c030c5 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -48,7 +48,7 @@ const MIGRATION = path.join(ROOT, 'packages/db/migrations/0003_related_persons_f // missing from the work DB would make every evidence write fail at ship time instead of at load. const MIGRATION_EVIDENCE = path.join( ROOT, - 'packages/db/migrations/0006_interest_link_evidence.sql', + 'packages/db/migrations/0009_interest_link_evidence.sql', ); const REPORT = path.join(STAGING, 'findings.md'); // Bumped for #279: classify-2 (КДА added to the joint-stock bar) + tr-1 (identity now rests on a diff --git a/scripts/ship-e2e.test.mjs b/scripts/ship-e2e.test.mjs index 60d6d67ef..1d1509148 100644 --- a/scripts/ship-e2e.test.mjs +++ b/scripts/ship-e2e.test.mjs @@ -31,7 +31,7 @@ const MIG = resolve(ROOT, 'packages/db/migrations/0003_related_persons_foundatio // 0006 too: the evidence seal is one of the shipped tables (#279, ADR-0033), so it appears in the ship's // TABLES and WIPE_ORDER. A target built from 0003 alone makes the generated wipe abort on „no such table" // before a single row moves — the schema under test has to be the schema the ship writes. -const MIG_EVIDENCE = resolve(ROOT, 'packages/db/migrations/0006_interest_link_evidence.sql'); +const MIG_EVIDENCE = resolve(ROOT, 'packages/db/migrations/0009_interest_link_evidence.sql'); const D1_NAME = 'sigma-test-local'; // Derived from BOTH production constants, and the run below does NOT override either: one row past diff --git a/scripts/ship-reseed.test.mjs b/scripts/ship-reseed.test.mjs index 9c5b66172..44020c5c1 100644 --- a/scripts/ship-reseed.test.mjs +++ b/scripts/ship-reseed.test.mjs @@ -17,7 +17,7 @@ const HERE = dirname(fileURLToPath(import.meta.url)); const MIG = resolve(HERE, '..', 'packages/db/migrations/0003_related_persons_foundation.sql'); // 0006 adds interest_link_evidence, which references interest_links — so it is part of the FK graph // this test exists to police, and the wipe must clear it before its parent (#279, ADR-0033). -const MIG_EVIDENCE = resolve(HERE, '..', 'packages/db/migrations/0006_interest_link_evidence.sql'); +const MIG_EVIDENCE = resolve(HERE, '..', 'packages/db/migrations/0009_interest_link_evidence.sql'); // Stub only the two 0000 tables 0002's FKs reference (bidders, authorities) — a PK column is all a FK needs — // then apply the real 0002 and seed one row down every FK chain so a parent delete has live children. From 0d8be1e1a0d9553ea91ebfa4d5a232b9ab9795e2 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 14 Aug 2026 12:39:11 +0000 Subject: [PATCH 49/50] =?UTF-8?q?refactor(db):=20=D0=BF=D1=80=D0=B5=D0=B8?= =?UTF-8?q?=D0=BC=D0=B5=D0=BD=D1=83=D0=B2=D0=B0=D0=B9=20=D0=BF=D1=80=D0=BE?= =?UTF-8?q?=D0=BC=D0=B5=D0=BD=D0=BB=D0=B8=D0=B2=D0=B8=D1=82=D0=B5=20=D0=BD?= =?UTF-8?q?=D0=B0=20=D0=BC=D0=B8=D0=B3=D1=80=D0=B0=D1=86=D0=B8=D0=B8=D1=82?= =?UTF-8?q?=D0=B5=209/10=20=D0=B2=20=D1=82=D0=B5=D1=81=D1=82=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D1=82=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Преномерирането смени пътищата, но остави имената migration6Path/migration7Path, което е подвеждащо и се сблъсква с едноименните променливи, които #307 и #308 въведоха за 0006/0007/0008. --- .../db/src/contractor-identity-sql.test.ts | 4 ++-- .../etl-entity-canonicalization-sql.test.ts | 4 ++-- packages/db/src/integrity-checks.test.ts | 4 ++-- packages/db/src/migrations.test.ts | 24 +++++++++---------- packages/db/src/refresh-slice.test.ts | 16 ++++++------- packages/db/src/related-persons-sql.test.ts | 4 ++-- packages/db/src/search-sql.test.ts | 4 ++-- .../db/src/value-flag-annex-step-sql.test.ts | 4 ++-- .../db/src/value-flag-stotinki-sql.test.ts | 4 ++-- 9 files changed, 34 insertions(+), 34 deletions(-) diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index ce1961acb..b463831c2 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -17,7 +17,7 @@ const migration3 = readFileSync( 'utf8', ); // …and 0006, which those same blocks now join for the Trade Register evidence gate (#279, ADR-0033). -const migration6 = readFileSync( +const migration9 = readFileSync( resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'), 'utf8', ); @@ -71,7 +71,7 @@ function build(path: 'normalize' | 'refresh'): DatabaseSync { const db = new DatabaseSync(':memory:'); db.exec(schema); db.exec(migration2); - db.exec(migration3 + migration6); + db.exec(migration3 + migration9); db.exec(staging); db.exec(seed); if (path === 'normalize') { diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index da3ed90ff..4c8950883 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -12,7 +12,7 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ // refresh-slice.sql's officials block reads interest_links (0003); build it so the script doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). -const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -42,7 +42,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index 415fe7510..f6d9c7ff0 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -30,7 +30,7 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ // precompute.sql's officials block reads interest_links (0003); build it so precompute doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). -const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const precomputePath = resolve(root, 'scripts/precompute.sql'); function sqlite(dbPath: string, sql: string): void { @@ -74,7 +74,7 @@ function freshDb(): string { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); sqlite(dbPath, CLEAN_FIXTURE); return dbPath; } diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index a879754a0..a37050b2e 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -11,8 +11,8 @@ const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); const migration2 = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); -const migration6 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); -const migration7 = resolve(root, 'packages/db/migrations/0010_publishing_gate_constraints.sql'); +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration10 = resolve(root, 'packages/db/migrations/0010_publishing_gate_constraints.sql'); const backfill = resolve(root, 'scripts/backfill-current-value-currency.sql'); const precompute = resolve(root, 'scripts/precompute.sql'); @@ -133,7 +133,7 @@ describe('served migrations', () => { ); readScript(dbPath, migration2); readScript(dbPath, migration3); - readScript(dbPath, migration6); + readScript(dbPath, migration9); readScript(dbPath, backfill); readScript(dbPath, precompute); @@ -163,7 +163,7 @@ describe('served migrations', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-0006-')); const dbPath = resolve(dir, 'test.sqlite'); try { - for (const m of [migration0, migration1, migration2, migration3, migration6]) + for (const m of [migration0, migration1, migration2, migration3, migration9]) readScript(dbPath, m); expect( @@ -188,7 +188,7 @@ describe('served migrations', () => { (link_key, evidence_kind, matched_fact, lookup_date, rules_version, live_status) VALUES ('k', 'document', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live');`, ); - readScript(dbPath, migration6); // idempotent re-apply + readScript(dbPath, migration9); // idempotent re-apply expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_link_evidence;').trim()).toBe('1'); // The FK is real: a seal for a link that does not exist is rejected. D1 enforces foreign keys, @@ -215,7 +215,7 @@ describe('served migrations', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-')); const dbPath = resolve(dir, 'test.sqlite'); try { - for (const m of [migration0, migration3, migration6, migration7]) readScript(dbPath, m); + for (const m of [migration0, migration3, migration9, migration10]) readScript(dbPath, m); const seedLink = (status: string) => sqlite( @@ -315,11 +315,11 @@ describe('served migrations', () => { const legacy = resolve(dir, 'legacy.sqlite'); try { // (1) fresh: the migration chain exactly as a new environment applies it. - for (const m of [migration0, migration3, migration6, migration7]) readScript(fresh, m); + for (const m of [migration0, migration3, migration9, migration10]) readScript(fresh, m); // (2) legacy: 0003 + 0006 as they were BEFORE this PR, then 0007 retrofits. `interest_links` is // recreated without constraints because that is the shape every already-deployed database holds. - for (const m of [migration0, migration3, migration6]) readScript(legacy, m); + for (const m of [migration0, migration3, migration9]) readScript(legacy, m); sqlite( legacy, `DROP TABLE interest_link_evidence; @@ -341,7 +341,7 @@ describe('served migrations', () => { lookup_date TEXT NOT NULL, rules_version TEXT NOT NULL, live_status TEXT NOT NULL, sealed_at TEXT NOT NULL DEFAULT (datetime('now')));`, ); - readScript(legacy, migration7); + readScript(legacy, migration10); const link = (db: string, id: string, status: string, cls = 'private_ownership') => sqlite( @@ -393,7 +393,7 @@ describe('served migrations', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-')); const dbPath = resolve(dir, 'test.sqlite'); try { - for (const m of [migration0, migration3, migration6]) readScript(dbPath, m); + for (const m of [migration0, migration3, migration9]) readScript(dbPath, m); // Simulate a database provisioned BEFORE today: 0003 now declares the constraints, so the legacy // shape has to be recreated explicitly. This is the state every already-deployed environment is in // — `CREATE TABLE IF NOT EXISTS` never revisited it and ship-related-persons only wipes rows. @@ -441,7 +441,7 @@ describe('served migrations', () => { 'private_ownership', 'published ');`, ); - readScript(dbPath, migration7); + readScript(dbPath, migration10); // The valid rows survive — nothing is rebuilt, so nothing can be lost… expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( 'd:keep', @@ -490,7 +490,7 @@ describe('served migrations', () => { // Migrations here are applied by a bare `d1 execute --file` with no applied-migrations tracking, // so a second application MUST be a no-op rather than an error or a data loss. - readScript(dbPath, migration7); + readScript(dbPath, migration10); expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('2'); expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( 'd:keep', diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 232307c90..f5d8dc250 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -14,7 +14,7 @@ const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_ // refresh-slice.sql / precompute.sql officials block reads interest_links (0003) — build it in every chain. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). -const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const deriveAmendmentsPath = resolve(root, 'scripts/derive-amendments.sql'); @@ -187,7 +187,7 @@ function initWorkDb(dbPath: string): void { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); } @@ -579,7 +579,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -664,7 +664,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration3Path); // 0006 too: refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279), so the // script cannot parse against a DB that stops at 0003 — every site here applies both. - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); // An EOP procedure (tender + base contract) with УНП UNP-SLICE / tender.id TENDER-SLICE. An @@ -848,7 +848,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -900,7 +900,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -952,7 +952,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -1090,7 +1090,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index 66cee0ebb..ddcf4287a 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -26,7 +26,7 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); // …and 0006: SURFACED_OWNERSHIP now requires a Trade Register evidence seal (#279, ADR-0033). -const migration6 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }).trim(); @@ -143,7 +143,7 @@ describe('свързани-лица SQL (real SQLite)', () => { try { readScript(dbPath, migration0); readScript(dbPath, migration2); - readScript(dbPath, migration6); + readScript(dbPath, migration9); sqlite(dbPath, FIXTURE); return fn(dbPath); } finally { diff --git a/packages/db/src/search-sql.test.ts b/packages/db/src/search-sql.test.ts index 4b412d217..59f9a95c8 100644 --- a/packages/db/src/search-sql.test.ts +++ b/packages/db/src/search-sql.test.ts @@ -16,7 +16,7 @@ import { SEARCH_HITS_SQL, SEARCH_HITS_SQL_NO_CONFLICT } from './queries/search'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); -const migration6 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function readScript(dbPath: string, path: string): void { execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${path}\n`, stdio: 'pipe' }); @@ -141,7 +141,7 @@ function withDb(fn: (dbPath: string) => void): void { try { readScript(dbPath, migration0); readScript(dbPath, migration2); - readScript(dbPath, migration6); + readScript(dbPath, migration9); exec(dbPath, FIXTURE); exec(dbPath, POPULATE_INDEX); fn(dbPath); diff --git a/packages/db/src/value-flag-annex-step-sql.test.ts b/packages/db/src/value-flag-annex-step-sql.test.ts index c5cbb2a19..1ffd29f3c 100644 --- a/packages/db/src/value-flag-annex-step-sql.test.ts +++ b/packages/db/src/value-flag-annex-step-sql.test.ts @@ -29,7 +29,7 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script // only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it // ever reaches the value_flag CASE these tests are about. -const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const promotePath = resolve(root, 'scripts/promote-amendments.sql'); @@ -64,7 +64,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/value-flag-stotinki-sql.test.ts b/packages/db/src/value-flag-stotinki-sql.test.ts index 615ffc3aa..3a7225410 100644 --- a/packages/db/src/value-flag-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-stotinki-sql.test.ts @@ -23,7 +23,7 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person // refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script // only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it // ever reaches the value_flag CASE these tests are about. -const migration6Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -53,7 +53,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); - readScript(dbPath, migration6Path); + readScript(dbPath, migration9Path); readScript(dbPath, stagingPath); run(dbPath); } finally { From 215123debc93b5b73573bc3faf09a61120902ccc Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 14 Aug 2026 14:04:44 +0000 Subject: [PATCH 50/50] Merge origin/main into feat/registry-evidence-links (#304) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Единственият конфликт е познатият: новият тест на #304 строи схемата без 0009, а refresh-slice.sql чете interest_link_evidence. Добавена е миграцията в неговата верига. db 487, web 493, ingest 84, etl 20, shared 45, config 10. Typecheck и prettier чисти. --- packages/db/src/value-flag-lot-stotinki-sql.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/db/src/value-flag-lot-stotinki-sql.test.ts b/packages/db/src/value-flag-lot-stotinki-sql.test.ts index 388d31729..67fea3526 100644 --- a/packages/db/src/value-flag-lot-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-lot-stotinki-sql.test.ts @@ -28,6 +28,8 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +// #279/ADR-0033: refresh-slice.sql's свързани-лица block reads interest_link_evidence, so 0009 too. +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -60,6 +62,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); + readScript(dbPath, migration9Path); readScript(dbPath, stagingPath); run(dbPath); } finally {