diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e15e693de..2ed7a02ac 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -145,6 +145,26 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0003_related_persons_foundation.sql + # 0009 attaches the Trade Register evidence seal (#279, ADR-0033). Both migrations are + # applied by name here because wrangler's migration ledger is empty on this D1 (the base + # schema was created out-of-band), so `d1 migrations apply` would collide on 0000. + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0009_interest_link_evidence.sql + # 0010 owns the publishing-gate enforcement for EVERY database (#279 §2) — fresh and already + # deployed alike. It is deliberately NOT declared in 0003: that migration is already applied + # everywhere, `CREATE TABLE IF NOT EXISTS` never revisits an existing table, and the ship step + # wipes ROWS, not definitions — so an in-place CHECK would exist only on databases built after + # the edit, and be absent on exactly the database that serves the site. Enforced with BEFORE + # INSERT/UPDATE triggers, NOT a table rebuild: the create-copy-drop-rename route would expose + # the foreign keys and strip every evidence seal, emptying the public surface until the next + # monthly run (see the migration header). `control_hash` stays NULLABLE by design — the register + # omits it on some declarations; the natural-key index folds those with COALESCE instead. + # Idempotent: every statement is `IF NOT EXISTS` over a converging definition, so re-applying it + # on each deploy is a no-op. Verified by three consecutive applications, gate still enforcing. + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0010_publishing_gate_constraints.sql # The base schema was created out-of-band with `d1 execute --file`, so wrangler's migration # ledger is empty and `d1 migrations apply` would collide on 0000. Probe the actual table # instead. SQLite has no `ADD COLUMN IF NOT EXISTS`; a completion-marker table is created only diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index a133ed54b..2beb27062 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -16,17 +16,63 @@ name: Related-persons data foundation (build + ship) # XML (fetch.mjs skips files already on disk) — it fills gaps, not a from-scratch rebuild unless the cache is cleared. on: + # MONTHLY, and deliberately on the EXISTING workflow rather than a second one. #279 §9 asks for the + # publishing decisions to run on a cadence; duplicating the job would create a second ship path with + # its own copy of the credential guards, the D1-target guard and the ship floor — and the copy is the + # one nobody exercises. + # + # WHY MONTHLY AND NOT DAILY, which is what §9 and ADR-0033 originally describe. The decision and its + # rejected alternatives are recorded in ADR-0034, which supersedes ADR-0033's cadence line — this + # comment is the operational summary, not the record, so a reader who disagrees has somewhere to argue. + # A decision cannot be recomputed + # without the raw deeds — evidenceVerdict's strongest rung matches the declarant's name against the + # register's own text — and the raw deeds must not be persisted between runs: they carry the names and + # addresses of co-owners and managers who hold no public office, and an Actions cache entry lives on + # GitHub's storage under a restore-keys chain where ADR-0033 decision 5's 35-day retention cannot reach + # it. So the deeds live and die with one runner, and every run that decides must also crawl. A daily + # decision would therefore mean ~400 daily requests against somebody else's register, which is what + # spec §3.3 exists to prevent — so the cadence follows the deeds, not the other way round. + # + # Restoring a daily decision needs the crawl to emit the per-(link, ЕИК) match verdict, so only + # booleans cross a run boundary and nothing has to hold a name. That is a design change, not a + # schedule change, and it is deliberately not folded in here. + # + # A scheduled run takes the same path as a manual one, with two differences: + # • it targets STAGING (the environment default below). Production stays manual, so the + # assertD1TargetAuthorized guard keeps a prod write a deliberate act. + # • `full_crawl` is empty and therefore false, so it never crawls the CACBG corpus — it re-runs the + # decisions over the cached corpus, which is exactly what #279 §9 means by a pure, zero-network + # decision. Registry lookups are the one exception and run from inside this job on the 1st of the + # month (see „Decide whether to hit the register"): they cannot be a separate workflow, because the + # raw deeds they fetch carry third-party names and must not be persisted between runners. + # If the corpus cache has been evicted, extract yields nothing, the ship floor refuses, and the run + # fails loudly rather than wiping the surface. + schedule: + # 03:00 UTC on the 1st. See the note above: the decision and the crawl are one pass, so the decision + # cadence is the crawl cadence — #279 §9's monthly registry rhythm. + - cron: '0 3 1 * *' workflow_dispatch: inputs: environment: description: Target D1 environment type: choice - options: [dev, staging, production] - default: dev + # No `dev` environment exists — every fallback in this file is `|| 'staging'` and the note above + # says the job targets staging. A `dev` default sent a manual dispatch to an environment with no + # secrets, so it failed at the credential guard rather than running where it was meant to. + options: [staging, production] + default: staging full_crawl: description: Crawl the CACBG corpus (slow). RESUMES from the cached raw XML — fetch.mjs skips files already on disk, so this fills gaps rather than re-fetching from scratch (clear the cache for a true rebuild). Off = extract from the existing raw cache without crawling. type: boolean default: false + tr_max_age_days: + description: Re-fetch deeds older than this many days (default 30). Use a large value to force a full refresh. + type: string + default: '30' + tr_limit: + description: Stop after this many registry lookups (blank = all pending). Bound a first run while watching for a 429. + type: string + default: '' # Least-privilege: this workflow only reads the repo; it authenticates to Cloudflare with a scoped token. permissions: @@ -34,19 +80,20 @@ permissions: concurrency: # Never let two data-foundation writes to the same environment overlap; do not cancel one in flight. - group: related-persons-data-${{ inputs.environment }} + group: related-persons-data-${{ inputs.environment || 'staging' }} cancel-in-progress: false jobs: build-and-ship: runs-on: ubuntu-latest # A from-scratch corpus crawl of the whole register is the long pole (all year-folders, politely - # throttled) and must fit in ONE job with extract/resolve/audit/ship after it — 120 was too tight and - # timed out mid-crawl. 300 leaves headroom under GitHub's 6-hour hard cap. A run past this is wedged + # throttled) and must fit in ONE job with the registry lookups, extract/resolve/audit/ship after it — + # 120 was too tight and timed out mid-crawl. The Търговски регистър pass adds ~20 minutes on top + # (~400 candidates at 1 request / 3 s). 300 leaves headroom under GitHub's 6-hour hard cap. A run past this is wedged # (stalled gov-server I/O) — fail rather than burn the slot. The raw corpus is cached (below) so a # re-run resumes instead of re-crawling from scratch. timeout-minutes: 300 - environment: ${{ inputs.environment }} + environment: ${{ inputs.environment || 'staging' }} env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} @@ -54,7 +101,7 @@ jobs: SIGMA_D1_NAME: ${{ vars.SIGMA_D1_NAME }} # The DECLARED intended environment — the independent anchor the ship guard checks the (name, id) pair # against (T48), so a consistent-but-wrong pair can't wipe the wrong DB. See assertD1TargetAuthorized. - SIGMA_SHIP_ENV: ${{ inputs.environment }} + SIGMA_SHIP_ENV: ${{ inputs.environment || 'staging' }} # Suppression (takedown) list keying — load.mjs fingerprints link-suppressions.jsonl under this salt and # refuses to build if the list is non-empty but the salt is unset (fail-closed, ADR-0031). Without this # the first suppression entry would break every ship; the key version guards a coordinated salt rotation. @@ -87,20 +134,25 @@ jobs: # live in each Environment's SIGMA_D1_NAME var; the ship step additionally requires SIGMA_SHIP_ENV to match # and PROVES the name↔SIGMA_D1_ID pair against Cloudflare before deleting — assertD1TargetAuthorized.) # The slot set is kept in lockstep with PRODUCTION_SLOTS in scripts/ship-related-persons.mjs (#226). + # $SIGMA_SHIP_ENV, never `${{ inputs.environment }}` inline. `type: choice` is enforced by the UI + # only — a POST to .../dispatches can pass an arbitrary string, which an inline expression pastes + # into the shell before `[` ever evaluates it. This block is also the one that decides whether the + # production guard fires, so a crafted value could both inject and walk past the guard it was + # meant to trip. The env binding above passes the same value as data, where it cannot be code. - name: Guard — SIGMA_D1_NAME matches the environment run: | case "${SIGMA_D1_NAME}" in sigma-blue|sigma-green) IS_PROD_SLOT=1 ;; *) IS_PROD_SLOT=0 ;; esac - if [ "${{ inputs.environment }}" = "production" ]; then + if [ "${SIGMA_SHIP_ENV}" = "production" ]; then if [ "$IS_PROD_SLOT" != "1" ]; then echo "::error::production requires SIGMA_D1_NAME to be a prod slot (sigma-blue|sigma-green), got '${SIGMA_D1_NAME}' — refusing to ship." exit 1 fi else if [ "$IS_PROD_SLOT" = "1" ]; then - echo "::error::SIGMA_D1_NAME='${SIGMA_D1_NAME}' (a PRODUCTION slot) on a '${{ inputs.environment }}' run — a rebuild would WIPE prod. Set a non-production SIGMA_D1_NAME variable for this Environment." + echo "::error::SIGMA_D1_NAME='${SIGMA_D1_NAME}' (a PRODUCTION slot) on a '${SIGMA_SHIP_ENV}' run — a rebuild would WIPE prod. Set a non-production SIGMA_D1_NAME variable for this Environment." exit 1 fi fi @@ -151,11 +203,29 @@ jobs: work="$GITHUB_WORKSPACE/data/work" mkdir -p "$work" rm -f "$work/eop.sql" "$work/backfill.sqlite" + # interest_links + interest_link_evidence come along too, and NOT for the resolver's benefit — + # load.mjs drops and rebuilds them. They are the input to the monotonicity gate: load.mjs + # exports the currently-published set BEFORE the wipe, and audit.mjs fails the run on a link + # that was published last time, under an unchanged rules_version, and is not published now. + # Without them the work DB starts with no interest_links at all, the pre-wipe export reads + # „no such table", the snapshot is [] on every run, and the gate protects nothing. + # They are absent on a first run (0003/0006 not yet applied), which is why they are exported + # separately and tolerated as missing rather than folded into the parity loop below. pnpm --filter @sigma/web exec wrangler d1 export "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote -y \ --table bidders --table contracts --table tenders --table authorities \ --output "$work/eop.sql" sqlite3 "$work/backfill.sqlite" < "$work/eop.sql" + if pnpm --filter @sigma/web exec wrangler d1 export "$SIGMA_D1_NAME" \ + --config wrangler.deploy.jsonc --remote -y \ + --table interest_links --table interest_link_evidence \ + --output "$work/prior-links.sql"; then + sqlite3 "$work/backfill.sqlite" < "$work/prior-links.sql" + echo " prior surface: $(sqlite3 "$work/backfill.sqlite" \ + "SELECT COUNT(*) FROM interest_links WHERE status='published';") published link(s)" + else + echo "::notice::No prior interest_links on this D1 (first run) — the monotonicity gate has no baseline yet." + fi for t in bidders contracts tenders authorities; do local_n=$(sqlite3 "$work/backfill.sqlite" "SELECT COUNT(*) FROM $t;") remote_json=$(pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ @@ -169,9 +239,60 @@ jobs: fi done + # ── Trade Register lookups ──────────────────────────────────────────────────────────────────── + # In THIS job, deliberately, and not in a workflow of their own. The decision run needs the raw + # deeds (evidenceVerdict matches declarant names against the register's own text, which is exactly + # the data the index refuses to store), and those deeds carry the names and addresses of co-owners + # and managers who hold no public office. Handing them between runners means persisting them — + # an Actions cache entry survives on GitHub's storage under a restore-keys chain, where the 35-day + # retention ADR-0033 decision 5 promises cannot reach it. Same runner, same job, deleted with it. + # + # The previous split also could not work on its own terms: the crawl job gated on a candidate list + # only the decision job produced, the decision job refused without a cache only the crawl job + # produced, and neither persisted its half. Both were permanently green no-ops. + - name: Emit the candidate ЕИК list for the crawler + run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/load.mjs --emit-candidates + + # ALWAYS, on every run that reaches here — not conditionally. The deeds do not survive the runner, + # so „decide without crawling" is not a state this job can be in: load.mjs's coverage gate below + # would refuse the empty cache and the run would fail. Skipping the crawl is only meaningful once + # the crawl emits verdicts rather than deeds (see the note on the schedule above). + # + # Pace and refusals live in scripts/tr/fetch-deeds.mjs and are not configurable here on purpose: + # 1 request / 3 s, sequential, a closed candidate set, and a 429 ends the run without marking + # anything. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the limiter is + # the operator's only way to state a rate preference, so we do not tune around it. + - name: Refresh the Trade Register deed cache + env: + MAX_AGE: ${{ inputs.tr_max_age_days || '30' }} + LIMIT: ${{ inputs.tr_limit }} + run: | + set -euo pipefail + ARGS=(--eiks-file scratch/cacbg/staging/candidate-eiks.txt --max-age-days "$MAX_AGE") + [ -n "$LIMIT" ] && ARGS+=(--limit "$LIMIT") + set +e + node scripts/tr/fetch-deeds.mjs "${ARGS[@]}" + code=$? + set -e + # Exit 2 is the rate limiter, and it is NOT a build failure: the run stopped politely, marked + # nothing, and the partial cache is valid. It is not a licence to publish either — load.mjs's + # coverage gate below refuses a partial cache on its own, which is where that decision belongs. + if [ "$code" -eq 2 ]; then + echo "::warning::The register rate-limited us; the crawl stopped and the cache is resumable." + exit 0 + fi + exit "$code" + - name: Resolve → build domain in a work sqlite (libel gate must pass) run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/load.mjs + # The raw deeds are third-party personal data and exist only to produce booleans. They are deleted + # with the runner regardless; deleting them explicitly means a failure AFTER this point cannot + # leave them sitting in a job artifact or a debug upload someone adds later. + - name: Delete the raw deeds + if: always() + run: rm -rf scratch/tr/deeds + - name: Audit published links (fails the run on any libel-invariant violation) run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/audit.mjs @@ -185,6 +306,20 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0003_related_persons_foundation.sql + # 0006 attaches the Trade Register evidence seal (#279, ADR-0033). Both migrations are + # applied by name here because wrangler's migration ledger is empty on this D1 (the base + # schema was created out-of-band), so `d1 migrations apply` would collide on 0000. + pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0009_interest_link_evidence.sql + # 0007 retrofits the publishing-gate CHECK constraints and control_hash NOT NULL onto an + # ALREADY-PROVISIONED D1 (#279 §2). 0003 declares them now, but `CREATE TABLE IF NOT EXISTS` + # never revisits an existing table and the ship step wipes ROWS, not definitions — so without + # this a live database keeps the unconstrained shape no matter how often the data reloads. + # Rebuild-based and therefore idempotent: re-applying copies an already-constrained table. + pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0010_publishing_gate_constraints.sql - name: Ship свързани-лица tables → D1 # ship-related-persons.mjs shells out to `wrangler d1 execute` with cwd=apps/web. Two things it @@ -198,7 +333,14 @@ jobs: node scripts/wrangler-render.mjs apps/web/wrangler.jsonc cp apps/web/wrangler.deploy.jsonc apps/web/wrangler.jsonc export PATH="$GITHUB_WORKSPACE/apps/web/node_modules/.bin:$GITHUB_WORKSPACE/node_modules/.bin:$PATH" - node scripts/ship-related-persons.mjs --work-db data/work/backfill.sqlite --remote --yes + # --min-links is passed EXPLICITLY, not left at its default of 50. Under #279 the surface is + # measured at ~329 links, so 50 no longer protects anything: a partially-restored Trade + # Register cache yields roughly 80 published links, which clears a floor of 50, ships a + # decimated surface and wipes the rest. The loader's coverage gate is the first defence and + # this is the second (ADR-0033 decision 7). Lower it deliberately, in a PR, if a genuinely + # smaller surface is ever expected. + node scripts/ship-related-persons.mjs --work-db data/work/backfill.sqlite --remote --yes \ + --min-links 250 - name: Reindex search projection → officials searchable (same batch the cron runs) # ship writes ONLY the свързани-лица domain tables (interest_links, persons, …), NOT search_index. diff --git a/.github/workflows/scripts-test.yml b/.github/workflows/scripts-test.yml index b4dd794fd..61deb9f7a 100644 --- a/.github/workflows/scripts-test.yml +++ b/.github/workflows/scripts-test.yml @@ -30,10 +30,11 @@ jobs: # ubuntu-latest image, and the test fails loudly (never skips) if it is ever absent. - run: node --test scripts/*.test.mjs - # свързани-лица CACBG pipeline tests (parse/classify/load/audit/tr-census/extract). These use - # node:sqlite (DatabaseSync) and import the shared companyNameKey .ts via the register-ts resolve - # hook, so they need Node 24 (node:sqlite + native TS type-stripping, no experimental flag) rather - # than the Node 22 above. The libel-critical resolution logic gates merges here. + # свързани-лица pipeline tests — the CACBG leg (parse/classify/load/audit/extract) and the + # Търговски регистър leg (scripts/tr: ЕИК checksum, HTTP client, deed cache). These use node:sqlite + # (DatabaseSync) and import the shared companyNameKey .ts via the register-ts resolve hook, so they + # need Node 24 (node:sqlite + native TS type-stripping, no experimental flag) rather than the Node 22 + # above. The libel-critical resolution logic gates merges here. cacbg: runs-on: ubuntu-latest timeout-minutes: 10 @@ -49,4 +50,6 @@ jobs: # parse.mjs imports fast-xml-parser (a workspace dep), so this job needs node_modules — unlike # the plain-node scripts above. Install before the scraper tests run. - run: pnpm install --frozen-lockfile - - run: node --import ./scripts/cacbg/register-ts.mjs --test scripts/cacbg/*.test.mjs + # Both legs, or the glob silently excludes a whole module: scripts/tr/*.test.mjs matched NO job + # until this line existed, so every test in it would have been decorative. + - run: node --import ./scripts/cacbg/register-ts.mjs --test scripts/cacbg/*.test.mjs scripts/tr/*.test.mjs diff --git a/apps/web/app/components/ConflictCards.tsx b/apps/web/app/components/ConflictCards.tsx index 4bbfeba41..a716c425e 100644 --- a/apps/web/app/components/ConflictCards.tsx +++ b/apps/web/app/components/ConflictCards.tsx @@ -20,6 +20,7 @@ import { officialHref, partitionContracts, relationLabel, + registryEvidenceLabel, temporalLabel, } from '../lib/conflicts'; @@ -181,6 +182,26 @@ function ConflictCard({ )} + {/* The Trade Register fact the link's identity rests on (#279, ADR-0033). This is what makes + „every shown link explains itself" true rather than a promise: a reader can open the same act + we read and check it. The wording is careful — the register records a ROLE, it does not + certify the ownership claim, which comes from the official's own declaration. */} +
+
Регистър
+
+ + + {registryEvidenceLabel(l)} + {l.registryEntryDate ? ` · вписване ${l.registryEntryDate}` : ''} + {/* The entry NUMBER, not just its date: a date does not identify a record, and this is + what a reader types to find the same act we read. Rendered only when present — + a seat/ЕИК confirmation cites no act entry, and an empty „№" would read as missing + data rather than as an inapplicable field. */} + {l.registryEntryNumber ? ` · № ${l.registryEntryNumber}` : ''} + {l.registryLookupDate ? ` · справка ${l.registryLookupDate}` : ''} + +
+
{l.contractCount > 0 && ( diff --git a/apps/web/app/lib/conflicts.test.ts b/apps/web/app/lib/conflicts.test.ts index a4bfd5f8b..6cec490ac 100644 --- a/apps/web/app/lib/conflicts.test.ts +++ b/apps/web/app/lib/conflicts.test.ts @@ -21,6 +21,7 @@ import { partitionContracts, relationLabel, temporalLabel, + registryEvidenceLabel, } from './conflicts'; function link(over: Partial = {}): ConflictLink { @@ -43,7 +44,13 @@ function link(over: Partial = {}): ConflictLink { contemporaneousValueEur: 40_000_000, firstContractYear: '2021', lastContractYear: '2024', - sourceUrl: 'https://register.cacbg.bg/2024/i.xml', + sourceUrl: 'https://register.cacbg.bg/2024/x.xml', + // #279: a link only reaches the DTO when its identity rests on a Trade Register fact. + evidenceKind: 'document', + registryRole: 'owner', + registryEntryNumber: '20110502101007', + registryEntryDate: '2011-05-02', + registryLookupDate: '2026-08-05', ...over, }; } @@ -523,3 +530,31 @@ describe('authorityShareDisplay', () => { }); }); }); + +describe('registryEvidenceLabel', () => { + // The wording is load-bearing. The register records a ROLE; it does not certify that the official owns + // anything — that claim comes from their own declaration and is rendered separately. A label that said + // „собственик според ТР" would assert something the evidence does not support (ADR-0033 decision 2). + it('reports what the act records, never an ownership conclusion', () => { + expect(registryEvidenceLabel({ evidenceKind: 'document', registryRole: 'owner' })).toBe( + 'лицето е вписано като съдружник/собственик', + ); + expect(registryEvidenceLabel({ evidenceKind: 'document', registryRole: 'manager' })).toBe( + 'лицето е вписано като управител', + ); + }); + + it('a seat/ЕИК confirmation claims identity, not a registry role', () => { + // „Потвърдено" means the COMPANY was identified from something the official declared — nobody was + // found in the act, so the label must not imply anyone was. + const label = registryEvidenceLabel({ evidenceKind: 'confirmed', registryRole: null }); + expect(label).toBe('самоличност, потвърдена по декларирани данни'); + expect(label).not.toMatch(/вписан/); + }); + + it('never renders the word „собственик" for a mere confirmation', () => { + expect(registryEvidenceLabel({ evidenceKind: 'confirmed', registryRole: 'owner' })).not.toMatch( + /собственик/, + ); + }); +}); diff --git a/apps/web/app/lib/conflicts.ts b/apps/web/app/lib/conflicts.ts index 50a6f16ba..7e0908c16 100644 --- a/apps/web/app/lib/conflicts.ts +++ b/apps/web/app/lib/conflicts.ts @@ -24,10 +24,45 @@ const RELATION_LABEL: Record = { }; /** Bulgarian label for a declared relation. Unknown values pass through — never invent a stronger claim. */ +/** + * How the company's identity was established, in the register's own terms (#279, ADR-0033). + * + * Deliberately does NOT say the official owns anything: „вписан съдружник/собственик" reports what the + * act RECORDS, while the ownership claim itself comes from the official's own declaration and is + * rendered separately as „дялово участие". „Потвърдено" means the company was identified by a fact the + * official declared — the seat or the ЕИК — not that anybody was found in the act. + */ +export function registryEvidenceLabel(l: { + evidenceKind: 'document' | 'confirmed'; + registryRole: 'owner' | 'manager' | null; +}): string { + if (l.evidenceKind === 'confirmed') return 'самоличност, потвърдена по декларирани данни'; + return l.registryRole === 'manager' + ? 'лицето е вписано като управител' + : 'лицето е вписано като съдружник/собственик'; +} + export function relationLabel(relation: string): string { return RELATION_LABEL[relation] ?? relation; } +/** + * How to describe a PAGE's set of links in prose (#279 §2.6). The card labels above are already + * family-aware; the surrounding page copy was not, and asserted „собствен дял" — an OWN stake — above + * cards that correctly read „свързано лице". On a family-only page that is a false claim about the named + * official, and it is the second source of truth the card-label fix set out to remove. + * + * Derived from the links themselves rather than passed in, so a page cannot describe a set it isn't + * rendering. Mixed sets get the neutral wording: it is the only phrasing true of every card. + */ +export function declaredStakeNoun(links: { relation: string }[]): string { + const anyFamily = links.some((l) => l.relation === 'related'); + const anySelf = links.some((l) => l.relation !== 'related'); + if (anyFamily && !anySelf) return 'дял на свързано лице'; + if (anyFamily && anySelf) return 'деклариран дял — собствен или на свързано лице'; + return 'собствен дял'; +} + // Defense in depth: the slug is base64url and the ЕИК numeric today (so encoding is a no-op), but if either // assumption ever drifts, an un-escaped `/`, `?` or `#` would break routing and the cache key. Escape the // dynamic segments unconditionally (ydimitrof #226, conflicts.ts). diff --git a/apps/web/app/routes/conflict.company.tsx b/apps/web/app/routes/conflict.company.tsx index 9dd285950..5de8e12e6 100644 --- a/apps/web/app/routes/conflict.company.tsx +++ b/apps/web/app/routes/conflict.company.tsx @@ -9,7 +9,7 @@ import { ConflictCards } from '../components/ConflictCards'; import { publicCache } from '../lib/cache'; import { withDbRetry } from '../lib/retry'; import { seoMeta } from '../lib/meta'; -import { companyProfileHref } from '../lib/conflicts'; +import { companyProfileHref, declaredStakeNoun } from '../lib/conflicts'; // Officials with a published declared interest in one winner (by ЕИК). Reads interest_links only. 404 when // no official has a published link to this company — never an empty page under a company's name. @@ -59,7 +59,7 @@ export default function ConflictCompany({ loaderData }: Route.ComponentProps) { } title={company} - lede={`Длъжностни лица, декларирали собствен дял в това дружество пред КПКОНПИ. ${count(links.length)} ${plural(links.length, 'връзка', 'връзки')} — всяка е точно съвпадение по фирмено име.`} + lede={`Длъжностни лица, декларирали ${declaredStakeNoun(links)} в това дружество пред КПКОНПИ. ${count(links.length)} ${plural(links.length, 'връзка', 'връзки')} — всяка почива на проверим факт от Търговския регистър.`} /> diff --git a/apps/web/app/routes/conflict.methodology.tsx b/apps/web/app/routes/conflict.methodology.tsx index 32936a301..d7b1f9d4c 100644 --- a/apps/web/app/routes/conflict.methodology.tsx +++ b/apps/web/app/routes/conflict.methodology.tsx @@ -182,6 +182,76 @@ export default function ConflictMethodology() { връзката е двусмислена и не се публикува — никога не отгатваме кой от субектите е имал предвид деклараторът.

+ +

Проверка в Търговския регистър

+

+ Съвпадението по име вече не е достатъчно, за да се публикува + връзка. За всяко дружество правим отделна справка в{' '} + Търговския регистър (публична, без регистрация) и връзката се + показва само ако самоличността на дружеството е потвърдена от{' '} + проверим факт в акта. Редът е следният — прилага се първото, което + е изпълнено: +

+
    +
  1. + Акционерна форма (АД, ЕАД, КДА) — не се показва никога. Книгата + на акционерите не е публична, тоест твърдението е непроверимо; освен това дялът + може да е миноритарен или борсов, тоест несъществен. +
  2. +
  3. + Документ. И трите имена на лицето се срещат в живо поле на акта — + управители, съдружници, едноличен собственик или физическо лице-търговец. Пазим + ролята (съдружник/собственик или само управител), защото тя не е едно и също.{' '} + + Съвпадението по име само по себе си не стига: изисква се и нещо извън фирменото + наименование да установи, че дружеството е декларираното + {' '} + — деклариран ЕИК, декларирано седалище, съвпадащо с вписаното, или достатъчно + отличително фирмено наименование. Иначе връзката остава скрита. Причината е, че + дружеството се разпознава сред изпълнителите по обществени поръчки: ако + лицето притежава едноименно дружество, което никога не е кандидатствало, търсенето + би посочило чуждото — и съименник в неговия акт би „потвърдил" връзка, невярна и в + двете си части. +
  4. +
  5. + Потвърдено. Лицето не е намерено в акта, но декларирано от него + седалище съвпада с регистрираното за този ЕИК, или самият + декларатор е изписал ЕИК. Тук се потвърждава{' '} + кое е дружеството, а не кой го притежава. +
  6. +
  7. + Оборена. Само за собствен дял: лицето не фигурира никъде в акта, + а най-късното вписване за собственост е преди декларирания период + — тоест записът покрива целия период и не назовава лицето. Връзката отпада. +
  8. +
  9. + Неизвестна — всичко останало. Остава скрита. +
  10. +
+

+ Какво доказва това и какво не. Регистърът потвърждава{' '} + самоличността на дружеството; той не удостоверява, че лицето притежава дял + — това твърдение идва от неговата собствена декларация. Затова на картата пише + „лицето е вписано като съдружник/собственик", а не „лицето е собственик според ТР". +

+

+ Ограниченията, честно. Търговският регистър{' '} + не съдържа ЕГН, затова съвпадението по трите имена не е абсолютно + доказателство за самоличност — възможен е съименник. Затова се + изисква пълно съвпадение и на трите имена в{' '} + един и същ запис на акта (две от три не се приема; имена с инициали + или с латински букви не се приемат). Седалищата се променят, затова регистрираното + седалище се приема само ако е вписано преди декларирания период —{' '} + и само когато този период е известен. Ако от декларациите не може + да се извлече година, проверката няма с какво да се направи и връзката остава + скрита. Заличените записи се пропускат навсякъде. +

+

+ Справките се опресняват месечно, а датата на справката и номерът на + вписването се показват на всяка връзка — така всяко твърдение носи своя източник и + своята давност. Актовете се пазят само служебно, за срок до 35 дни; имена на трети + лица от тях не се съхраняват и не влизат в никакъв публичен запис. +

@@ -267,7 +337,11 @@ export default function ConflictMethodology() { Прекратен дял. Ако лице подаде по-късна декларация за собственост, която вече не изброява дадено дружество, връзката се смята за приключила и се оттегля от публичната част — така остарял запис - не внушава конфликт, който е приключил. + не внушава конфликт, който е приключил. Това обаче е извод от мълчание: + най-честата причина да спреш да декларираш е изтекъл мандат, а не продажба. Затова + преди оттеглянето сверяваме с живия акт в Търговския регистър — + ако лицето и днес е вписано като съдружник или собственик, дялът не е прекратен и + връзката остава.

@@ -363,11 +437,13 @@ export default function ConflictMethodology() {

Разглеждаме сигнала в срок до 7 работни дни. При основателен сигнал - връзката се премахва незабавно и се записва в списък за премахване, - който оцелява при всяко следващо обновяване — веднъж премахната - връзка не се възстановява при повторно зареждане на данните. Премахваме връзка при - доказана неточност; не заличаваме коректни записи по искане на изпълнител или - възложител — данните идват от публични източници и остават публични. + връзката се премахва веднага след решението и се записва в списък + за премахване, който оцелява при всяко следващо обновяване — веднъж + премахната връзка не се възстановява при повторно зареждане на данните. Премахването + се прилага незабавно в базата; страницата може да се вижда в кеш{' '} + до един час след това. Премахваме връзка при доказана неточност; не + заличаваме коректни записи по искане на изпълнител или възложител — данните идват от + публични източници и остават публични.

diff --git a/apps/web/app/routes/conflict.official.tsx b/apps/web/app/routes/conflict.official.tsx index 8ef918790..01710bf5f 100644 --- a/apps/web/app/routes/conflict.official.tsx +++ b/apps/web/app/routes/conflict.official.tsx @@ -8,6 +8,7 @@ import { ConflictCards } from '../components/ConflictCards'; import { publicCache } from '../lib/cache'; import { withDbRetry } from '../lib/retry'; import { seoMeta } from '../lib/meta'; +import { declaredStakeNoun } from '../lib/conflicts'; // One office-holder's declared ownership links. Reads private-ownership interest_links only. 404 (not an // empty page) when the person has no published link — a bare page under someone's name reads as an @@ -39,6 +40,9 @@ export async function loader({ params, context }: Route.LoaderArgs) { export default function ConflictOfficial({ loaderData }: Route.ComponentProps) { const { official, links } = loaderData; + // Family-AWARE, not family-blind: the page must not assert an own stake above cards that say „свързано + // лице", and must not go vague where the stake really is the official's own (§2.6). + const stake = declaredStakeNoun(links); return ( <>

Данните са от собствените декларации на лицето (публичен регистър на КПКОНПИ), - съпоставени точно с регистъра на изпълнителите. Показваме само 100% съвпадения и само - собствен деклариран дял. Сигнал за неточност:{' '} - Методология → Поправки. + съпоставени точно с регистъра на изпълнителите. Показваме само 100% съвпадения, и само + когато самоличността на дружеството е потвърдена от Търговския регистър. Сигнал за + неточност: Методология → Поправки.

= {}): ConflictLink { contemporaneousValueEur: 30_000_000, firstContractYear: '2020', lastContractYear: '2024', - sourceUrl: 'https://register.cacbg.bg/2024/i.xml', + sourceUrl: 'https://register.cacbg.bg/2024/x.xml', + // #279: a link only reaches the DTO when its identity rests on a Trade Register fact. + evidenceKind: 'document', + registryRole: 'owner', + registryEntryNumber: '20110502101007', + registryEntryDate: '2011-05-02', + registryLookupDate: '2026-08-05', ...over, }; } @@ -94,6 +100,29 @@ describe('/conflicts/official/:id — render', () => { expect(card.textContent).not.toContain('Кмет Тестов'); }); + it('a family-only page never asserts the official owns the stake (§2.6)', async () => { + // The CARD labels were made tense-neutral and family-aware, but the page lede and the section hint + // still read „декларирало собствен дял" — a second source of truth on the very page that renders a + // relative's stake. On a family-only page that is a false claim about the named official, printed + // above a card that correctly says „свързано лице". + await mount(ConflictOfficial as never, { + official: 'Кмет Тестов', + links: [link({ relation: 'related', company: 'ЕВРОСТРОЙ 21 ЕООД', eik: '333' })], + }); + expect(text()).not.toContain('собствен дял'); + expect(text()).toContain('деклариран дял на свързано лице'); + }); + + it('an own-stake page still says so — the wording is family-AWARE, not family-blind', async () => { + // POSITIVE CONTROL. Removing the claim everywhere would satisfy the assertion above while making the + // page vaguer than the data warrants: a self stake IS the official's own and should read that way. + await mount(ConflictOfficial as never, { + official: 'Кмет Тестов', + links: [link({ relation: 'owns', company: 'ЕВРОСТРОЙ 21 ЕООД', eik: '333' })], + }); + expect(text()).toContain('собствен дял'); + }); + it('meta() names the person in the title and marks the page noindex', () => { const tags = officialMeta({ data: { official: 'Иван Петров', links: [] }, @@ -133,6 +162,35 @@ describe('/conflicts/company/:eik — render', () => { }); describe('/conflicts/methodology — render', () => { + it('discloses the matching rule verbatim — every rung, and what each may conclude', async () => { + // ADR-0021 E10 makes this page the disclosure of the rule, and ADR-0033 decision 7 makes it a LAUNCH + // CONDITION rather than a follow-up: a heuristic that asserts something about a named person is only + // defensible if the reader can see exactly what was asserted and why. Nothing but a test keeps the + // page in step with the ladder — the rule can change in evidence.mjs and leave the page describing a + // system that no longer exists, which is worse than not disclosing it at all. + await mount(ConflictMethodology as never, {}); + const t = text(); + // every rung of the ladder, by the name the seal and the card use + for (const rung of ['Документ', 'Потвърдено', 'Оборена', 'Неизвестна']) + expect(t).toContain(rung); + // rung 1 — the joint-stock bar and its reason (the „11 акции" trap) + expect(t).toContain('Акционерна форма'); + expect(t).toContain('не е публична'); + // rung 2 — all three names, one record, and the two refusals + expect(t).toContain('пълно съвпадение и на трите имена'); + expect(t).toContain('един и същ запис'); + // ADR-0035 — the company gate, the part a reader most needs to judge the claim + expect(t).toContain('Съвпадението по име само по себе си не стига'); + // R10 — the seat's temporal guard, both halves + expect(t).toContain('вписано преди декларирания период'); + expect(t).toContain('когато този период е известен'); + // the honest limit: no ЕГН, so a homonym is possible + expect(t).toContain('не съдържа ЕГН'); + expect(t).toContain('съименник'); + // what the register proves and what it does not — the distinction the whole surface rests on + expect(t).toContain('самоличността на дружеството'); + }); + it('states the three libel rails in plain language', async () => { await mount(ConflictMethodology as never, {}); const t = text(); diff --git a/apps/web/app/routes/conflicts.render.test.tsx b/apps/web/app/routes/conflicts.render.test.tsx index 68dad5fbb..814c36708 100644 --- a/apps/web/app/routes/conflicts.render.test.tsx +++ b/apps/web/app/routes/conflicts.render.test.tsx @@ -36,6 +36,12 @@ function link(over: Partial = {}): ConflictLink { firstContractYear: '2020', lastContractYear: '2024', sourceUrl: 'https://register.cacbg.bg/2024/i.xml', + // #279: a link only reaches the DTO when its identity rests on a Trade Register fact. + evidenceKind: 'document', + registryRole: 'owner', + registryEntryNumber: '20110502101007', + registryEntryDate: '2011-05-02', + registryLookupDate: '2026-08-05', ...over, }; } @@ -47,6 +53,13 @@ const familyLink = link({ company: 'ЕВРОСТРОЙ 21 ЕООД', eik: '333', relation: 'related', // a close relative's stake — anonymized + // The shape the QUERY actually yields for a family link, not the base factory's self-link default. + // `findPerson` searches the act for the OFFICIAL; a relative's stake is registered to the relative, + // so the official is never found and no `document` rung is reachable — only a seat/ЕИК confirmation. + // A fixture carrying `document`/`owner` here tested a row that cannot exist and hid the assertion + // below, which is the one that matters: no registry-role claim on an anonymized card. + evidenceKind: 'confirmed' as const, + registryRole: null, ownInstitution: false, contractCount: 1, contractValueEur: 250_000, @@ -188,8 +201,16 @@ describe('/conflicts route — render', () => { (a) => a.textContent === 'декларация', ); expect(sourceAnchor?.getAttribute('href')).toBe('https://register.cacbg.bg/2024/i.xml'); - // the zero-contract link has no toggle (contractCount === 0) and a muted „—" source - expect(text()).toContain('—'); + // Scoped to the card that actually has sourceUrl: null. A page-wide toContain('—') passes on any + // em-dash anywhere — including the ones the value and date cells render — so it would survive the + // source branch being deleted outright. + const cards = container.querySelectorAll('.conflict-card'); + const noSourceCard = [...cards].find((c) => c.textContent?.includes('ПРАЗЕН ООД'))!; + expect(noSourceCard).toBeTruthy(); + expect( + [...noSourceCard.querySelectorAll('a')].some((a) => a.textContent === 'декларация'), + ).toBe(false); + expect(noSourceCard.textContent).toContain('—'); }); it('a zero-contract link renders no „Виж договорите" toggle', async () => { @@ -238,3 +259,57 @@ describe('/conflicts route — render', () => { ).not.toBeNull(); }); }); + +describe('Trade Register evidence on the card (#279, ADR-0033)', () => { + it('renders the registry fact the link rests on, so the card explains itself', async () => { + await renderConflicts([link({ evidenceKind: 'document', registryRole: 'owner' })]); + const text = container.textContent ?? ''; + expect(text).toContain('Регистър'); + expect(text).toContain('лицето е вписано като съдружник/собственик'); + expect(text).toContain('вписване 2011-05-02'); // WHICH entry + expect(text).toContain('справка 2026-08-05'); // and HOW FRESH it is + // The entry NUMBER is what makes the claim findable in the register — a date alone does not + // identify a record. It was carried to every client in the DTO and never rendered, which is the + // one payload that costs bytes and answers nothing (cefothe, #309). + expect(text).toContain('20110502101007'); + }); + + it('omits the entry number rather than printing an empty label when there is none', async () => { + // POSITIVE CONTROL for the row's shape: a confirmed link (seat/ЕИК) has no act entry to cite, so + // the label must be absent entirely — not „· №" with nothing after it, which reads as missing data + // rather than as an inapplicable field. Scoped to the evidence label: „№" alone is the card RANK. + await renderConflicts([ + link({ evidenceKind: 'confirmed', registryRole: null, registryEntryNumber: null }), + ]); + const text = container.textContent ?? ''; + expect(text).toContain('Регистър'); + expect(text).not.toContain('· №'); + }); + + it('a seat/ЕИК confirmation never implies somebody was found in the act', async () => { + await renderConflicts([link({ evidenceKind: 'confirmed', registryRole: null })]); + const text = container.textContent ?? ''; + expect(text).toContain('самоличност, потвърдена по декларирани данни'); + expect(text).not.toContain('вписано като'); + }); + + it('a FAMILY card never carries a registry-role claim — the relative is not in the act we read', async () => { + // The production shape for a family link, which the fixture used to contradict: `findPerson` looks + // for the OFFICIAL, and a relative's stake is registered to the relative, so the official is never + // found and the rung can only ever be `confirmed`/`registryRole: null`. A card that said „лицето е + // вписано като съдружник/собственик" here would assert that the named official is recorded in the + // register as an owner of this company — a false, named, libel-shaped claim, on the one card whose + // whole design is that the stakeholder stays anonymous (ADR-0030/0032). + await renderConflicts([familyLink]); + const familyCard = container.querySelector('.conflict-card')!; + expect(familyCard.textContent).toContain('деклариран дял на свързано лице'); + expect(familyCard.textContent).toContain('самоличност, потвърдена по декларирани данни'); + expect(familyCard.textContent).not.toContain('вписано като'); + }); + + it('links out to the register so a reader can check the same act we read', async () => { + await renderConflicts([link({ eik: '201122335' })]); + const hrefs = [...container.querySelectorAll('a')].map((a) => a.getAttribute('href') ?? ''); + expect(hrefs.some((h) => h.includes('201122335'))).toBe(true); + }); +}); diff --git a/coverage-baseline.json b/coverage-baseline.json index 5abe85c95..064f448fe 100644 --- a/coverage-baseline.json +++ b/coverage-baseline.json @@ -6,24 +6,24 @@ "branches": 58.2 }, "apps/web": { - "lines": 89.7, - "branches": 81.8 + "lines": 91, + "branches": 82.4 }, "packages/config": { "lines": 92.8, "branches": 72.2 }, "packages/db": { - "lines": 94.2, - "branches": 79 + "lines": 94.5, + "branches": 79.3 }, "packages/ingest": { - "lines": 85.8, - "branches": 80 + "lines": 86.3, + "branches": 80.4 }, "packages/shared": { - "lines": 95.4, - "branches": 80 + "lines": 95.5, + "branches": 80.8 } } } diff --git a/docs/adr/0007-scope-and-certainty-bar.md b/docs/adr/0007-scope-and-certainty-bar.md index 18fcb4462..c41f42554 100644 --- a/docs/adr/0007-scope-and-certainty-bar.md +++ b/docs/adr/0007-scope-and-certainty-bar.md @@ -1,6 +1,7 @@ # ADR-0007: Scope and certainty bar - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — решения 2 и 3 (изричното изключение за „евристика, която твърди") - Date: 2026-07-05 - Deciders: lb (Head of AI, ИО), Claude - Related: spec §1–2, §8 diff --git a/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md b/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md index 7803e901a..8783b3ac3 100644 --- a/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md +++ b/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md @@ -1,8 +1,9 @@ # ADR-0009: Name-uniqueness is not absolute → single-ЕИК guard + publish tiers -- Status: Accepted +- Status: Superseded by [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Date: 2026-07-05 - Deciders: lb, Claude +- Superseded by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Related: spec §5; refines [ADR-0008](0008-deterministic-name-to-eik-resolution.md) ## Context diff --git a/docs/adr/0010-pii-posture.md b/docs/adr/0010-pii-posture.md index 8f09ead85..cf9898a04 100644 --- a/docs/adr/0010-pii-posture.md +++ b/docs/adr/0010-pii-posture.md @@ -1,6 +1,7 @@ # ADR-0010: PII posture - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — решения 3 и 6 (кешът от актове и срокът на съхранение) - Date: 2026-07-05 - Deciders: lb, Claude - Related: spec §8; КЗЛД guidance; GDPR Art. 6(1)(c)/(f), Art. 85 diff --git a/docs/adr/0015-tr-name-uniqueness-census.md b/docs/adr/0015-tr-name-uniqueness-census.md index b3fb27891..c86d8eece 100644 --- a/docs/adr/0015-tr-name-uniqueness-census.md +++ b/docs/adr/0015-tr-name-uniqueness-census.md @@ -1,8 +1,9 @@ # ADR-0015: TR name-uniqueness census (promoting tier-C generic-name matches) -- Status: Accepted (design; implemented as a Phase-1 pipeline step) +- Status: Superseded by [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Date: 2026-07-05 - Deciders: lb, Claude +- Superseded by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Related: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md); spec §5 ## Context diff --git a/docs/adr/0017-name-collision-tier-gate.md b/docs/adr/0017-name-collision-tier-gate.md index 8d4a234be..38ed31b52 100644 --- a/docs/adr/0017-name-collision-tier-gate.md +++ b/docs/adr/0017-name-collision-tier-gate.md @@ -1,8 +1,9 @@ # ADR-0017: A globally non-unique name cannot ride the name-distinctive tier — even with a certain ЕИК -- Status: Accepted +- Status: Superseded by [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Date: 2026-07-05 - Deciders: lb, Claude +- Superseded by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — the name-distinctive tier it gates leaves the publishing path; the `nameGloballyUnique` precondition survives, narrowed to the weakest evidence rung - Related: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md), [ADR-0016](0016-free-text-entity-resolution.md); `scripts/cacbg/load.mjs`, `scripts/cacbg/audit.mjs` ## Context diff --git a/docs/adr/0021-methodology-page-and-temporal-freshness.md b/docs/adr/0021-methodology-page-and-temporal-freshness.md index 810a74b21..d2f0d9cd2 100644 --- a/docs/adr/0021-methodology-page-and-temporal-freshness.md +++ b/docs/adr/0021-methodology-page-and-temporal-freshness.md @@ -1,6 +1,7 @@ # ADR-0021: Public methodology/corrections page (E10) + temporal freshness & divestment expiry (E11) - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — E11 (етикетите за настояще време се отлагат за фаза 2) - Date: 2026-07-06 - Deciders: lb, Claude - Related: [ADR-0007](0007-scope-and-certainty-bar.md), [ADR-0019](0019-private-interest-vs-ex-officio-classification.md), [ADR-0020](0020-conflict-explorer-surface-posture.md); spec §5/§8/§9; `scripts/cacbg/load.mjs`, `apps/web/app/routes/conflict.methodology.tsx`, `packages/db/src/queries/related-persons.ts` diff --git a/docs/adr/0023-anonymized-family-ownership-surface.md b/docs/adr/0023-anonymized-family-ownership-surface.md index 3ead38243..d5dcea50b 100644 --- a/docs/adr/0023-anonymized-family-ownership-surface.md +++ b/docs/adr/0023-anonymized-family-ownership-surface.md @@ -1,8 +1,9 @@ # ADR-0023: Anonymized close-relative (family) ownership surface -- Status: Accepted +- Status: Superseded by [ADR-0030](0030-family-ownership-withheld-nameless-aggregate.md) - Date: 2026-07-07 - Deciders: lb, Claude +- Superseded by: [ADR-0030](0030-family-ownership-withheld-nameless-aggregate.md) - Related: [ADR-0007](0007-scope-and-certainty-bar.md), [ADR-0010](0010-pii-posture.md), [ADR-0019](0019-private-interest-vs-ex-officio-classification.md), [ADR-0022](0022-public-surface-private-ownership-only.md); `scripts/cacbg/parse.mjs`, `scripts/cacbg/load.mjs`, `packages/db/src/queries/related-persons.ts`, `apps/web/app/routes/conflict*.tsx` ## Context diff --git a/docs/adr/0028-declared-eik-is-a-determining-identifier.md b/docs/adr/0028-declared-eik-is-a-determining-identifier.md index 9f123482e..9ad76597b 100644 --- a/docs/adr/0028-declared-eik-is-a-determining-identifier.md +++ b/docs/adr/0028-declared-eik-is-a-determining-identifier.md @@ -1,6 +1,7 @@ # ADR-0028: Declared ЕИК is a determining identifier — tier `A_eik`, exempt from the TR census - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — декларираният ЕИК остава определящ, но вече иска и регистърно доказателство - Date: 2026-07-11 - Deciders: lb, Claude - Related: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md), [ADR-0015](0015-tr-name-uniqueness-census.md), [ADR-0016](0016-free-text-entity-resolution.md); `scripts/cacbg/load.mjs`, `scripts/cacbg/classify.mjs` diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md new file mode 100644 index 000000000..82633fb76 --- /dev/null +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -0,0 +1,364 @@ +# ADR-0033: A link publishes only against a Trade Register fact — the evidence ladder replaces the name-distinctiveness tier + +- Status: Accepted (design; implemented as the #279 pipeline change) +- Date: 2026-08-05 +- Deciders: Todor (maintainer), Claude +- Supersedes: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md), [ADR-0015](0015-tr-name-uniqueness-census.md), [ADR-0017](0017-name-collision-tier-gate.md) +- Amends: [ADR-0007](0007-scope-and-certainty-bar.md) (decisions 2 and 3), [ADR-0010](0010-pii-posture.md) (decisions 3 and 6), [ADR-0021](0021-methodology-page-and-temporal-freshness.md) (E11 only — E10 stands and is strengthened), [ADR-0028](0028-declared-eik-is-a-determining-identifier.md) +- Related: [ADR-0008](0008-deterministic-name-to-eik-resolution.md), [ADR-0011](0011-host-scoped-tls-pinning.md), [ADR-0026](0026-person-grain-name-institution.md), [ADR-0027](0027-overmerge-gate-is-telemetry-not-a-gate.md), [ADR-0031](0031-suppressions-version-controlled-fingerprinted.md), [ADR-0032](0032-family-ownership-published-under-public-interest.md); spec §3.3/§5/§8/§9, [related-persons-lia.md](../spec/related-persons-lia.md); `scripts/tr/`, `scripts/cacbg/{load,classify}.mjs`, `packages/db/src/queries/related-persons.ts`, `apps/web/app/routes/conflict*.tsx` + +## Context + +Today the identity of the company behind a declared name is decided by a heuristic. `B_distinctive` +(ADR-0009) publishes a link when the name is *structurally* distinctive — a digit, a Latin token, or ≥3 +content words. It fails in both directions: it publishes on a bare name coincidence, and it withholds +everything else. Of the 101 links published today, **36 rest on name coincidence alone**. + +The Trade Register answers the identity question directly, over a public endpoint with no authentication: +`GET https://portal.registryagency.bg/CR/api/Deeds/{ЕИК}`. Either the declarant is named in the company's +live deed, or facts they declared match the ones registered against that ЕИК. + +Four things must be on the table before the decision, because each of them changes what is being decided. + +**1. This is an expansion, not a tightening.** The measured surface grows from 101 links to **329 links / +311 people / 239 companies**. „From held / published / withdrawn = 163 / 65 / 101" means **264 links we +deliberately withhold today become named public claims**. At 329 links a 1% error rate is three falsely +named people. The precision bar has to go **up**, not stay flat — which is why the acceptance gate below is +a hand-labelled sample and not the control totals. + +**2. The second rung is a heuristic used to assert — stated precisely.** What the registry evidence +establishes is **the identity of the company**, not that the official owns it: the ownership claim comes +from the official's own filed declaration and is not a heuristic at all. Rung 2 („Документ") confirms that +*the company this declared name refers to is the same legal entity as the winner we matched*, by finding +the declarant's full name inside that company's live fields. A homonym failure therefore does not invent an +ownership claim — it **attaches the official to the wrong company's ЕИК, contracts and money**. That is a +different error from the one the adversarial reading suggests, and it is still a false public claim about a +named person. + +A three-name subset match against free text, in a register that carries no ЕГН, is **not deterministic**. +ADR-0007's decision list rules it out twice: item 2 (published only when the official↔company↔ЕИК +resolution is *deterministic*) and item 3 (heuristics may *withhold* or *triage*, **never assert**). #279 +replaces a weaker heuristic with a stronger one and promotes it to grounds for assertion. That is the +decision this ADR exists to make explicit; it cannot be smuggled in as an implementation detail. + +**3. The shape of the access is the permitted one.** Spec §3.3 permits „bounded per-ЕИК … thousands, not +~900k" and forbids bulk scraping; ADR-0007 decision 1 already anticipated the „per-ЕИК lookup only, later +leg". The crawl is one lookup per candidate ЕИК — on the order of hundreds — with a closed candidate set: +the crawler never follows a link out of a deed. This is that later leg, not the bulk reuse. + +**4. The endpoint rate-limits, and the limiter is a stated preference.** An earlier spike against this same +API triggered **HTTP 429 at roughly 50 cumulative requests ending in a burst**, and the block was then +*sustained* — every subsequent request returned 429, including simple `/Deeds/{eik}` calls that had worked +seconds earlier. There is no `Retry-After` and no `X-RateLimit-*` header, so a client can only avoid +tripping the limit, never pace against a published one; 25 spaced requests were fine, so pacing matters more +than total volume. #279 §3's „~400 companies, about 20 minutes" is therefore feasible only at its stated +1 request / 3 seconds with no burst, and only if the crawl is resumable. The limiter is the operator's only +available way to express a rate preference, and tuning around it empirically is precisely the posture spec +§3.3's „NEVER bulk-scrape" exists to forbid. This is the evidence behind decision 7's crawl hygiene. + +**5. Parts of the source description in #279 were wrong, and were corrected by live probing** (9 sequential +requests, 2026-08-04 — well inside the pacing above, so no 429 was encountered): + +- `?entryDate=` **is** honoured — a bogus-parameter control returned byte-identical output to the + plain request, so the difference is the parameter, not noise. `…/Fields/{ident}/History` **does** return + JSON, not application HTML. #279 §3 states the opposite for both. +- A wrong `subUIC` yields **HTTP 500 `GL_ERROR_L`**, not 404 — so a 500 can never be read as „no history" + or „outside the register". +- `legalForm` 4 = ООД and 10 = ЕООД (as #279 says); the nomenclature endpoint does **not** settle the enum + — `legalForm=4` and `legalForm=10` return a byte-identical catalogue. +- The envelope's `fullName` carries the legal form (`"ПИМК" ООД`) while `CR_F_2_L` is bare. The joint-stock + bar can therefore be derived from the ЗТРРЮЛНЦ-mandated suffix, at zero extra requests. +- Erasure is **structural, not textual**. Entities inside one field are separated by + `
` blocks delimited by `
`. + An erased entity is such a block carrying an erasure marker. **Correction, on implementation:** this ADR + first named that marker `field-text--erased`. Re-measured against the full live deed for ЕИК 115536179, + that class occurs **zero** times; the register emits `
` (with + ``) and the erased container carries **no `field-text` paragraph at + all**. The parser therefore treats *either* spelling as erasure — honouring both costs nothing, while + assuming one costs a wrong publish. `fieldOperation` is **not** the signal: it reads 2 on both erased + fields in that deed but 1 on the erased history records W0 sampled, so it is an undocumented enum we do + not rely on. +- Erased versions are **live in the current deed**: the first company sampled carries a fully-erased + `CR_F_23_L` dated 2013-07-16, which read naïvely becomes „latest ownership entry: 2013-07-16" and feeds + the refutation rule. +- Seats move. The same company's `CR_F_5_L` shows entry dates 2010-11-04 and 2014-01-23. +- **An ЕИК that is not a търговец answers `HTTP 200` with a ZERO-BYTE body** — not the 404, and not the + HTML, that §3 predicts. Measured on Община София (`000696327`): empty on two consecutive requests, + while a real company returned its full 34,398-byte deed in the same window, so it is the register's + answer rather than an outage. This is what the „извън ТР" rung actually looks like on the wire. The + distinction that keeps R6 honest is therefore the **status**, not the empty body: empty under 200 is + a documented negative and may be cached permanently; empty under 5xx is a failure and stays transient. + +The as-of capability that `?entryDate=` unlocks would fix the two weakest rungs, but it **re-baselines every +control number in #279 §10**. It is therefore deliberately out of scope here and becomes a separate change +under a new rules version — which decision 6 makes the only sanctioned way the surface may move. + +## Decision + +### 1. The evidence ladder replaces the publish tiers + +For every resolved link (person × ЕИК) with a fetched deed, **the first matching rung wins**: + +| # | Outcome (data label) | Condition | Effect | +|---|---|---|---| +| 1 | **„Бар: акционерна форма"** — joint-stock bar | the company is a joint-stock form (АД / ЕАД / КДА) | never published, whatever follows | +| 2 | **„Документ"** — documented | the declarant's full name appears in a live `CR_F_7_L` / `CR_F_18_L` / `CR_F_19_L` / `CR_F_23_L` | published; the role (owner vs manager only) is kept for the label | +| 3 | **„Потвърдено"** — confirmed | the normalized declared seat equals the registered seat for that ЕИК, **or** the declarant wrote the ЕИК in the declaration | published | +| 4 | **„Оборена"** — refuted | *own* stake only: the person appears in no live field, **and** the latest entry date across the live ownership fields is strictly before the first declared year | link withdrawn | +| 5 | **„Неизвестна"** — unknown | anything else | held | +| 6 | **„Извън ТР"** — outside the register | the ЕИК is not in the register (ДЗЗД, БУЛСТАТ associations) | held | + +The Bulgarian labels are the persisted vocabulary — they reach the data, the audit and the methodology +page — so they are fixed here rather than translated at each layer. + +Erased versions are skipped everywhere the live state is read — otherwise the date on an erased record can +„certify" a state that is not in force. + +`publish_tier` carries the evidence kind. **`B_distinctive` leaves the publishing path entirely**; +name distinctiveness survives only as an ordering signal for the review queue. Rung 3's seat leg takes the +declared seat **only from declarations by the same person for the same company**: 4.9% of company-name keys +carry more than one distinct declared seat, so a company-only key would let one person's seat confirm +another person's link. + +This supersedes ADR-0009 (the tier ladder) and ADR-0015 (the name census, whose only job was to unblock +`C_hold`; `tr-census.mjs` and its `promote()` are deleted). ADR-0028's holding survives — a declared ЕИК +*is* the identity — but it is now rung 3 rather than a tier of its own, it is subject to rung 1, and its +census-exemption clause is moot. + +### 2. A heuristic may ground an assertion here — the argued exception to ADR-0007 + +Rung 2 asserts on a name match. We accept it, on these grounds and no wider: + +- **It is a full-subset match, patronymic included.** Partial (2-of-3) matching is refused outright: of 301 + matches measured, 46 were two-token only — precisely the homonym risk. A declarant name with fewer than + three tokens can never earn „Документ". +- **It is scoped to a single registry entity.** Tokens are matched only within one `record-container` + block, after erased blocks are dropped and after HTML entities are decoded. Matching against a whole + field's text would combine the given name of one person with the surname of another — the defect that + ships a libel. +- **The company identity behind it passed preventive control.** The candidate ЕИК comes from the + deterministic exact-name resolution of ADR-0008/#226, not from fuzzy matching. + + > **Superseded on this point by [ADR-0035](0035-registry-evidence-must-also-establish-the-company.md).** + > Deterministic is not the same as correct: that resolution ranges over PROCUREMENT WINNERS only, so an + > official whose real company never bid resolves to a same-named winner, and a homonym in the winner's + > deed completes a link false in both halves. Rung 2 now requires a corroborator for the COMPANY — + > declared ЕИК, a matching declared seat, or a distinctive фирма — and the uncorroborated remainder is + > withheld as `document_uncorroborated` and counted. +- **It is corroborated, and that is the whole difference.** A bare TR name is spec §4's *weakest* join — + no ЕГН, no birthdate, not even ADR-0026's `(name, ведомство)` grain. The registry-graph spike that + explored this API concluded, for its own use, that no person-derived edge may reach a user-visible + surface at all, and it named the one cheap corroborator that would change that: *an officer who also + appears as a declarant with a declared stake in the same ЕИК, confirmed by an independent source*. That + is exactly what rung 2 is. The register supplies the name-in-this-company fact; the official's own filing + supplies the stake. Neither alone would publish. **The corroboration is what licenses the assertion, so + the rule must never be extended to a name-only join** — matching a person across two companies, or + treating a registry name as an identity in its own right, stays forbidden. +- **The filters that can only withhold are retained, not removed.** `nameGloballyUnique` and + `nameDistinctiveness` stay in the pipeline as an **AND-gate on the weakest rung only** („Потвърдено"). + They cost near-zero recall and preserve ADR-0017's outcome even though its subject — the name-distinctive + tier — is gone. +- **Homoglyphs are not folded.** `company-name-key.ts` deliberately does not fold Cyrillic↔Latin; person + names take the same posture. A Latin letter inside a name means no „Документ", and the occurrence is + counted rather than silently dropped. + +**The residual collision rate is estimated, not measured, and we say so.** Bulgarian names are three-part +by statute (ЗГР чл. 9) and the API renders full triples consistently. A triple of common components — +Георги · Иванов · Петров — is on the order of 10⁻⁵ of ~3.2M men, so roughly 29 people nationally share it; +but the population that can produce a false link is company officers, a ~1% slice, which squares the +probability to ~10⁻⁴ per colliding pair. That is an argument for plausibility, not a measurement, and the +residual concentrates on exactly the common names. It is therefore **not** the basis on which this +publishes — the hand-labelled sample of decision 7 is. Hyphenated surnames count as one token; Latin-script +names are counted the same way but flagged, since a three-token foreign name is not a patronymic triple. + +The tie-breaker for every rung is the repo's own sentence, from +`packages/shared/src/company-name-key.ts`: **„When in doubt the key stays MORE specific (a recall miss is +safe; an over-merge is not)."** + +This **amends ADR-0007 decisions 2 and 3**: a published claim may now rest on a disclosed, bounded, +entity-scoped full-name match against a public register, in addition to deterministic facts. Everything +else in ADR-0007 stands — in particular that every heuristic is disclosed on the methodology page and +labelled in the data, which decision 7 makes a launch condition rather than a follow-up. + +### 3. The joint-stock bar is a union of three independent signals + +The bar exists because the shareholder book is not public (so the claim is unverifiable) and a parcel of +listed shares is not a material ownership conflict (the „11 Trace shares → €88M" trap). It fires if **any** +of the following says joint-stock: + +1. `closelyHeldForm` on the **declared** name (`load.mjs`) — a different, earlier stage; it **stays**, and + is not redundant with the two below; +2. the фирма suffix taken from the deed envelope's `fullName`, through that same tested predicate; +3. the numeric `legalForm` code from the deed. + +**An unknown `legalForm` code withholds and is reported** — it never falls through to publication. The enum +is known only from four observed values and the nomenclature endpoint does not settle it (Context 5), so +fail-open here would be a bar that silently stops barring. `КДА` — which rung 1 explicitly requires barring +— is absent from both `JOINT_STOCK` and `FORM_TOKENS` in `classify.mjs` today and is added. + +**The named open question is ЕАД.** #279 §3 lists 5 = АД, and the spike's catalogue-derived table also +reads 5 as АД/ЕАД jointly — but ООД and ЕООД turned out **not** to share a code (4 and 10), so a separate +ЕАД code is the likelier reading and no observation settles it. A single-owner joint-stock company is +precisely the shape a declarant is most likely to hold and rung 1 most needs to bar, so this is the one +place where a fail-open enum would do real damage. Signal 2 (the ЗТРРЮЛНЦ suffix on `fullName`) covers it +independently of the code, which is why the bar is a union and not a lookup. + +### 4. Termination is reconciled against the live deed — reversing ADR-0021 E11 for own stakes + +ADR-0021 E11 marks an ownership link `withdrawn` when the company is absent from the person's latest +ownership filing. That is an inference from silence, and its commonest cause is a finished mandate rather +than a sale. Before the withdrawal takes effect, every terminated **own** stake is reconciled against the +live deed: a person still named in a live ownership field (`CR_F_18/19/23_L`) has not divested, and the +link surfaces. Family stakes are never reconciled — the relative's name is neither stored nor checked +(ADR-0010 decision 4, ADR-0032 decision 2), so declared termination applies to them directly. + +**Phase 1 uses the reconciliation only to avoid withdrawing the link.** The „и към днешна дата" labels of +#279 §7, and the post-period contracts they license, are **deferred to phase 2 behind an LIA addendum**: +those labels assert a present tense about a named person, on evidence whose freshness is bounded by the +cache refresh cycle, and they change the claim's shape enough to need the balancing assessment updated +first. The derived live status is therefore recomputed on every run and **never sealed**. + +ADR-0021 **E10 is untouched and strengthened** — see decision 7. + +### 5. The deed cache — amends ADR-0010 + +A deed contains third-party personal data: addresses of natural persons, and the names of people who are +not office holders. + +- **Extraction stays inside ADR-0010 decision 3.** `CR_F_5_L` yields only the „Населено място:" segment; + no parser function returns an address. Decision 3 is re-affirmed, not overridden. +- **Storage is what changes, and that is ADR-0010 decision 6.** Raw deeds are cached under git-ignored + `scratch/tr/`, behind the same refuse-to-run guard as the declaration cache, with a **35-day retention** + (one refresh cycle plus slack) and a purge step in the same job. Decision 6's scope extends from „raw + declaration XML … deleted post-spike" to a second source with a stated TTL. +- **The cache index holds no name at all** — only ЕИК, dates, codes and verdicts, plus a body hash instead + of any content excerpt. Names exist solely in the raw JSON files, are read only to produce a boolean, and + never enter a public table, a response, or a log. ЕГН was absent from every payload examined, and the + index additionally refuses any ten-digit run in a text column — sound because an ЕИК is 9 or 13 digits, + never 10. +- **The deed's beneficial-ownership fields are deliberately not used.** A deed also carries `CR_F_550_L` + (действителни собственици, чл. 63 ЗМИП) and the control fields `CR_F_537/538_L`. Using them would make a + beneficial-ownership claim, which ADR-0007 decision 1 parked after CJEU C-37/20. Rung 2 reads the four + fields #279 names and no others; widening it is a new decision, not an improvement. +- **The sealed `matched_fact` is a closed vocabulary** (`seat:`, `role:owner:`, `eik`), + enforced by a pattern check in the audit. Without that, the matched *name* eventually gets stored there + as a convenience, which is exactly what #279 §9 forbids. + +### 6. Monotonicity is a gate, not a store — and #279 §8 is corrected + +#279 §8 requires the evidence seal to be kept „forever" and recomputation to be strictly additive. As +written that is false, and the implementation must not pretend otherwise: + +- §7's labels flip. A person who leaves the register must lose the label *and* the post-period contracts it + licensed. (Phase 1 avoids this by not shipping the labels at all — decision 4.) +- A permanent seal and a live, expiring cache contradict each other. +- A **court-annulled entry** (чл. 29 ЗТРРЮЛНЦ) invalidates the evidence without any rules change. It is + wired to the correction path of ADR-0031 and named as a ground in the suppression runbook. + +Therefore: seals are **re-derived deterministically** on every run — a seal is written for *every* link, +including held ones, so the review queue explains itself — and monotonicity is enforced as a **gate**. The +audit compares against the pre-wipe export and raises a hard finding when a previously published link +disappears under an **unchanged** `rules_version`; under a changed version it degrades to a printed diff. +Removal remains an intentional event, and each ground has an expressible mechanism — a gate that hard-fails +the only removals it sanctions is a deadlock, not a rail: + +| Ground | Mechanism | Why not one of the others | +|---|---|---| +| The rules changed | a `rules_version` bump | — | +| The evidence is void (court annulment, wrong person) | ADR-0031 suppression — `status` flips to `suppressed`, so the audit reads the current status and treats it as declared | the link is correctly built; only its publication is wrong | +| The input was wrong | `scripts/cacbg/link-corrections.jsonl`, fingerprinted like the suppression list; `load.mjs` flags the key in the pre-wipe snapshot | correcting the input *unbuilds* the link, so a suppression on it matches nothing and trips the unused-entry rail | + +Both non-rules grounds are one-shot, version-controlled, and reviewed in git; neither is silent — the audit +prints every declared removal with the ground that licensed it. An acknowledgement that matches nothing +fails the build, because a stale one would pre-clear a *future* disappearance of that same link. + +### 7. Launch gates + +None of these are follow-ups. + +- **Precision is proven by a hand-labelled sample, not by the control totals.** A human verifies against the + portal: every reconciled link, every refuted link, and random samples of „Документ" and „Потвърдено". + **Pass mark: zero wrong-company and zero wrong-person errors.** The control numbers of #279 §10 are a + reproducibility check, and two things had to be settled before they could serve as one. + + **The §5/§10 discrepancy is a dropped histogram category, not a disputed measurement.** §5's rungs carry + their own control counts: 4 barred + 281 document + 102 seat + 3 ЕИК + 21 refuted + 156 unknown = 567, + plus the 4 links whose ЕИК is not in the register at all (§5 scopes the ladder to links „с изтеглен акт", + so those reach no rung) = **571**. §10's identity row is 281 / 102 / 156 / 21 / 4 / 4 = **568**. The + difference is exactly the „3 по ЕИК". + + The cause is visible in the labels. The identity row names its second bucket **„седалище"** — one leg of + rung 3 — while the evidence row two lines below names the same rung **„потвърдено"** (251 / 78). Rung 3 + has two legs; the row tallied one and presented the result as a partition of the resolved set. And the + stated total 568 equals that row's sum exactly, which makes it a figure derived from the histogram rather + than measured independently. So both move together: the bucket becomes **„потвърдено: 105"** and the + resolved total becomes **571**. F8 measures against those and reports if its own count disagrees — the + reconciliation is not permitted to adopt whichever reading makes it pass. + + **Not a discrepancy, and not to be „fixed":** „извън ТР" is 4 in §10 and 3 in §3/§11 because the units + differ — 3 ЕИК that are not in the register, carrying 4 links between them. §10's row counts links. + + The numbers must also be re-measured on top of the §12 phantom-row fix, which changed the corpus — that + fix has landed as #281 (87 phantom rows across 15 sets; 256,286 announced − 87 = 256,199, covered exactly + by 255,582 fetched + 617 missing at source). +- **Every anti-false-zero control is a positive control.** „Sofia seat confirmations: 0" is indistinguishable + from a broken normalizer without one; so is a joint-stock bar with no marginal effect over + `closelyHeldForm`, and a matcher that always returns false (ADR-0027's lesson). +- **A partial cache must fail closed.** A missing *or* incomplete registry cache makes the loader throw. An + 80%-restored cache would otherwise yield roughly 80 published links — above the ship floor of 50 — and so + would ship a decimated surface and wipe the rest from production. The ship floor rises accordingly and is + passed explicitly rather than defaulted. +- **Crawl hygiene.** „Outside the register" is permanent only from a documented positive response; 429, + 5xx and timeouts are transient and are never cached as a negative. A 429 stops the run rather than + marking anything, and is never retried — „5 retries with growing backoff" and „429 stops the run" are + consistent only if retries exclude 429. Given Context 4, the crawl is sequential, paced, resumable, and + ends the run on the first 429 rather than backing off into it. **The deed's returned UIC must equal the + requested ЕИК**, or the deed is refused: an ЕИК is TEXT everywhere because public bodies' identifiers are + exactly the `000…` shape that loses its leading zeros to a numeric round-trip, and the failure mode of + losing them is fetching, and then publishing against, a different company's deed. +- **The methodology page carries the rule verbatim** (ADR-0021 E10): the ladder, the ≥3-token requirement, + the entity-boundary rule, the disclosure that identity rests on a name match **without ЕГН**, the homonym + and seat caveats, the lookup date, the refresh cadence and the retention. E10's existing promise that + held links „се показват едва след като регистърът стане достъпен" is finally kept by this change. +- **The PII rails are asserted, not assumed** — over the shipped dump and over the loader's own output. + +## Consequences + +- The surface roughly triples, and 264 links that are withheld today become named public claims. The + failure mode of the whole change is a **wrong-company** attribution, not a wrong publication decision. +- `tr-census.mjs` and the open-data census pipeline of ADR-0015 are removed. Its premise — that a name + proven nationally unique identifies the company — is the premise this ADR abandons. +- The pipeline gains a hard dependency on an external registry. Every path that lacked one before now has a + fail-closed branch: no cache, partial cache, unknown legal form, rate limit. +- We accept a slower, more fragile refresh in exchange for evidence. Decisions stay a pure, zero-network + function of declarations, cached deeds and contracts, but they no longer run on the 6-hourly ETL cycle + (#279 §9 assumes they do; they do not — the loader needs `node:sqlite`, the declaration corpus and the + full winner set, none of which exist inside a Worker). Cadence becomes two scheduled workflows: decisions + daily, registry lookups monthly. **Superseded by [ADR-0034](0034-registry-lookups-and-decisions-share-one-monthly-run.md):** + the split rests on the decision run being able to work from the cache alone, and it cannot — the strongest + rung compares the declarant's name against the deed text itself, which the index deliberately does not + store, so the raw deeds must be present; and they must not survive the runner. One monthly job, therefore. + Nothing else in this ADR is affected. +- Two capabilities are deliberately left on the table and become separate changes under new rules versions: + as-of evidence via `?entryDate=`, and the §7 „и към днешна дата" labels with their post-period contracts. +- The seat rung is structurally capped and we accept the cap: a declared seat is captured **only** from the + ООД/ЕООД holdings table of an *asset* declaration (`parse.mjs`), so it can never rescue a link declared + solely in an interests declaration. This is probably consistent with the 102 measured for the seat leg + specifically (not the 105 of the whole rung, whose other three come from the ЕИК leg) — rung 1 bars the + joint-stock cases anyway — but it is confirmed against the corpus before launch, not assumed. + +### Measured outcome — recorded on completion + +This ADR is accepted on the design. Exactly one amendment is permitted afterwards: this subsection is +filled in with the measured result, and nothing else in the file is rewritten (repo convention — an +accepted ADR is superseded, not edited). To be recorded: + +- the realized ladder split against the #279 §10 control row **as corrected above** (281 / 105 / 156 / 21 / + 4 / 4 = 571), with each number naming the SQL that produced it, and the gap between identified (281 + 105) + and surfaced (329) attributed bucket by bucket — 57 links on the corrected reading, 54 on the row as + written, so the figure itself distinguishes the two; +- the true candidate-ЕИК count (#279's „~400" is an assumption; the figure is `COUNT(DISTINCT eik)` over + all links, not just published ones); +- the hand-labelled sample result; +- the marginal effect of the registry joint-stock bar over `closelyHeldForm`, and the count of unknown + `legalForm` codes encountered. diff --git a/docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md b/docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md new file mode 100644 index 000000000..c9afd007e --- /dev/null +++ b/docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md @@ -0,0 +1,63 @@ +# ADR-0034 — Справките в ТР и решенията текат в едно месечно задание + +- **Статус:** Прието +- **Дата:** 2026-08-11 +- **Обхват:** конвейерът за свързани лица — `related-persons-data.yml`, `scripts/cacbg/load.mjs`, + `scripts/tr/fetch-deeds.mjs`. Заменя единствено твърдението за каденцията в последствията на + [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md); всичко останало в него стои. + +## Контекст + +ADR-0033 записа каденцията така: **„решения ежедневно, справки в регистъра месечно"**. Разделянето +почиваше на едно предположение — че решаващият ход може да работи само върху кеша, а обхождането е +отделен, по-рядък ход. Прегледът на PR-а показа, че това предположение не се държи. + +Двете страни на противоречието: + +1. **Суровите актове не бива да преживяват изпълнителя.** Кешираният акт носи имена на трети лица — + съдружници и управители без публична длъжност, и адреса на дружеството. Пренасянето му между ходове + изисква хранилище (кеш на Actions или артефакт), чието изгонване е политика за капацитет, не за + задържане: 35-дневният срок по решение 5 на ADR-0033 се изпълнява от `purgeExpired` на изпълнителя и + няма никаква власт над копие в хранилището на GitHub. Това беше находка B3 на прегледа. +2. **Решението не може да се вземе без суровите актове.** Най-силното стъпало сравнява трите имена на + декларатора със самия текст на записа. Индексът на кеша нарочно не пази нито едно име (§9), така че + „кеширай само индекса" премахва точно данните, от които `evidenceVerdict` се нуждае. + +Двете заедно значат: **всеки ход, който решава, трябва и да обхожда.** А ежедневно решение тогава значи +~400 заявки дневно към чужд публичен регистър, което спец. §3.3 („НИКОГА не обхождай ТР масово"; +КЗЛД, CJEU C-200/23) не разрешава да се приема с лека ръка — ограничителят на регистъра е единственият +начин операторът да изрази предпочитание за темп и ние не се настройваме около него емпирично. + +## Решение + +**Едно задание, месечно.** `related-persons-data.yml` изпълнява последователно: зареждане на списъка с +кандидати (`load.mjs --emit-candidates`, върху захвърляемо копие) → обхождане на регистъра → решение и +публикуване (`load.mjs`) → одит (`audit.mjs`) → изтриване на суровите актове в стъпка `if: always()`. +`scratch/tr` **не се кешира никога**; кешира се само `scratch/cacbg/raw` (публични декларации). + +`related-persons-tr-refresh.yml` е премахнат — нямаше какво да разделя. + +**Отхвърлена алтернатива: пренасяне на актовете между ходове** (кеш или артефакт с правило за +жизнен цикъл). Работи технически, но връща B3: срокът на задържане престава да е наш и става политика на +чуждо хранилище. R2 със собствено правило за изтриване остава разумен вариант, ако някога се наложи — +днес не се налага. + +**Отхвърлена алтернатива: ежедневно решение.** Единственият начин да се запази е обхождането да издава +**присъди по (връзка, ЕИК)** — булеви стойности без нито едно име — така че само те да пресичат +границата между ходовете. Това е коректната посока и остава на масата, но е промяна в дизайна на +доказателствения слой, не настройка на график, и не влиза в тази промяна. + +## Последствия + +- Публикуваната повърхност се преизчислява **месечно**, не ежедневно. Практическата цена е по-малка, + отколкото звучи: входът също се движи бавно (декларациите се подават годишно, вписванията в ТР — + рядко), а датата на справката се показва на всяка връзка, така че давността е видима за читателя, а не + подразбираща се. +- **Спешна корекция не чака графика.** Свалянето на връзка минава през ADR-0031 и ръчния ход, а при + жива експозиция — през пряк `UPDATE` върху обслужващата D1; вж. + [ръководството](../runbooks/related-persons-suppression.md). Каденцията определя кога се преоткриват + връзки, не кога може да се махне една. +- Едно задание значи един режим на отказ: ако обхождането спре при 429, гейтът за покритие на `load.mjs` + отказва частичния кеш и ходът не публикува нищо. Това е желаната посока — по-скоро без обновяване, + отколкото с орязана повърхност — и е същият гейт, който вече пазеше разделения вариант. +- #279 §9 продължава да описва ежедневни решения. Този ADR е записът, че не е така, и защо. diff --git a/docs/adr/0035-registry-evidence-must-also-establish-the-company.md b/docs/adr/0035-registry-evidence-must-also-establish-the-company.md new file mode 100644 index 000000000..4ba01f701 --- /dev/null +++ b/docs/adr/0035-registry-evidence-must-also-establish-the-company.md @@ -0,0 +1,81 @@ +# ADR-0035 — Регистровото доказателство трябва да установи и ДРУЖЕСТВОТО, не само лицето + +- **Статус:** Прието +- **Дата:** 2026-08-12 +- **Обхват:** доказателственото стъпало 2 („Документ") — `scripts/tr/evidence.mjs`, `scripts/cacbg/load.mjs`. + Заменя единствено твърдението в решение 2 на + [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md), че стъпало 2 не носи допълнителен + гейт; всичко останало в него стои. + +## Контекст + +ADR-0033 записа за стъпало 2: „по-силното стъпало не е гейтнато — регистърът е назовал това лице в ТОВА +дружество, което прави ключа по име без значение". Прегледът на PR-а показа, че изводът важи само ако +„ТОВА дружество" е дружеството, което лицето е декларирало. Не е гарантирано. + +Веригата, по която се стига до акта: + +1. `resolveEntity` (`load.mjs`) свежда декларираното фирмено наименование до **едно ЕИК на ПЕЧЕЛИВШ** — + участник в обществена поръчка. Дружества, които никога не са участвали, не съществуват за резолвера. +2. `nameGloballyUnique` проверява уникалност **само в множеството на участниците**, не в Търговския + регистър. „Единствено" тук значи „единственото сред печелившите". + +Оттук следва сценарият, който прегледът описа и който възпроизведохме: + +- Длъжностното лице действително притежава «АЛФА ООД» с ЕИК **A**, което никога не е участвало в поръчка. +- Друго, несвързано «АЛФА ООД» с ЕИК **B** е печеливш и единственият участник с това наименование. + Резолверът избира **B**. +- Актът на **B** съдържа **съименник** на декларатора — същите три имена, което в България не е рядкост. + Стъпало 2 съвпада и публикува връзка, **невярна и в двете си половини**. + +Регистровото доказателство доказва, че *някой с това име* е съдружник в **B** — не че *това лице* е. +Тоест стъпалото **премести** риска от съвпадение на имена в регистъра, вместо да го затвори, а точно +затварянето му е основанието на #279. Съществуващият карантинен механизъм за двусмислени ключове покрива +само сблъсък печеливш-срещу-печеливш; случаят печеливш-срещу-непечеливш не беше нито обработен, нито +записан като приет остатъчен риск. + +## Решение + +**Преди стъпало 2 да твърди, нещо извън фирменото наименование трябва да каже, че дружеството е +декларираното.** Съвпадението по име остава необходимо, но вече не е достатъчно само по себе си. + +Приема се, ако е изпълнено **поне едно** от: + +| Потвърдител | Защо стига | +|---|---| +| деклариран **ЕИК** | националният идентификатор по ЗТРРЮЛНЦ решава дружеството еднозначно, независимо от споделена фирма (ADR-0028) | +| декларирано **седалище**, съвпадащо с вписаното | близнак в друго населено място отпада; същият факт, на който публикува стъпало 3, включително темпоралната гаранция R10 | +| **отличително** фирмено наименование (`nameDistinctiveness`) | национален близнак е малко вероятен изобщо да съществува | + +Съвпадение по име, което не мине гейта, връща **нов, отделен въздържащ се резултат +`document_uncorroborated`** — никога не публикува, не носи роля и не носи `matched_fact`. + +**Защо отделен вид, а не пропадане към `unknown`.** „Съвпаднахме лице, но не установихме дружеството" и +„не съвпаднахме нищо" са различни факти за една връзка. Печатът се пише за всяка връзка, включително +задържаните, именно за да е прегледаема опашката (migration 0006); сливането им я заслепява точно там, +където решението има нужда от числото. + +**Третият потвърдител е граница, не доказателство.** Затова се **брои** (`document_uncorroborated` в +обобщението на зареждането) — това е числото, което F8 чете, за да реши дали гейтът да се стегне до +първите два. + +**Отхвърлена алтернатива: строго потвърждаване** (само ЕИК или седалище). По-висока точност, но +декларирано седалище се извлича само от таблицата ООД/ЕООД на **имуществена** декларация — никога от +декларация за интереси — така че цената в покритие е неизвестна преди F8 и можеше да свали повърхността +под прага за публикуване. Остава на масата и е точно решението, което измереният остатък ще информира. + +## Последствия + +- Стъпало 2 вече е **гейтнато**, което ADR-0033 решение 2 изрично казваше, че не е. Това е единственото + твърдение оттам, което този ADR заменя; наборът от стъпала, редът им и правилото „първото съвпадение + печели" стоят. +- Гейтът се проверява по подразбиране **затворено**: `companyNameDistinctive` е `false`, ако не бъде + подадено. За разлика от `nameGloballyUnique` — чието разрешаващо подразбиране е ограничено до + най-слабото стъпало — този гейт пази основното публикуващо стъпало. +- Стъпала 2 и 3 споделят **една** реализация на съвпадението по седалище (`matchDeclaredSeat`), заедно с + R10. Две копия на „какво брои за съвпадение по седалище" рано или късно се разминават за това кои връзки + могат да се публикуват. +- Покритието пада с неизмерена величина — по построение точно върху връзките, за които дружеството е + най-слабо установено. Това е приетата посока: по-скоро задържана вярна връзка, отколкото публикувана + невярна, назоваваща действително лице. +- Методологичната страница оповестява правилото дословно (ADR-0021 E10). diff --git a/docs/adr/README.md b/docs/adr/README.md index 0bd84ab6a..0d63413fc 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -16,15 +16,15 @@ | [0006](0006-eop-wins-dedup.md) | Dedup на два източника: EOP печели по `contract_number` | Прието | | [0007](0007-scope-and-certainty-bar.md) | Свързани лица: обхват и праг на сигурност (само 100% детерминистични съвпадения) | Прието | | [0008](0008-deterministic-name-to-eik-resolution.md) | Детерминистично разрешаване име→ЕИК (авто-публикуване; ръчна опашка само при двусмислие) | Прието | -| [0009](0009-name-uniqueness-guard-and-publish-tiers.md) | Пазач за уникалност на името + нива на публикуване (A_seat / B_distinctive / C_hold) | Прието | +| [0009](0009-name-uniqueness-guard-and-publish-tiers.md) | Пазач за уникалност на името + нива на публикуване (A_seat / B_distinctive / C_hold) | Заменено от [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) | | [0010](0010-pii-posture.md) | Позиция за лични данни: без ЕГН/адреси; третите лица — само за вътрешна проверка | Прието | | [0011](0011-host-scoped-tls-pinning.md) | TLS pinning само за хоста (счупена верига на register.cacbg.bg), не глобален байпас | Прието | | [0012](0012-crawler-and-persistence-architecture.md) | Архитектура на обхождането и съхранението (resumable; кеш по xml_file + ControlHash) | Прието | | [0013](0013-two-declaration-templates.md) | Два шаблона декларации (имущество + интереси) в един парсер | Прието | | [0014](0014-match-output-layers-and-interpretation.md) | Слоеве на съвпадението и тълкуване (собственост/контрол, времеви, собствена институция) | Прието | -| [0015](0015-tr-name-uniqueness-census.md) | Преброяване за уникалност на имена от ТР — промотира глобално уникални tier-C връзки | Прието | +| [0015](0015-tr-name-uniqueness-census.md) | Преброяване за уникалност на имена от ТР — промотира глобално уникални tier-C връзки | Заменено от [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) | | [0016](0016-free-text-entity-resolution.md) | Разрешаване на субекти от свободен текст (деклариран ЕИК + извличане от проза) | Прието | -| [0017](0017-name-collision-tier-gate.md) | Гейт срещу колизия на имена извън отличаващото ниво | Прието | +| [0017](0017-name-collision-tier-gate.md) | Гейт срещу колизия на имена извън отличаващото ниво | Заменено от [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) | | [0018](0018-folder-discovery-and-republication-dedup.md) | Откриване на папки от индекса + dedup на препубликувани декларации (ControlHash) | Прието | | [0019](0019-private-interest-vs-ex-officio-classification.md) | Разделяне на частен финансов интерес от служебни борд-роли (multi-declarant tell) | Прието | | [0020](0020-conflict-explorer-surface-posture.md) | Повърхност на експлорера — само interest_links, noindex до одобрение, произход на всеки ред | Прието | @@ -40,5 +40,8 @@ | [0030](0030-family-ownership-withheld-nameless-aggregate.md) | Дял на свързано лице не влиза в поименната повърхност (v1): събира се и се одитира, но се отчита само като безименен сбор (GDPR C-37/20; заменя ADR-0023) | Заменено от [ADR-0032](0032-family-ownership-published-under-public-interest.md) | | [0031](0031-suppressions-version-controlled-fingerprinted.md) | Свалянията на връзки са версиониран списък с HMAC-отпечатък (salt = CI secret), прилаган при зареждане — не служебна таблица; не изтича сигнала „кой е свален" към прод | Прието | | [0032](0032-family-ownership-published-under-public-interest.md) | Дял на свързано лице се публикува на поименната повърхност наравно със собствения — на основание надделяващ обществен интерес (ЗДОИ чл. 41и; C-184/20 забранява само поименните данни); близкият никога не се назовава, връзката не се твърди; заменя ADR-0030 | Прието | +| [0033](0033-registry-evidence-replaces-name-distinctiveness.md) | Връзка се публикува само срещу проверим факт от Търговския регистър — доказателственият ред заменя нивата по отличителност на името; евристиката вече обосновава твърдение (изрично изключение от ADR-0007), барът за акционерни форми е обединение от три сигнала, монотонността е гейт, не хранилище; заменя ADR-0009/0015/0017 | Прието (дизайн) | +| [0034](0034-registry-lookups-and-decisions-share-one-monthly-run.md) | Справките в ТР и решенията текат в едно месечно задание — суровите актове не преживяват изпълнителя, а решението не може да се вземе без тях; заменя твърдението за каденцията в ADR-0033 | Прието | +| [0035](0035-registry-evidence-must-also-establish-the-company.md) | Стъпало 2 („Документ") публикува само ако извън фирменото наименование нещо установи, че дружеството е декларираното — деклариран ЕИК, съвпадащо седалище или отличителна фирма; иначе `document_uncorroborated`, което се брои; заменя твърдението в ADR-0033, че стъпало 2 не е гейтнато | Прието | Свързан проектен документ: [spec/related-persons-foundation.md](../spec/related-persons-foundation.md). diff --git a/docs/deploy.md b/docs/deploy.md index ac3e845ce..e296eda86 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -170,8 +170,17 @@ domain таблиците и преизчислява rollup-ите + FTS. > [scripts/precompute.sql](../scripts/precompute.sql) — не изпращайте FTS съдържание през dump. > Промени в схемата след първото зареждане се прилагат out-of-band, за всяка среда: -> `SIGMA_D1_NAME=sigma-stage wrangler d1 migrations apply sigma-stage --remote`. Деплоите не мигрират -> и не презареждат данни. +> `SIGMA_D1_NAME=sigma-stage wrangler d1 migrations apply sigma-stage --remote`. Деплоите **не презареждат +> данни**. +> +> **Уточнение (беше неточно):** тук пишеше и „деплоите не мигрират". Вече не е вярно — +> [deploy.yml](../.github/workflows/deploy.yml) прилага изрично `0003`, `0009` и `0010` с +> `d1 execute --file` при всеки деплой. Причината е, че ledger-ът на wrangler на тези бази е празен +> (базовата схема е създадена out-of-band), тъй че `d1 migrations apply` би се сблъскал с `0000`. И трите +> са безопасни за повторно прилагане: `CREATE … IF NOT EXISTS` и, при `0010`, тригери и индекси, които +> се пресъздават до същото състояние. Отделно от тях има и стъпки, които сондират таблицата и добавят +> само липсващите колони — за `0002` и за колоните на анексите. Останалите схемни промени наистина си +> остават out-of-band. ## 3. Конфигуриране на GitHub Environments diff --git a/docs/review-testing.md b/docs/review-testing.md index 3ab4b106a..0cbd4b0af 100644 --- a/docs/review-testing.md +++ b/docs/review-testing.md @@ -43,6 +43,26 @@ - Нов workspace с тестове се добавя и в `coverage-baseline.json` (и получава `vitest.config.ts` с `sharedCoverage(...)`); `packages/api-contract` е освободен, докато няма тестове. +### `scripts/` е извън ratchet-а — прегледано изключение, не пропуск + +Ratchet-ът покрива само `apps/*` и `packages/*`. `scripts/` (~7 хил. реда, сред които клеветнически +критичната логика на свързаните лица — доказателственият ред, съвпадението по имена, гейтовете при +публикуване) не се измерва от него. Записано е тук, за да е решение, а не дупка, която някой открива +по-късно. + +**Защо:** тези файлове са `.mjs`, пускат се от три отделни runner-а извън `turbo run test` и голяма част +от тях са I/O обвивки около мрежа и SQLite, за които процент покритие би измервал колко от обвивката е +докосната, а не дали правилото е вярно. + +**Какво пази качеството вместо процент:** и трите набора са в CI и са задължителни — +`node --test scripts/*.test.mjs`, `node --import ./scripts/cacbg/register-ts.mjs --test +scripts/cacbg/*.test.mjs scripts/tr/*.test.mjs` — а гейтовете, при които грешка означава поименно +твърдение за реален човек, се проверяват **мутационно**: тестът трябва да пада при обърнато условие, +иначе не е гейт. Виж ADR-0027 за защо тест, който минава и преди реализацията, е фалшив гейт. + +**Кога това трябва да се преразгледа:** ако `scripts/` започне да носи логика, която не е нито +чиста функция с мутационно проверен тест, нито тънка I/O обвивка. + ## Integrity gate - Пуска се върху обслужвания D1 след `precompute` в `ship-domain.mjs` и след `runSliceDerive()` в diff --git a/docs/runbooks/related-persons-suppression.md b/docs/runbooks/related-persons-suppression.md index ac2b140c9..7d407cd37 100644 --- a/docs/runbooks/related-persons-suppression.md +++ b/docs/runbooks/related-persons-suppression.md @@ -4,6 +4,21 @@ A contested, corrected, or legally-challenged link on the свързани-ли the version-controlled suppression list. See [ADR-0031](../adr/0031-suppressions-version-controlled-fingerprinted.md) for why this is a git list keyed on an HMAC fingerprint, not a DB row. +## Grounds that require a takedown + +Two of these are specific to the Trade Register evidence (#279, ADR-0033) and are easy to miss, because +neither shows up as a bug: the pipeline is working correctly and the claim is still wrong. + +- **A court-annulled entry (чл. 29 ЗТРРЮЛНЦ).** A registry entry set aside by a court is void, but the + deed we cached may still carry it, and the rules that read it are unchanged — so a rules-version bump + will NOT remove the link. This is an evidence-invalidation ground, and suppression is the mechanism. +- **A match against the wrong person (a namesake).** The register carries no ЕГН, so a three-name match + inside one entity is strong evidence of identity but not proof. If the named official is not the person + in the act, the link is a false public claim about them and must come down immediately — do not wait for + a rules change or a re-crawl. +- The ordinary grounds are unchanged: a contested link under review, a correction the source has since + published, or a legal challenge. + ## What you need - The exact `link_key`. Self links are `pid|eik`; a close-relative link is `pid|eik|family`. Get it from the @@ -46,6 +61,38 @@ wrangler d1 execute --remote \ Then do the Standard takedown steps above **in the same shift**. The direct UPDATE is undone by the next ship (which rebuilds the table); only the git list makes it permanent. +## Correcting a wrong INPUT (a different list — read this before reaching for a suppression) + +Suppression keeps a *built* link out of the public surface. It is the wrong tool when the link should never +have been built at all — a misparsed declaration row, a stake attributed to the wrong filing, an entity +that was never declared. There you fix the input, and the link stops existing. + +That removal still has to get past the monotonicity gate ([ADR-0033](../adr/0033-registry-evidence-replaces-name-distinctiveness.md) +decision 6), which hard-fails on a link that was published last run and is not published now. And you +cannot use a suppression to clear it: correcting the input unbuilds the link, so the fingerprint would +match nothing and `load.mjs` fails on the unmatched-entry rail instead. The two removals fail in opposite +directions, which is why there is a second list. + +1. Compute the fingerprint exactly as above (same salt, same function, same `key_version`). +2. Append one line to `scripts/cacbg/link-corrections.jsonl`: + + ```json + {"fp":"","key_version":"1","reason":"the declaration row was misparsed — ","signal_ref":"","corrected_at":"2026-08-11"} + ``` + +3. Land it **in the same PR as the input fix**. The loader flags that key in the pre-wipe snapshot and the + audit reads it as a declared removal — printed, never silent, and attributed to this ground rather than + flattened into "removed". + +**An acknowledgement is one-shot. Delete the line in the next change.** Once the corrected link stops being +published it also stops appearing in the prior set, so the entry matches nothing and the build fails on the +same rail suppressions use. That is deliberate: a stale acknowledgement would sit in the list and silently +pre-clear a *future* disappearance of that same link — the exact regression the gate exists to catch, with +nobody having decided it. + +Do not use this list to clear a removal you do not understand. If a published link vanished and you cannot +name why, that is the gate doing its job; find the cause. + ## Fail-closed behaviour `load.mjs` aborts the build (non-zero exit) on any of three conditions — each would otherwise silently @@ -65,6 +112,30 @@ If a ship fails on any of these, fix the cause; do **not** empty the list or dro A rotated salt invalidates every existing fingerprint, so it is a coordinated change guarded by `key_version`: re-derive `fp` for each entry from its original `link_key` under the new salt (you need the plaintext keys, kept out-of-band, e.g. in the incident tickets), **bump every entry's `key_version`** to the -new value, replace the file in one commit, and update both CI secrets (`SUPPRESSION_SALT` and -`SUPPRESSION_KEY_VERSION`). Because the loader refuses any entry on a non-current `key_version`, a -half-finished rotation fails the build loudly instead of silently un-suppressing. +new value, replace the file in one commit, and update **both** CI settings — but note they are different +kinds: `SUPPRESSION_SALT` is a repository **secret**, while `SUPPRESSION_KEY_VERSION` is a repository +**variable** (`${{ vars.SUPPRESSION_KEY_VERSION || '1' }}` in `related-persons-data.yml`). The version is +not sensitive — it is a counter — and looking for it under Secrets is a dead end during an incident. +Because the loader refuses any entry on a non-current `key_version`, a half-finished rotation fails the +build loudly instead of silently un-suppressing. + +## Verifying a takedown actually worked + +A takedown that silently failed looks exactly like one that succeeded: the entry sits in the list, the +build is green, and the link is still public. The list is applied at LOAD time, so nothing changes on the +served surface until the next data run ships — check the surface, not the commit. + +1. **The loader saw it.** The run refuses an entry matching no built link (the B3 unused-entry rail), so a + green run already proves the fingerprint matched something. A run that fails with „suppression matched + NO link" means the `link_key` was wrong — a family link needs its `|family` suffix. +2. **The row is gone from the served D1**, which is the only copy a reader can reach: + ``` + wrangler d1 execute "$SIGMA_D1_NAME" --remote \ + --command "SELECT status FROM interest_links WHERE link_key = ''" + ``` + Expect `suppressed`, or no row at all. Anything else means the ship did not carry the decision. +3. **The page is gone**, allowing for cache: the link's page must 404 and the official's page must not + list it. `publicCache(3600)` means a reader can still see it for up to an hour after the write — if it + is still there beyond that, the takedown did not land. +4. **It stays gone.** Re-run the next scheduled load and repeat step 2: the list is what makes a takedown + survive a rebuild, and this is the only step that proves the survival rather than assuming it. diff --git a/docs/spec/related-persons-foundation.md b/docs/spec/related-persons-foundation.md index 8c50fa949..61d399f52 100644 --- a/docs/spec/related-persons-foundation.md +++ b/docs/spec/related-persons-foundation.md @@ -70,6 +70,13 @@ Of #128's five checkboxes: **1** (shared owners) contingently unblocked via §3. **Per-ЕИК public lookup** (`portal.registryagency.bg`) is free, no fee. Recoverable per ЕИК: **управители, съдружници/собственици на капитала + stakes, капитал, legal form, seat.** **ЕГН masked** → owner identity name-based. Run **only** for the bounded set (bidders + resolved declared/donor companies) — thousands, not ~900k. **Hard constraints (DPA-verified):** **NEVER bulk-scrape TR** (КЗЛД ruled bulk provision unlawful; CJEU C-200/23 climate). **Never store scraped ЕГН**, even if leaked in a document image. Phase-0 confirms endpoint, fields, and rate limits. +**Phase-0 is CLOSED — measured 2026-08-04/05 (#279, ADR-0033).** Endpoint: `GET portal.registryagency.bg/CR/api/Deeds/{ЕИК}`, public JSON, no authentication. Facts worth recording because three of them contradict what the issue predicted: +- **The register rate-limits, and the block is sustained.** HTTP 429 at ~50 cumulative requests ending in a burst, then 429 to everything — no `Retry-After`, no quota header. So a 429 is an instruction to stop, never a transient to retry through: the crawler ends the run, marks nothing, and resumes later. The implemented pace is 1 request / 3 s, sequential, over a **closed** candidate set the crawler cannot extend (it never follows a link out of a deed). +- **„Not in the register" is an HTTP 200 with a ZERO-BYTE body**, not a 404 and not HTML. Verified on Община София (`000696327`): empty twice while a real company returned its full deed in the same window. Permanence therefore keys on the **status**, not the empty body — empty under 200 is an answer and may be cached; empty under 5xx is a failure and stays transient. +- **`fetch` does not work against this host.** The identical request returns 500/empty via undici and 200 with the full deed via `node:https`. +- Field values are HTML fragments inside the JSON envelope, and one field routinely holds several people separated by `
`; erasure is marked structurally (`erasure-text-inline`), not textually. Matching must happen **inside one entity**, or one person's given name combines with another's surname. +- **ЕГН is absent** from every payload examined, and is never stored either way. Raw deeds live only under git-ignored `scratch/tr/` with a 35-day retention; the cache index holds no name at all. + ## 4. Data model New domain tables, built by `normalize-raw.sql` from persistent staging; `interest_links` written by a **JS resolver pass** (§5), NOT by fuzzy SQL. @@ -182,6 +189,7 @@ The public site MUST carry a plain-language methodology page. It is part of the 1. **Sources + legal basis** — CACBG declarations (ЗСП чл. 75, Сметна палата); ЕРИК (Изборен кодекс); targeted TR per-ЕИК (public company facts). Each with a direct source link. 2. **What is shown and what is not** — officials' own declared holdings; **family holdings are NOT published** (v1); addresses/ЕГН never shown. 3. **The matching rule, verbatim** — declared company **full name** (incl. legal form) is normalized and matched **exactly** against the winner's registered name, which is paired 1:1 with its ЕИК in the procurement source; because Bulgarian trade names are **nationally unique** (ЗТРРЮЛНЦ/ТЗ), an exact match is the same legal entity. Ambiguous cases (truncated names, no exact match) are **excluded or human-confirmed, never guessed**. Every link shows its full evidence chain. + **Superseded in part by #279 / ADR-0033:** a name match is no longer *sufficient* to publish. Every link additionally rests on a checkable **Търговски регистър** fact, and the methodology page carries the six-rung evidence ladder verbatim, together with the disclosures that make it honest: the register carries **no ЕГН**, so a three-name match is not proof of identity and a namesake is possible; the match must therefore be a **full three-token subset inside a single registry entity**; seats move, so a registered seat only confirms a period it predates; erased entries are skipped; and every published link shows its **entry number and lookup date**. 4. **Certainty & framing** — only deterministic exact matches (and human-confirmed cases) appear; each is a „**модел за проверка, не обвинение**"; the tool shows *declared* links only and **does not claim to find hidden ownership** (it is a floor, not the full picture). 5. **Temporal meaning** — a declaration is a point-in-time snapshot; how overlap with a contract date is (and isn't) interpreted. 6. **Known gaps** — the recall holes (§5), self-reporting limits, sources it cannot see. diff --git a/docs/spec/related-persons-lia.md b/docs/spec/related-persons-lia.md index 1309c42a3..075239405 100644 --- a/docs/spec/related-persons-lia.md +++ b/docs/spec/related-persons-lia.md @@ -128,6 +128,40 @@ v Poland* (2428/05) приема онлайн публикуване на дек регистър. Приемаме този оспорим елемент съзнателно. **Резервен план при оспорване:** един филтър (сваляне на семейните връзки към еднолични дружества), не спиране на повърхността. +## 9а. Допълнение: справки в Търговския регистър (#279, ADR-0033) + +Нов **източник** и нова **обработка**, затова оценката се допълва, а не само се препотвърждава. + +**Каква е обработката.** За всяко дружество-кандидат правим една справка в публичния Търговски регистър и +четем живия акт. Актът съдържа **лични данни на трети лица** — имена на съдружници, собственици и +управители, които не заемат публична длъжност, както и адрес на дружеството. + +**Цел и основание.** Целта е тясна и проверима: да се установи **самоличността на дружеството** зад +декларирано име, преди срещу конкретно лице да бъде публикувано твърдение. Това е мярка, която *намалява* +риска за субектите на данни — досега самоличността се решаваше по евристика за отличителност на името, а +сега — по документ. Основанието е същият надделяващ обществен интерес по ЗДОИ чл. 41и, ал. 4, приложен към +по-точна, а не към по-широка публикация. + +**Минимизиране — какво НЕ правим.** +- Имената от акта се ползват **само в булеви сравнения** и не влизат в никаква публична таблица, отговор или + лог. Индексът на кеша не съдържа нито едно име — само ЕИК, дати, кодове и хеш на тялото. +- Адресът **не се извлича**: от полето за седалище се чете единствено сегментът „Населено място". +- Полетата за **действителен собственик** (`CR_F_550_L`) и за контрол (`537/538`) **не се ползват** — това + би било претенция за действителна собственост, оставена настрана след CJEU C-37/20 (ADR-0007, т. 1). +- Суровите актове се пазят само служебно, в git-игнорирана директория, със **срок 35 дни**. +- Обхватът е **затворен**: справка се прави само за ЕИК, които вече са резултат от съпоставянето; обхождащият + никога не следва връзка навън от акт. Това не е масово изтегляне (спец. §3.3). + +**Остатъчен риск — съименник.** Регистърът **не съдържа ЕГН**, затова съвпадението по трите имена е силно, но +не абсолютно доказателство за самоличност. Смекчаване: изисква се пълно съвпадение и на трите имена в **един и +същ запис** на акта; имена с инициали или с латински букви не се приемат; фамилни съвпадения не се ползват +никъде. Грешката тук не измисля твърдение за собственост — тя **закача длъжностното лице за друго дружество**, +което е също толкова невярно и се сваля по реда на ADR-0031 (виж runbook-а: „съвпадение с грешно лице"). + +**Отложено съзнателно.** Етикетите „и към днешна дата" по т. 7 на #279 **не се пускат** в тази фаза: те +твърдят сегашно време за поименно лице, а давността им е ограничена от цикъла на опресняване. Влизат само +след отделен преглед на настоящата оценка. + ## 10. Статус и преглед Draft за правен преглед; изпраща се паралелно с пускането, не като блокер (ADR-0032, т. 7). Ревизира се при промяна diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index 84009da8b..75f100cbf 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -710,6 +710,15 @@ export interface ConflictLink { firstContractYear: string | null; lastContractYear: string | null; sourceUrl: string | null; // a representative declaration URL — provenance, never a fabricated value + // Trade Register evidence (#279, ADR-0033). A link only reaches this DTO when its identity rests on a + // checkable registry fact, so these describe WHICH fact — the surface's whole point is that every shown + // link can explain itself. `registryRole` is the role the register records, NOT a claim about who owns + // what: the ownership claim comes from the official's own declaration. + evidenceKind: 'document' | 'confirmed'; // the only two rungs that publish + registryRole: 'owner' | 'manager' | null; // set only for evidenceKind='document' + registryEntryNumber: string | null; // TEXT — a fieldEntryNumber exceeds the exact-integer range + registryEntryDate: string | null; // the registry entry the evidence rests on + registryLookupDate: string; // when the deed was read — the freshness bound on the claim } /** One contract of a linked winner, marked by whether it was signed during the declared-stake window. diff --git a/packages/db/migrations/0009_interest_link_evidence.sql b/packages/db/migrations/0009_interest_link_evidence.sql new file mode 100644 index 000000000..e91869d1e --- /dev/null +++ b/packages/db/migrations/0009_interest_link_evidence.sql @@ -0,0 +1,50 @@ +-- Trade Register evidence seal for a свързани-лица link (#279 §8, ADR-0033). +-- +-- WHY A SIDE TABLE and not columns on interest_links, since both were on the table: +-- 1. SQLite's ALTER TABLE ... ADD COLUMN has no IF NOT EXISTS, and migrations here are applied by a +-- bare `wrangler d1 execute --file` with no applied-migrations tracking (see the workflow steps), +-- so a re-apply MUST be a no-op. `CREATE TABLE IF NOT EXISTS` is; ADD COLUMN is not. +-- 2. scripts/cacbg/load.mjs rebuilds the CACBG tables from 0003 ALONE. A column added here would have +-- to be duplicated into 0003 and kept in step with it forever, and the day the two diverge the +-- loader silently drops the evidence for every link it writes. +-- 3. §8 describes the seal as an attached artefact of a link, which is what this is. +-- +-- A seal is written for EVERY link, not only published ones. The seals on held and withdrawn links are +-- what make the review queue reviewable — without them „why is this one hidden?" has no answer. +-- +-- PII rail: `matched_fact` is a CLOSED VOCABULARY — 'seat:' | 'role:owner:' | +-- 'role:manager:' | 'eik'. It must NEVER carry the matched name. The registry deed's names are +-- read only to produce a boolean and never leave git-ignored scratch (ADR-0033 decision 5); storing one +-- here would put third-party personal data on the served surface, which #279 §9 forbids outright. The +-- audit enforces the vocabulary with a pattern check, because a schema cannot. + +CREATE TABLE IF NOT EXISTS interest_link_evidence ( + link_key TEXT PRIMARY KEY REFERENCES interest_links(link_key), + -- ADR-0033 decision 1, plus document_uncorroborated (ADR-0035: a name match whose COMPANY nothing + -- corroborated). CHECKed because the read gate filters on this column — SURFACED_OWNERSHIP admits + -- exactly 'document' and 'confirmed' — so an unlisted value is either a link that silently stops + -- surfacing or, if it collides with a publishing name, one that surfaces unproven. + evidence_kind TEXT NOT NULL + CHECK (evidence_kind IN ('document','confirmed','document_uncorroborated','refuted', + 'bar_joint_stock','unknown','outside_tr')), + -- owner | manager | NULL — only meaningful for evidence_kind='document'. The card renders this as + -- „вписан като …", so a stray value becomes a public claim about a named person's registry role. + registry_role TEXT CHECK (registry_role IS NULL OR registry_role IN ('owner','manager')), + matched_fact TEXT, -- the closed vocabulary above. NEVER a name. + -- TEXT, not INTEGER: a fieldEntryNumber like 20130716101007 is already 14 digits and exceeds the + -- exact-integer range once it round-trips through JSON/JS. + entry_number TEXT, + entry_date TEXT, -- ISO date of the registry entry the evidence rests on + lookup_date TEXT NOT NULL, -- when the deed was fetched — the freshness bound on the claim + rules_version TEXT NOT NULL, -- evidence.mjs RULES_VERSION; §8's monotonicity gate keys on this + -- live | terminated_owner_still | terminated_manager_still | terminated. + -- RE-DERIVED on every run and deliberately NOT part of the seal's permanence: it asserts a present + -- tense about a named person, and its freshness is bounded by the cache refresh cycle (ADR-0033 R3). + live_status TEXT NOT NULL + CHECK (live_status IN ('live','terminated_owner_still','terminated_manager_still', + 'terminated')), + sealed_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- The surface filters on evidence_kind ('document' | 'confirmed' publish); the audit scans by kind too. +CREATE INDEX IF NOT EXISTS idx_ile_kind ON interest_link_evidence(evidence_kind); diff --git a/packages/db/migrations/0010_publishing_gate_constraints.sql b/packages/db/migrations/0010_publishing_gate_constraints.sql new file mode 100644 index 000000000..b3e9d4cf0 --- /dev/null +++ b/packages/db/migrations/0010_publishing_gate_constraints.sql @@ -0,0 +1,123 @@ +-- The publishing-gate constraints, for EVERY database — freshly built and already-deployed alike (#279 §2). +-- +-- WHY THIS MIGRATION OWNS THEM RATHER THAN 0003 (todorkolev, #309). The obvious move is to add the CHECKs +-- to `interest_links` in 0003, where the table is declared. That does not work and is worse than not +-- trying: 0003 is ALREADY APPLIED everywhere, `CREATE TABLE IF NOT EXISTS` is a no-op against an existing +-- table, and `ship-related-persons.mjs` wipes ROWS (`DELETE FROM`, WIPE_ORDER), never definitions. So an +-- in-place CHECK would exist only on databases built after the edit — putting two different schemas under +-- one name, with the constraint absent precisely on the served database. That is where a hand-run +-- `UPDATE … SET status='published '` during an incident actually lands. +-- +-- So 0003 stays byte-identical to what was applied, and enforcement lives here, reached by both paths: +-- a fresh chain runs 0007 after 0003, and a deployed database gets it as a retrofit. One shape, one +-- mechanism. `packages/db/src/migrations.test.ts` holds the two shapes to the same rejections and the same +-- acceptances, so this stays true rather than merely intended. +-- +-- WHY TRIGGERS AND NOT A TABLE REBUILD: SQLite cannot add a CHECK in place, so the textbook route is +-- create-copy-drop-rename. That route is WRONG here, and a real `wrangler d1 migrations apply` proved it: +-- `interest_link_evidence` references interest_links(link_key), D1 enforces foreign keys, and +-- `PRAGMA defer_foreign_keys` does not survive the statement-by-statement execution wrangler performs — +-- the rebuild aborts with SQLITE_CONSTRAINT_FOREIGNKEY and the Durable Object rolls back. Dropping the +-- child first would work mechanically but would strip every evidence seal, and since the read gate +-- REQUIRES a seal that empties the public surface until the next monthly data run. +-- +-- A BEFORE INSERT/UPDATE trigger that RAISEs enforces the identical invariant for every writer — including +-- the hand-run UPDATE above, which no CHECK on a legacy table would ever have covered — with no rebuild, +-- no FK exposure and no seal loss. 0006 keeps its own CHECKs: it is NEW in this change, has never been +-- applied to a served environment, so declaring them there edits no applied history. +-- +-- WHY IT IS SAFE TO RE-RUN: migrations here are applied by a bare `wrangler d1 execute --file` with no +-- applied-migrations tracking (see related-persons-data.yml / deploy.yml), so re-application MUST be a +-- no-op. Everything below is `IF NOT EXISTS` over statements that converge. + +-- ── declarations ──────────────────────────────────────────────────────────────────────────────────── +-- The table-level `UNIQUE (xml_file, control_hash)` did not constrain what it was written for. SQLite +-- counts NULLs as DISTINCT and `control_hash` is genuinely optional at the source (the register omits +-- on some declarations), so every hashless declaration was mutually unique and re-imported +-- as a NEW row on each run, double-counting the stakes it carries. `xml_file` is also not unique across +-- folders — the register reuses basenames per year — which load.mjs already handles by namespacing the +-- declaration id with the folder. +-- +-- Replaced by an expression index over COALESCE(control_hash, '') plus folder_year: NULL-proof, and the +-- same natural key `id` already encodes. The column stays NULLABLE on purpose — NOT NULL would convert an +-- optional source field into a run-stopping loader failure, and a fabricated placeholder hash would +-- assert an integrity check nobody performed. +-- +-- No table rebuild is needed for this one: the table-level `UNIQUE (xml_file, control_hash)` declared in +-- 0003 STAYS (0003 is not edited — see the header), and the correct index is simply added alongside it. +-- The stale constraint is strictly WEAKER than the index — it treats NULLs as distinct, so it accepts a +-- superset — and therefore rejects nothing the index accepts. The index governs. +-- +-- Duplicates ALREADY stored under the old non-constraint would block the index, so they are collapsed +-- first, keeping the earliest row per natural key. +-- +-- The DELETE below changes live data, so it ANNOUNCES itself first (cefothe, #309): the count is emitted +-- before the rows go, and a run that removes nothing says so. Without it the only evidence a deployment +-- silently dropped declarations would be a row count nobody recorded beforehand. It needs no explicit +-- transaction — `wrangler d1 execute --file` runs the file as one implicit transaction, so a failure +-- anywhere below rolls the DELETE back with it rather than leaving a half-collapsed table. +-- The subtraction is parenthesised deliberately: `||` binds TIGHTER than `-` in SQLite, so without it +-- this reads as ('…' || countA) - (countB || '…') — two strings coerced to numbers — and reports a +-- meaningless figure instead of the row count. It did, before a real apply showed „notice: -2". +SELECT 'migration 0007: collapsing ' || + ((SELECT COUNT(*) FROM declarations) - + (SELECT COUNT(*) FROM (SELECT 1 FROM declarations + GROUP BY xml_file, folder_year, COALESCE(control_hash, '')))) || + ' duplicate declaration row(s) before the natural-key index' AS notice; +DELETE FROM declarations WHERE id NOT IN ( + SELECT MIN(id) FROM declarations GROUP BY xml_file, folder_year, COALESCE(control_hash, '') +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_declarations_natural_key + ON declarations(xml_file, folder_year, COALESCE(control_hash, '')); + +-- ── interest_links: status / interest_class ───────────────────────────────────────────────────────── +-- These two ARE the publishing gate: the surface is `status = 'published'` AND a surfaced +-- `interest_class`. A value that merely resembles one — 'published ' with a trailing space, the case +-- #279 §2 names — passes every writer and then fails the gate silently, hiding a link that should show. +-- +-- One trigger per operation, since SQLite triggers are per-event. `RAISE(ABORT)` rolls back the +-- statement, so a bad write fails loudly at its source instead of surfacing later as a missing link. +DROP TRIGGER IF EXISTS trg_interest_links_status_ins; +CREATE TRIGGER trg_interest_links_status_ins +BEFORE INSERT ON interest_links +WHEN NEW.status NOT IN ('published','internal','held','withdrawn','suppressed') + OR NEW.interest_class NOT IN ('private_ownership','family_ownership','ex_officio_board','management_role') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_links.status/interest_class outside the publishing-gate enum'); +END; + +DROP TRIGGER IF EXISTS trg_interest_links_status_upd; +CREATE TRIGGER trg_interest_links_status_upd +BEFORE UPDATE ON interest_links +WHEN NEW.status NOT IN ('published','internal','held','withdrawn','suppressed') + OR NEW.interest_class NOT IN ('private_ownership','family_ownership','ex_officio_board','management_role') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_links.status/interest_class outside the publishing-gate enum'); +END; + +-- ── interest_link_evidence: evidence_kind ─────────────────────────────────────────────────────────── +-- The read gate filters on this column — SURFACED_OWNERSHIP admits exactly 'document' and 'confirmed' — +-- so an unlisted value either silently stops a link surfacing or, if it collides with a publishing name, +-- surfaces one that was never proven. 0006 declares this as a CHECK for new databases; the trigger is +-- the same rule for those provisioned before it. +DROP TRIGGER IF EXISTS trg_ile_kind_ins; +CREATE TRIGGER trg_ile_kind_ins +BEFORE INSERT ON interest_link_evidence +WHEN NEW.evidence_kind NOT IN ('document','confirmed','document_uncorroborated','refuted', + 'bar_joint_stock','unknown','outside_tr') + OR (NEW.registry_role IS NOT NULL AND NEW.registry_role NOT IN ('owner','manager')) + OR NEW.live_status NOT IN ('live','terminated_owner_still','terminated_manager_still','terminated') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_link_evidence enum outside the ADR-0033/0035 vocabulary'); +END; + +DROP TRIGGER IF EXISTS trg_ile_kind_upd; +CREATE TRIGGER trg_ile_kind_upd +BEFORE UPDATE ON interest_link_evidence +WHEN NEW.evidence_kind NOT IN ('document','confirmed','document_uncorroborated','refuted', + 'bar_joint_stock','unknown','outside_tr') + OR (NEW.registry_role IS NOT NULL AND NEW.registry_role NOT IN ('owner','manager')) + OR NEW.live_status NOT IN ('live','terminated_owner_still','terminated_manager_still','terminated') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_link_evidence enum outside the ADR-0033/0035 vocabulary'); +END; diff --git a/packages/db/src/amendments-slice-resolve.test.ts b/packages/db/src/amendments-slice-resolve.test.ts index 931dc4497..93dee1db5 100644 --- a/packages/db/src/amendments-slice-resolve.test.ts +++ b/packages/db/src/amendments-slice-resolve.test.ts @@ -27,6 +27,8 @@ const migrations = [ 'packages/db/migrations/0006_amendment_restated.sql', 'packages/db/migrations/0007_amendment_value_suspect.sql', 'packages/db/migrations/0008_amendment_provenance.sql', + // #279/ADR-0033: refresh-slice.sql reads interest_link_evidence. + 'packages/db/migrations/0009_interest_link_evidence.sql', ].map((p) => resolve(root, p)); const workStagingSchema = resolve(root, 'scripts/work-staging-schema.sql'); const refreshSlice = resolve(root, 'scripts/refresh-slice.sql'); diff --git a/packages/db/src/amendments-sql.test.ts b/packages/db/src/amendments-sql.test.ts index e4f56dfaa..9e54acddd 100644 --- a/packages/db/src/amendments-sql.test.ts +++ b/packages/db/src/amendments-sql.test.ts @@ -22,6 +22,8 @@ const migration6 = resolve(root, 'packages/db/migrations/0006_amendment_restated const migration7 = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); // #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. const migration8 = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +// #279/ADR-0033: refresh-slice.sql + normalize-raw.sql read interest_link_evidence, so 0009 must be applied too. +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); @@ -74,6 +76,7 @@ function withDb(fn: (dbPath: string) => T): T { readScript(dbPath, migration6); readScript(dbPath, migration7); readScript(dbPath, migration8); + readScript(dbPath, migration9); return fn(dbPath); } finally { rmSync(dir, { recursive: true, force: true }); diff --git a/packages/db/src/amendments-total-restated.test.ts b/packages/db/src/amendments-total-restated.test.ts index 2eee4fade..1212ea334 100644 --- a/packages/db/src/amendments-total-restated.test.ts +++ b/packages/db/src/amendments-total-restated.test.ts @@ -27,6 +27,8 @@ const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_rest const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); // #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +// #279/ADR-0033: refresh-slice.sql + normalize-raw.sql read interest_link_evidence, so 0009 must be applied too. +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); @@ -66,6 +68,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); + readScript(dbPath, migration9Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/amendments-total-suspect.test.ts b/packages/db/src/amendments-total-suspect.test.ts index e34fdbab0..9b9b5f34b 100644 --- a/packages/db/src/amendments-total-suspect.test.ts +++ b/packages/db/src/amendments-total-suspect.test.ts @@ -28,6 +28,8 @@ const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_rest const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); // #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +// #279/ADR-0033: refresh-slice.sql + normalize-raw.sql read interest_link_evidence, so 0009 must be applied too. +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const derivePath = resolve(root, 'scripts/derive-amendments.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); @@ -71,6 +73,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); + readScript(dbPath, migration9Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index b46d01bbe..e0fda0213 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -16,6 +16,11 @@ const migration3 = readFileSync( resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'), 'utf8', ); +// …and 0009, which those same blocks now join for the Trade Register evidence gate (#279, ADR-0033). +const migration9 = readFileSync( + resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'), + 'utf8', +); // #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). const migration6 = readFileSync( resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'), @@ -81,7 +86,7 @@ function build(path: 'normalize' | 'refresh'): DatabaseSync { const db = new DatabaseSync(':memory:'); db.exec(schema); db.exec(migration2); - db.exec(migration3); + db.exec(migration3 + migration9); db.exec(migration6); db.exec(migration7); db.exec(migration8); diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index 53012248c..574c8cec4 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -11,6 +11,8 @@ const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // refresh-slice.sql's officials block reads interest_links (0003); build it so the script doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); // #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); @@ -45,6 +47,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index 82d28de48..f6d9c7ff0 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -29,6 +29,8 @@ const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bid const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // precompute.sql's officials block reads interest_links (0003); build it so precompute doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const precomputePath = resolve(root, 'scripts/precompute.sql'); function sqlite(dbPath: string, sql: string): void { @@ -72,6 +74,7 @@ function freshDb(): string { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); sqlite(dbPath, CLEAN_FIXTURE); return dbPath; } diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index e5296aaa9..a37050b2e 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -11,6 +11,8 @@ const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); const migration2 = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration10 = resolve(root, 'packages/db/migrations/0010_publishing_gate_constraints.sql'); const backfill = resolve(root, 'scripts/backfill-current-value-currency.sql'); const precompute = resolve(root, 'scripts/precompute.sql'); @@ -131,6 +133,7 @@ describe('served migrations', () => { ); readScript(dbPath, migration2); readScript(dbPath, migration3); + readScript(dbPath, migration9); readScript(dbPath, backfill); readScript(dbPath, precompute); @@ -150,4 +153,350 @@ describe('served migrations', () => { rmSync(dir, { recursive: true, force: true }); } }); + + // 0006 attaches the Trade Register evidence seal to a link (#279, ADR-0033). A SIDE TABLE rather than + // columns on interest_links, for two reasons that are easy to forget: SQLite's ADD COLUMN has no + // IF NOT EXISTS and migrations are applied by a bare `d1 execute --file` with no tracking, so a + // re-apply must be a no-op; and load.mjs rebuilds the CACBG tables from 0003 alone, so any column + // added here would have to be duplicated into 0003 and kept in step forever. + it('0006 adds the evidence seal and re-applying it is a no-op', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-0006-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + for (const m of [migration0, migration1, migration2, migration3, migration9]) + readScript(dbPath, m); + + expect( + sqlite( + dbPath, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='interest_link_evidence';", + ).trim(), + ).toBe('1'); + + // A seal for a real link survives a second apply — the migration must not drop and recreate. + sqlite( + dbPath, + `INSERT INTO persons (id, name) VALUES ('person:a', 'Иван Петров Тестов'); + INSERT INTO bidders (id, name, eik_normalized, eik_valid) + VALUES ('eik:201122335', 'АЛФА СТРОЙ ООД', '201122335', 1); + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:k', 'k', 'person:a', 'eik:201122335', '201122335', 'АЛФА СТРОЙ ООД', + 'test', 'document', 'owns', 'private_ownership', 'published'); + INSERT INTO interest_link_evidence + (link_key, evidence_kind, matched_fact, lookup_date, rules_version, live_status) + VALUES ('k', 'document', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live');`, + ); + readScript(dbPath, migration9); // idempotent re-apply + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_link_evidence;').trim()).toBe('1'); + + // The FK is real: a seal for a link that does not exist is rejected. D1 enforces foreign keys, + // so this is what stops the ship path inserting seals before (or after wiping) their links. + expect(() => + sqlite( + dbPath, + `PRAGMA foreign_keys=ON; + INSERT INTO interest_link_evidence + (link_key, evidence_kind, matched_fact, lookup_date, rules_version, live_status) + VALUES ('nope', 'document', 'eik', '2026-08-05', 'tr-rules-1', 'live');`, + ), + ).toThrow(/FOREIGN KEY/i); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + // #279 §2: the publishing gate reads these columns as enums, but the schema let them hold anything. + // A value that merely LOOKS like a gate value — the trailing space in 'published ' §2 names — passes + // every writer and then silently fails `status = 'published'` (or worse, passes a LIKE somewhere). + // A CHECK is the only place this can be enforced for every writer at once, including a hand-run UPDATE. + it('constrains the publishing-gate enums and the declaration identity key', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + for (const m of [migration0, migration3, migration9, migration10]) readScript(dbPath, m); + + const seedLink = (status: string) => + sqlite( + dbPath, + `INSERT INTO persons(id, name) VALUES ('person:a', 'А') ON CONFLICT DO NOTHING; + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:${status}', '${status}', 'person:a', 'eik:201122335', '201122335', + 'АЛФА СТРОЙ ООД', 'test', 'document', 'owns', 'private_ownership', '${status}');`, + ); + + // §2's named case: 'published ' is not 'published'. It must be rejected, not stored. + expect(() => seedLink('published ')).toThrow(/CHECK/i); + expect(() => seedLink('publushed')).toThrow(/CHECK/i); + // POSITIVE CONTROL — every real status still inserts. A CHECK that rejected everything would pass + // the assertions above while breaking the loader. + for (const s of ['published', 'held', 'withdrawn', 'suppressed', 'internal']) + expect(() => seedLink(s)).not.toThrow(); + + // interest_class gates the surface just as hard: a non-surfaced class is what keeps a management + // role off the public page, so a typo'd class is a leak in the same way. + expect(() => + sqlite( + dbPath, + `INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:c', 'c', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership ', 'held');`, + ), + ).toThrow(/CHECK/i); + + // evidence_kind decides whether a link may be read at all (SURFACED_OWNERSHIP). ADR-0035's + // document_uncorroborated must be a legal value — and a misspelling must not be. + const seal = (kind: string) => + sqlite( + dbPath, + `INSERT INTO interest_link_evidence + (link_key, evidence_kind, lookup_date, rules_version, live_status) + VALUES ('${kind}', '${kind}', '2026-08-12', 'tr-rules-1', 'live');`, + ); + expect(() => seal('document_uncorroberated')).toThrow(/CHECK|FOREIGN KEY/i); + for (const k of ['document', 'confirmed', 'document_uncorroborated', 'refuted']) + expect(() => { + sqlite( + dbPath, + `INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:${k}', '${k}', 'person:a', 'eik:1', '1', 'X', 't', '${k}', 'owns', + 'private_ownership', 'held'); + INSERT INTO interest_link_evidence + (link_key, evidence_kind, lookup_date, rules_version, live_status) + VALUES ('${k}', '${k}', '2026-08-12', 'tr-rules-1', 'live');`, + ); + }).not.toThrow(); + + // §2: `UNIQUE (xml_file, control_hash)` did not constrain re-import. SQLite counts NULLs as + // DISTINCT and control_hash is genuinely optional at the source, so two hashless imports of the + // SAME declaration both inserted and double-counted the stakes it carries. + const decl = (id: string, hash: string | null, folder = '2023') => + sqlite( + dbPath, + `INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('${id}', 'person:a', 'A.xml', ${hash === null ? 'NULL' : `'${hash}'`}, + '${folder}', 'assets', 'https://x/A.xml');`, + ); + expect(() => decl('d:n1', null)).not.toThrow(); + expect(() => decl('d:n2', null)).toThrow(/UNIQUE/i); // the case that used to slip through + expect(() => decl('d:1', 'H1')).not.toThrow(); + expect(() => decl('d:2', 'H1')).toThrow(/UNIQUE/i); + // POSITIVE CONTROLS — the key must not over-constrain. The register reuses basenames across + // FOLDERS, so the same xml_file in a different folder is a DIFFERENT declaration (load.mjs + // namespaces its id for this reason), and a genuinely different hash is a corrected re-filing. + expect(() => decl('d:f2', null, '2024')).not.toThrow(); + expect(() => decl('d:3', 'H2')).not.toThrow(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + // THE POINT OF 0007, stated as an executable claim rather than a comment (todorkolev, #309). + // + // 0003 is an ALREADY-APPLIED migration and is therefore never edited: `CREATE TABLE IF NOT EXISTS` is a + // no-op on a live database, so an in-place CHECK would exist only on freshly built ones. That would put + // two different schemas under one name — and the divergence would land precisely on the served database, + // which is where a hand-run `UPDATE status='published '` during an incident actually happens. + // + // So 0007 owns the enforcement for BOTH shapes, and this test is what keeps them one shape: a fresh + // 0000..0007 build and a legacy database retrofitted by 0007 must reject the SAME values and accept the + // SAME values. If either drifts, the parity assertions below fail rather than a reviewer noticing. + it('a fresh build and a retrofitted legacy database enforce the SAME gate', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-parity-')); + const fresh = resolve(dir, 'fresh.sqlite'); + const legacy = resolve(dir, 'legacy.sqlite'); + try { + // (1) fresh: the migration chain exactly as a new environment applies it. + for (const m of [migration0, migration3, migration9, migration10]) readScript(fresh, m); + + // (2) legacy: 0003 + 0006 as they were BEFORE this PR, then 0007 retrofits. `interest_links` is + // recreated without constraints because that is the shape every already-deployed database holds. + for (const m of [migration0, migration3, migration9]) readScript(legacy, m); + sqlite( + legacy, + `DROP TABLE interest_link_evidence; + DROP TABLE interest_links; + CREATE TABLE interest_links ( + id TEXT PRIMARY KEY, link_key TEXT NOT NULL UNIQUE, + person_id TEXT NOT NULL REFERENCES persons(id), bidder_id TEXT NOT NULL, eik TEXT NOT NULL, + entity_key TEXT NOT NULL, match_method TEXT, matcher_version TEXT NOT NULL, + publish_tier TEXT NOT NULL, relation TEXT NOT NULL, + interest_class TEXT NOT NULL DEFAULT 'management_role', + contemporaneous INTEGER NOT NULL DEFAULT 0, own_institution TEXT NOT NULL DEFAULT 'none', + evidence_count INTEGER NOT NULL DEFAULT 1, first_declared_year TEXT, last_declared_year TEXT, + contract_count INTEGER NOT NULL DEFAULT 0, contract_value_eur REAL, first_contract_year TEXT, + last_contract_year TEXT, status TEXT NOT NULL DEFAULT 'held', verified_by TEXT, + verified_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now'))); + CREATE TABLE interest_link_evidence ( + link_key TEXT PRIMARY KEY REFERENCES interest_links(link_key), evidence_kind TEXT NOT NULL, + registry_role TEXT, matched_fact TEXT, entry_number TEXT, entry_date TEXT, + lookup_date TEXT NOT NULL, rules_version TEXT NOT NULL, live_status TEXT NOT NULL, + sealed_at TEXT NOT NULL DEFAULT (datetime('now')));`, + ); + readScript(legacy, migration10); + + const link = (db: string, id: string, status: string, cls = 'private_ownership') => + sqlite( + db, + `INSERT INTO persons(id, name) VALUES ('person:a', 'А') ON CONFLICT DO NOTHING; + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:${id}', '${id}', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + '${cls}', '${status}');`, + ); + const seal = (db: string, key: string, kind: string) => + sqlite( + db, + `INSERT INTO interest_link_evidence + (link_key, evidence_kind, lookup_date, rules_version, live_status) + VALUES ('${key}', '${kind}', '2026-08-14', 'tr-rules-1', 'live');`, + ); + + for (const [label, db] of [ + ['fresh', fresh], + ['legacy', legacy], + ] as const) { + // REJECTED identically — 'published ' is #279 §2's named case, and it is the one a human types. + expect(() => link(db, `bad1-${label}`, 'published ')).toThrow(/CHECK/i); + expect(() => link(db, `bad2-${label}`, 'publushed')).toThrow(/CHECK/i); + expect(() => link(db, `bad3-${label}`, 'held', 'private_ownership ')).toThrow(/CHECK/i); + // ACCEPTED identically — the gate is a bound, not a blanket, on both shapes. + expect(() => link(db, `ok-${label}`, 'published')).not.toThrow(); + expect(() => seal(db, `ok-${label}`, 'document')).not.toThrow(); + expect(() => seal(db, `ok-${label}`, 'document_uncorroborated')).toThrow(/UNIQUE|CHECK/i); + // …and the evidence enum is enforced on both too. + expect(() => link(db, `k2-${label}`, 'held')).not.toThrow(); + expect(() => seal(db, `k2-${label}`, 'document_uncorroberated')).toThrow(/CHECK/i); + // The UPDATE path is the one an incident actually uses, and no CHECK would ever cover it on a + // legacy table — only the trigger does. + expect(() => + sqlite(db, `UPDATE interest_links SET status='published ' WHERE link_key='ok-${label}';`), + ).toThrow(/CHECK/i); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + // The retrofit has to be safe on a LIVE D1: ship-related-persons wipes rows, never table definitions, + // so a deployed database keeps its unconstrained 0003 shape until this migration rebuilds it. + it('0007 retrofits a pre-existing database, keeps its rows, and re-applies as a no-op', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + for (const m of [migration0, migration3, migration9]) readScript(dbPath, m); + // Simulate a database provisioned BEFORE today: 0003 now declares the constraints, so the legacy + // shape has to be recreated explicitly. This is the state every already-deployed environment is in + // — `CREATE TABLE IF NOT EXISTS` never revisited it and ship-related-persons only wipes rows. + sqlite( + dbPath, + `DROP TABLE declarations; + CREATE TABLE declarations ( + id TEXT PRIMARY KEY, person_id TEXT NOT NULL REFERENCES persons(id), xml_file TEXT NOT NULL, + control_hash TEXT, folder_year TEXT NOT NULL, declared_year TEXT, template TEXT NOT NULL, + category TEXT, institution TEXT, position TEXT, source_url TEXT NOT NULL, + UNIQUE (xml_file, control_hash)); + DROP TABLE interest_links; + CREATE TABLE interest_links ( + id TEXT PRIMARY KEY, link_key TEXT NOT NULL UNIQUE, + person_id TEXT NOT NULL REFERENCES persons(id), bidder_id TEXT NOT NULL, eik TEXT NOT NULL, + entity_key TEXT NOT NULL, match_method TEXT, matcher_version TEXT NOT NULL, + publish_tier TEXT NOT NULL, relation TEXT NOT NULL, + interest_class TEXT NOT NULL DEFAULT 'management_role', contemporaneous INTEGER NOT NULL DEFAULT 0, + own_institution TEXT NOT NULL DEFAULT 'none', evidence_count INTEGER NOT NULL DEFAULT 1, + first_declared_year TEXT, last_declared_year TEXT, contract_count INTEGER NOT NULL DEFAULT 0, + contract_value_eur REAL, first_contract_year TEXT, last_contract_year TEXT, + status TEXT NOT NULL DEFAULT 'held', verified_by TEXT, verified_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')));`, + ); + // The rows the retrofit must preserve — plus the two shapes it must not. + sqlite( + dbPath, + `INSERT INTO persons(id, name) VALUES ('person:a', 'А'); + INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('d:keep', 'person:a', 'K.xml', 'HK', '2023', 'assets', 'https://x/K.xml'); + INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('d:drop', 'person:a', 'D.xml', NULL, '2023', 'assets', 'https://x/D.xml'); + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:k', 'k', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership', 'published'); + -- The uninterpretable status §2 is about, already stored because nothing rejected it. + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:bad', 'bad', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership', 'published ');`, + ); + + readScript(dbPath, migration10); + // The valid rows survive — nothing is rebuilt, so nothing can be lost… + expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( + 'd:keep', + ); + expect(sqlite(dbPath, "SELECT link_key FROM interest_links WHERE link_key='k';").trim()).toBe( + 'k', + ); + // …including the pre-existing trailing-space row. A trigger constrains FUTURE writes; it cannot + // retroactively reject a row already stored, and deleting one would mean dropping a real link on a + // guess. The loader rewrites the table wholesale on the next run, which is what corrects it — and + // the read gate never showed it anyway, since 'published ' is not 'published'. + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('2'); + // The hashless row SURVIVES — it is a real declaration and NOT NULL was rejected for that reason — + // but it is now covered by the natural key, so re-importing it is refused rather than duplicated. + expect(sqlite(dbPath, "SELECT COUNT(*) FROM declarations WHERE id='d:drop';").trim()).toBe( + '1', + ); + expect(() => + sqlite( + dbPath, + `INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('d:dup', 'person:a', 'D.xml', NULL, '2023', 'assets', 'https://x/D.xml');`, + ), + ).toThrow(/UNIQUE/i); + // …and the constraint is in force for every writer afterwards, on INSERT and on UPDATE alike. + // UPDATE is the one that matters most: a hand-run status change during an incident is exactly when + // a stray character gets typed, and it is the path no application-level validation covers. + expect(() => + sqlite(dbPath, "UPDATE interest_links SET status='published ' WHERE link_key='k';"), + ).toThrow(/CHECK failed/i); + expect(() => + sqlite( + dbPath, + `INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:x', 'x', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership', 'publushed');`, + ), + ).toThrow(/CHECK failed/i); + // POSITIVE CONTROL: a legitimate status change still goes through — the trigger is a bound. + expect(() => + sqlite(dbPath, "UPDATE interest_links SET status='withdrawn' WHERE link_key='k';"), + ).not.toThrow(); + + // Migrations here are applied by a bare `d1 execute --file` with no applied-migrations tracking, + // so a second application MUST be a no-op rather than an error or a data loss. + readScript(dbPath, migration10); + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('2'); + expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( + 'd:keep', + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/packages/db/src/queries/related-persons.test.ts b/packages/db/src/queries/related-persons.test.ts index 519856517..8ba09404e 100644 --- a/packages/db/src/queries/related-persons.test.ts +++ b/packages/db/src/queries/related-persons.test.ts @@ -33,6 +33,13 @@ function row(over: Record = {}) { first_contract_year: '2021', last_contract_year: '2024', source_url: 'https://register.cacbg.bg/2024/i.xml', + // SURFACED_OWNERSHIP only returns rows carrying a publishing seal, so the fixture carries one too — + // a fixture without it would exercise a row shape the SQL cannot produce. + evidence_kind: 'document', + registry_role: 'owner', + entry_number: '20110502101007', + entry_date: '2011-05-02', + lookup_date: '2026-08-05', ...over, }; } @@ -197,3 +204,47 @@ describe('conflict reads soft-fail on an un-migrated env (no 500)', () => { } }); }); + +// The evidence seal is what licenses the card's registry sentence, and the two rungs assert very +// different things: 'document' renders „лицето е вписано като съдружник/собственик" — that the register +// names THIS person in THIS company — while 'confirmed' claims only that the declared data matched. +// The mapper used to read `kind === 'confirmed' ? 'confirmed' : 'document'`, so NULL, 'refuted', a rung +// added later, or a typo all became the STRONGEST claim we can make about a named human being. The SQL +// gate makes that unreachable today; the failure DIRECTION is still wrong, and this is the one place in +// the codebase where being wrong by default is libel rather than a rendering glitch. +describe('an unrecognised evidence seal withholds the link instead of upgrading it', () => { + for (const kind of ['refuted', 'unknown', 'bar_joint_stock', 'outside_tr', 'future_rung_v9']) + it(`'${kind}' never renders as a registry claim`, async () => { + const db = fakeDb({ '10': [row({ evidence_kind: kind })] }); + expect(await getConflictLeaderboard(db, 10)).toEqual([]); + }); + + it('a NULL seal — the half-migrated read the old comment invited — withholds too', async () => { + const db = fakeDb({ '10': [row({ evidence_kind: null })] }); + expect(await getConflictLeaderboard(db, 10)).toEqual([]); + }); + + it('both publishing rungs still map, and to DIFFERENT kinds — the guard bounds, it does not flatten', async () => { + for (const kind of ['document', 'confirmed']) { + const db = fakeDb({ '10': [row({ evidence_kind: kind })] }); + const links = await getConflictLeaderboard(db, 10); + expect(links).toHaveLength(1); + expect(links[0]!.evidenceKind).toBe(kind); + } + }); + + it('the official page 404s rather than render a page under a name with nothing left to show', async () => { + const db = fakeDb({ 'person:ivan': [row({ evidence_kind: 'refuted' })] }); + expect(await getOfficialConflicts(db, 'person:ivan')).toBeNull(); + expect( + await getCompanyConflicts(fakeDb({ '111': [row({ evidence_kind: null })] }), '111'), + ).toBeNull(); + }); + + it('one withheld row does not take its sealed siblings down with it', async () => { + const db = fakeDb({ + '10': [row({ link_key: 'ok|111' }), row({ link_key: 'bad|111', evidence_kind: 'refuted' })], + }); + expect((await getConflictLeaderboard(db, 10)).map((l) => l.linkKey)).toEqual(['ok|111']); + }); +}); diff --git a/packages/db/src/queries/related-persons.ts b/packages/db/src/queries/related-persons.ts index 05c3f228f..25f2a7524 100644 --- a/packages/db/src/queries/related-persons.ts +++ b/packages/db/src/queries/related-persons.ts @@ -19,6 +19,9 @@ const CONFLICT_TABLES = [ 'declared_interests', 'interest_link_authorities', 'related_persons_internal', + // 0006 (#279, ADR-0033). Listed here for the same reason as the rest: on an environment where 0006 + // has not been applied yet, the evidence join must degrade to an empty surface rather than a 500. + 'interest_link_evidence', ]; // „D1_ERROR: no such table: interest_links: SQLITE_ERROR" → capture the table name and test membership. const MISSING_TABLE = /no such table:\s*(?:main\.)?"?([a-z_]+)"?/i; @@ -79,11 +82,28 @@ interface LinkRow { first_contract_year: string | null; last_contract_year: string | null; source_url: string | null; + evidence_kind: string | null; + registry_role: string | null; + entry_number: string | null; + entry_date: string | null; + lookup_date: string | null; } // The winner's contracts, joined exactly as the ETL aggregate does (contracts→tenders→authorities→bidders, // matched by eik_normalized) so any read-time subset is a true subset of the stored contract_count/value. // Alias-distinct (cc/tt/aa/bb) so it composes as a correlated subquery under the LINK_SELECT `il`/`b` scope. +// +// `tt` and `aa` are NOT projected here, which makes both joins look dead and invites deleting them. They +// are not dead — they are the SHAPE, and its counterpart is the WRITER at scripts/cacbg/load.mjs (the +// per-winner contract query that fills contract_count / contract_value_eur). The two must stay identical. +// +// Removing them on the read side ALONE would widen the read to contracts the stored aggregate never +// counted: `contemporaneous_contract_count` could then exceed `contract_count`, and the EXISTS gate below +// (the one that decides whether a link surfaces at all) would publish links the I5 zero-contract gate had +// excluded. Removing them on BOTH sides is defensible — an unresolvable authority currently drops a +// contract from the money everywhere, consistently — but it changes published figures and so must be +// re-baselined against ADR-0033 §10's control totals, not slipped in as a cleanup. Tracked as #226 §1.6. +// `related-persons-sql.test.ts` pins this string to the writer's so neither can be "optimised" alone. const CONTRACT_JOIN = `FROM contracts cc JOIN tenders tt ON tt.id = cc.tender_id JOIN authorities aa ON aa.id = tt.authority_id @@ -116,8 +136,15 @@ export const NEXUS_ORDER = `(il.own_institution = 'exact') DESC, (contemporaneou // The two surfaced ownership classes (ADR-0032): the official's own stake (private_ownership) and a close // relative's (family_ownership). Anchored once here + in LINK_CONTRACTS_SQL so the read gate and the // drill-down never drift. +// …and the identity must rest on a Trade Register fact (#279, ADR-0033). `status='published'` already +// encodes the loader's decision, so this EXISTS is belt-and-braces: it makes the read path refuse a link +// whose seal is missing or whose evidence is a withholding rung, even if a future writer sets status +// wrongly. 'document' and 'confirmed' are the only two rungs that publish; bar_joint_stock, unknown, +// refuted and outside_tr never reach a reader. export const SURFACED_OWNERSHIP = `il.status = 'published' - AND il.interest_class IN ('private_ownership', 'family_ownership')`; + AND il.interest_class IN ('private_ownership', 'family_ownership') + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed'))`; // Redundant-family collapse (ADR-0032, per todorkolev review). A family link is DROPPED when the SAME official // already has a published OWN stake in the SAME winner. Rendering both a self row and a family row for one // (official, ЕИК) is a de-anonymisation vector: the office-holder is himself in that company's Търговски @@ -156,8 +183,13 @@ export const LINK_SELECT = `SELECT il.link_key, il.person_id, p.name AS official -- (name, institution), ADR-0026; same subquery the search projection uses). Correlated per row, but the -- leaderboard is ≤1000 rows and hourly-cached, so the extra scan is immaterial. (SELECT d.institution FROM declarations d WHERE d.person_id = il.person_id - ORDER BY d.declared_year DESC LIMIT 1) AS institution + ORDER BY d.declared_year DESC LIMIT 1) AS institution, + -- The evidence the link rests on, so the card can explain itself (ADR-0033 decision 7). LEFT JOIN + -- rather than an inner one: SURFACED_OWNERSHIP already requires a publishing seal, and an inner join + -- here would silently re-filter rather than surface a contradiction. + ev.evidence_kind, ev.registry_role, ev.entry_number, ev.entry_date, ev.lookup_date FROM interest_links il + LEFT JOIN interest_link_evidence ev ON ev.link_key = il.link_key JOIN persons p ON p.id = il.person_id JOIN bidders b ON b.id = il.bidder_id WHERE ${SURFACED_OWNERSHIP} @@ -170,6 +202,25 @@ export const LINK_SELECT = `SELECT il.link_key, il.person_id, p.name AS official -- …and drop a family link redundant with the official's own stake in the same winner (de-anon guard). AND ${NOT_REDUNDANT_FAMILY}`; +// The two rungs that license a public claim (ADR-0033 decision 1), and the ONLY two the card knows how +// to render. Anything else — 'refuted', 'unknown', 'bar_joint_stock', 'outside_tr', a rung added by a +// later rules_version, a typo, or NULL from a row with no seal — is withheld, never mapped. +// +// The mapper used to read `kind === 'confirmed' ? 'confirmed' : 'document'`, which turned every one of +// those into 'document' — the STRONGEST claim on the surface, rendering „лицето е вписано като +// съдружник/собственик": that the register names this specific person in this specific company. The SQL +// gate makes it unreachable today, but the direction was wrong, and this is the one mapping in the +// codebase where a default is a defamatory statement about a named human being rather than a glitch. +// The LEFT JOIN in LINK_SELECT is deliberately not an inner one so a contradiction SURFACES here; the +// old fallback then converted exactly that contradiction into the strongest possible label. +const PUBLISHING_EVIDENCE = new Set(['document', 'confirmed']); + +/** Rows whose seal licenses a public claim. Withholding is silent by design — the SQL already filters + * these out, so anything reaching here is a contradiction to drop, not a condition to report per row. */ +function sealed(rows: LinkRow[]): LinkRow[] { + return rows.filter((r) => PUBLISHING_EVIDENCE.has(String(r.evidence_kind))); +} + // own_institution is a 4-value verdict; only the deterministic 'exact' surfaces as true (the // name_contains/locality heuristics are disclosed elsewhere, never asserted as fact). function toLink(r: LinkRow): ConflictLink { @@ -193,6 +244,18 @@ function toLink(r: LinkRow): ConflictLink { firstContractYear: r.first_contract_year, lastContractYear: r.last_contract_year, sourceUrl: r.source_url, + // Narrowed, not defaulted — `sealed()` above has already dropped every other value, so this asserts + // what the filter guarantees instead of inventing a rung the row never carried. + evidenceKind: r.evidence_kind as 'document' | 'confirmed', + registryRole: + r.registry_role === 'owner' || r.registry_role === 'manager' ? r.registry_role : null, + registryEntryNumber: r.entry_number, + registryEntryDate: r.entry_date, + // Narrowed for the same reason as evidenceKind, not defaulted. `lookup_date` is NOT NULL in + // migration 0006 and the row only reaches here through the seal filter, so `?? ''` was a branch + // that could not run — and if the invariant ever broke it would have shipped an empty string as a + // date, which reads as a valid-but-blank provenance rather than as the contradiction it is. + registryLookupDate: r.lookup_date as string, }; } @@ -203,7 +266,7 @@ export const LEADERBOARD_SQL = `${LINK_SELECT} * relative's) in a procurement winner, ranked NEXUS-first (own-institution → contemporaneous → value). */ export async function getConflictLeaderboard(db: D1Database, limit = 100): Promise { try { - const rows = (await db.prepare(LEADERBOARD_SQL).bind(limit).all()).results; + const rows = sealed((await db.prepare(LEADERBOARD_SQL).bind(limit).all()).results); return rows.map(toLink); } catch (e) { if (conflictSchemaAbsent(e, 'leaderboard')) return []; // un-migrated env → empty surface, not a 500 @@ -221,7 +284,9 @@ export async function getOfficialConflicts( personId: string, ): Promise { try { - const rows = (await db.prepare(OFFICIAL_SQL).bind(personId).all()).results; + // Filtered BEFORE the emptiness check, so a person whose every link is withheld 404s rather than + // rendering an empty page under their name. + const rows = sealed((await db.prepare(OFFICIAL_SQL).bind(personId).all()).results); if (rows.length === 0) return null; const links = rows.map(toLink); return { official: links[0]!.official, links }; @@ -240,7 +305,7 @@ export async function getCompanyConflicts( eik: string, ): Promise { try { - const rows = (await db.prepare(COMPANY_SQL).bind(eik).all()).results; + const rows = sealed((await db.prepare(COMPANY_SQL).bind(eik).all()).results); if (rows.length === 0) return null; return { company: rows[0]!.company, eik, links: rows.map(toLink) }; } catch (e) { diff --git a/packages/db/src/queries/search.ts b/packages/db/src/queries/search.ts index 7a19f01e8..c3f3f02d0 100644 --- a/packages/db/src/queries/search.ts +++ b/packages/db/src/queries/search.ts @@ -112,15 +112,23 @@ interface HitRow { // One group's ranked hits. Company rows additionally carry a свързани-лица flag: a LEFT JOIN against the // published conflict links keyed on the winner's ЕИК (= the company row's `ident`). Published, self OR family -// stake, AND the winner must have LIVE contracts (the same read-time N9 gate LINK_SELECT applies), so search -// flags exactly the companies the /conflicts surface shows and never one whose page would 404. A family-only -// winner badges — the /conflicts page already publishes that link by name, so the badge discloses nothing the -// surface doesn't. Binds: kind, match, limit. +// stake, backed by a Trade Register evidence SEAL, AND the winner must have LIVE contracts (the same +// read-time N9 gate LINK_SELECT applies), so search flags exactly the companies the /conflicts surface shows +// and never one whose page would 404. A family-only winner badges — the /conflicts page already publishes +// that link by name, so the badge discloses nothing the surface doesn't. Binds: kind, match, limit. +// +// The seal EXISTS clause is the same one SURFACED_OWNERSHIP applies (related-persons.ts), and it belongs +// here for a sharper reason than symmetry: search is the WIDER surface. A published row whose seal is +// missing or withholding — a legacy row, a partial run, a loader bug — would badge a свързани-лица claim +// about a named official to every searcher, while the page behind the badge correctly showed nothing. +// Four copies of this predicate now exist (here, SURFACED_OWNERSHIP, precompute.sql, refresh-slice.sql); +// related-persons-sql.test.ts pins them to each other. // Exported so search-sql.test runs the EXACT SQL (not a copy). -// Built with or without the interest_links conflict join. The join is on the свързани-лица migration (0003); -// on an env where it has not been applied yet the join would make the ENTIRE search 500 with „no such table: -// interest_links". search() detects the table once per request and picks the no-conflict variant when absent, -// so search degrades to has_conflict=0 rather than breaking (ADR-0031 robustness ask). +// Built with or without the conflict join. The join reads BOTH свързани-лица migrations — interest_links +// (0003) and interest_link_evidence (0006) — and on an env where either is unapplied it would make the +// ENTIRE search 500 with „no such table". search() detects both tables once per request and picks the +// no-conflict variant when either is absent, so search degrades to has_conflict=0 rather than breaking +// (ADR-0031 robustness ask). const hitsSql = (withConflict: boolean): string => `SELECT search_index.ref, search_index.title, search_index.ident, search_index.subtitle, search_index.amount, rank, ct.kind AS entity_kind, ct.ownership_kind, ct.eik_valid, @@ -133,6 +141,8 @@ const hitsSql = (withConflict: boolean): string => `SELECT search_index.ref, sea ? `LEFT JOIN ( SELECT DISTINCT il.eik FROM interest_links il WHERE il.status = 'published' AND il.interest_class IN ('private_ownership', 'family_ownership') + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed')) AND EXISTS (SELECT 1 FROM contracts cc JOIN bidders bb ON bb.id = cc.bidder_id WHERE bb.eik_normalized = il.eik) ) cf ON search_index.kind = 'company' AND cf.eik = search_index.ident` @@ -161,12 +171,21 @@ export async function search(db: D1Database, rawQuery: string): Promise(); const counts = new Map(countRows.results.map((r) => [r.kind, r.n])); - // Pick the hits SQL once: on an env where the свързани-лица migration (0003) is not applied, the conflict - // join would 500 the whole search — fall back to the no-conflict variant (has_conflict=0) instead. + // Pick the hits SQL once: on an env where the свързани-лица migrations are not applied, the conflict join + // would 500 the whole search — fall back to the no-conflict variant (has_conflict=0) instead. + // + // BOTH tables are required, not just interest_links: the join now also reads interest_link_evidence + // (0006), so an env with 0003 but not 0006 would break every search on every kind. Requiring both also + // gives the right answer on such an env — with no seal table nothing is provably sealed, so nothing + // should badge. const hasConflictTable = - (await db - .prepare("SELECT 1 AS n FROM sqlite_master WHERE type='table' AND name='interest_links'") - .first<{ n: number }>()) != null; + ( + await db + .prepare( + "SELECT COUNT(*) AS n FROM sqlite_master WHERE type='table' AND name IN ('interest_links','interest_link_evidence')", + ) + .first<{ n: number }>() + )?.n === 2; const hitsSql = hasConflictTable ? SEARCH_HITS_SQL : SEARCH_HITS_SQL_NO_CONFLICT; const built = await Promise.all( diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 32ecd676d..3415a3b60 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -13,6 +13,8 @@ const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bid const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // refresh-slice.sql / precompute.sql officials block reads interest_links (0003) — build it in every chain. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); // #305 Tier-2: served amendments gained value_restated/value_treatment (refresh-slice promotes them). const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); // #305 residual: served amendments gained value_suspect (refresh-slice promotes it). @@ -191,6 +193,7 @@ function initWorkDb(dbPath: string): void { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); @@ -585,6 +588,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); @@ -670,6 +674,9 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + // 0006 too: refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279), so the + // script cannot parse against a DB that stops at 0003 — every site here applies both. + readScript(dbPath, migration9Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); @@ -856,6 +863,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); @@ -910,6 +918,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); @@ -964,6 +973,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); @@ -1104,6 +1114,7 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index af3380416..ddcf4287a 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -1,6 +1,6 @@ /// import { execFileSync } from 'node:child_process'; -import { mkdtempSync, rmSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -10,8 +10,11 @@ import { LEADERBOARD_SQL, LINK_CONTRACTS_LIMIT, LINK_CONTRACTS_SQL, + LINK_SELECT, OFFICIAL_SQL, + SURFACED_OWNERSHIP, } from './queries/related-persons'; +import { SEARCH_HITS_SQL } from './queries/search'; // Integration test for the свързани-лица SQL. The query layer's unit tests (queries/related-persons.test) // use a fake D1 and never run the aggregation; this runs the EXACT exported SQL against a real SQLite @@ -22,6 +25,8 @@ import { const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006: SURFACED_OWNERSHIP now requires a Trade Register evidence seal (#279, ADR-0033). +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }).trim(); @@ -122,6 +127,13 @@ INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, co -- Rollup row for the awarding body — the per-authority capture-share denominator the read query LEFT JOINs. INSERT INTO authority_totals (authority_id, name, spent_eur, contracts, suppliers, avg_eur) VALUES ('a:1','ОБЩИНА ТЕСТ',50000000,10,4,5000000); + +-- Every link carries a Trade Register evidence seal (#279, ADR-0033). SURFACED_OWNERSHIP now requires +-- one, so a fixture without seals renders an EMPTY surface and every assertion below passes vacuously. +-- Derived from interest_links itself, so a row added to the fixture later is sealed automatically and +-- cannot silently drop off the surface. +INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links; `; describe('свързани-лица SQL (real SQLite)', () => { @@ -131,6 +143,7 @@ describe('свързани-лица SQL (real SQLite)', () => { try { readScript(dbPath, migration0); readScript(dbPath, migration2); + readScript(dbPath, migration9); sqlite(dbPath, FIXTURE); return fn(dbPath); } finally { @@ -193,7 +206,9 @@ describe('свързани-лица SQL (real SQLite)', () => { INSERT INTO persons (id, name) VALUES ('person:zero','Нула Тестов'); INSERT INTO interest_links (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES - ('il:zerofam','person:zero|666|family','person:zero','eik:666','666','НУЛА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2021',0,0,NULL,NULL,'published');`, + ('il:zerofam','person:zero|666|family','person:zero','eik:666','666','НУЛА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2021',0,0,NULL,NULL,'published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence);`, ); const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); expect(board.some((r) => r.official === 'Нула Тестов')).toBe(false); // no live contracts → gated out @@ -217,6 +232,8 @@ describe('свързани-лица SQL (real SQLite)', () => { (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES ('il:alen','person:alen|701','person:alen','eik:701','701','АЛЕН КО ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'none',1,'2020','2021',1,1000000,'2021','2021','published'), ('il:boyan','person:boyan|702','person:boyan','eik:702','702','БОЯН КО ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'none',1,'2020','2021',1,10000000,'2021','2021','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES ('t:71','unp71','Обект А','a:1','открита процедура'),('t:72','unp72','Обект Б','a:1','открита процедура'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES @@ -322,6 +339,8 @@ describe('свързани-лица SQL (real SQLite)', () => { (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES ('il:dubl-self','person:dubl|800','person:dubl','eik:800','800','ДУБЪЛ ЕООД','exact_name_key','v1','B_distinctive','owns','private_ownership',0,'none',1,'2020','2022',2,60000,'2021','2022','published'), ('il:dubl-fam','person:dubl|800|family','person:dubl','eik:800b','800','ДУБЪЛ ЕООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2022',2,60000,'2021','2022','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES ('t:80','unp80','Обект Дубъл','a:1','открита процедура'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES ('c:80','t:80','eik:800',60000,'EUR','2021-05-01','Д-80',60000);`, ); @@ -350,6 +369,8 @@ describe('свързани-лица SQL (real SQLite)', () => { (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES ('il:asim-self-held','person:asim|850','person:asim','eik:850','850','АСИМ ЕООД','exact_name_key','v1','C_hold','owns','private_ownership',0,'none',1,'2020','2022',2,40000,'2021','2022','held'), ('il:asim-fam','person:asim|850|family','person:asim','eik:850','850','АСИМ ЕООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2022',2,40000,'2021','2022','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES ('t:85','unp85','Обект Асим','a:1','открита процедура'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES ('c:85','t:85','eik:850',40000,'EUR','2021-05-01','Д-85',40000);`, ); @@ -475,7 +496,9 @@ describe('свързани-лица SQL (real SQLite)', () => { INSERT INTO persons (id, name) VALUES ('person:praz','Празен Тестов'); INSERT INTO interest_links (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES - ('il:praz','person:praz|900','person:praz','eik:900','900','ПРАЗЕН ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'exact',1,'2020','2022',7,7000000,'2021','2022','published');`, + ('il:praz','person:praz|900','person:praz','eik:900','900','ПРАЗЕН ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'exact',1,'2020','2022',7,7000000,'2021','2022','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence);`, ); const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); expect(board.some((r) => r.official === 'Празен Тестов')).toBe(false); // gated out — no live contracts @@ -504,4 +527,173 @@ describe('свързани-лица SQL (real SQLite)', () => { ); }); }); + + // ── the evidence seal gate (#279, ADR-0033 decision 1) ────────────────────── + // + // This block exists because the gate had NO test pressure at all: deleting the entire + // `EXISTS (… interest_link_evidence …)` clause out of SURFACED_OWNERSHIP left the whole db suite + // green (361/361), because every fixture above seals every link 'document'. That clause is the only + // SQL between a WITHHELD link and a named public claim that a specific official owns a specific + // company, so an untested one is the most expensive kind of dead rail. + // + // Each rung below is a REAL withholding outcome of evidence.mjs, not an invented value: + // refuted — the register contradicts the declared stake + // bar_joint_stock — an АД/ЕАД, where a declared parcel of shares is not a material conflict + // unknown — no rung reached; the deed proves nothing either way + // outside_tr — ДЗЗД/BULSTAT, not in the Trade Register at all + // …plus a link with NO seal row whatsoever, which is what a half-loaded run produces. + describe('a link withheld by its evidence seal never reaches any public query', () => { + // All six share the SAME winner (eik 111, which has live contracts) and the same shape, so the + // ONLY thing that differs is the seal. Without that, an absent row could be absent for an unrelated + // reason and every assertion here would pass vacuously — which is exactly the bug being fixed. + const WITHHELD = [ + ['refuted', 'person:ref', 'Оборен Тестов'], + ['bar_joint_stock', 'person:bar', 'Акционер Тестов'], + ['unknown', 'person:unk', 'Неясен Тестов'], + ['outside_tr', 'person:out', 'Извън Тестов'], + ] as const; + + function seedRungs(dbPath: string): void { + const people: [string, string][] = [ + ...WITHHELD.map(([, id, name]) => [id, name] as [string, string]), + ['person:none', 'Безпечатен Тестов'], + ['person:ok', 'Потвърден Тестов'], + ]; + const links: string[] = [...WITHHELD.map(([, id]) => id), 'person:none', 'person:ok']; + sqlite( + dbPath, + `INSERT INTO persons (id, name) VALUES ${people.map(([id, n]) => `('${id}','${n}')`).join(',')}; + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES + ${links + .map( + (p) => + `('il:${p.split(':')[1]}','${p}|111','${p}','eik:111','111','ТРЕЙС ГРУП ХОЛД АД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'none',1,'2019','2023',3,1000,'2020','2021','published')`, + ) + .join(',')}; + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) VALUES + ${WITHHELD.map(([kind, id]) => `('${id}|111','${kind}',NULL,NULL,'2026-08-05','tr-rules-1','live')`).join(',')}, + ('person:ok|111','confirmed',NULL,'seat:СОФИЯ','2026-08-05','tr-rules-1','live');`, + ); + // person:none deliberately gets NO evidence row at all. + } + + it('the POSITIVE CONTROL surfaces — so every absence below is caused by the seal, not the fixture', () => { + withDb((dbPath) => { + seedRungs(dbPath); + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => r.official === 'Потвърден Тестов')).toBe(true); + expect(rows(dbPath, lit(OFFICIAL_SQL, 'person:ok'))).toHaveLength(1); + }); + }); + + for (const [kind, personId, name] of WITHHELD) { + it(`'${kind}' is withheld from the leaderboard, the official page, the company page and the drill-down`, () => { + withDb((dbPath) => { + seedRungs(dbPath); + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => r.official === name)).toBe(false); + expect(board.some((r) => r.link_key === `${personId}|111`)).toBe(false); + // The official's own page must 404 rather than render a withheld claim under their name… + expect(rows(dbPath, lit(OFFICIAL_SQL, personId))).toHaveLength(0); + // …the company page must not list them among that winner's office-holders… + expect(rows(dbPath, lit(COMPANY_SQL, '111')).some((r) => r.official === name)).toBe( + false, + ); + // …and the drill-down must not enumerate the contracts of a link it may not name. + expect(rows(dbPath, lit(LINK_CONTRACTS_SQL, `${personId}|111`))).toHaveLength(0); + }); + }); + } + + it('a link with NO seal row at all is withheld too — a half-loaded run must not publish', () => { + // The LEFT JOIN in LINK_SELECT would happily return this row with NULL evidence columns; only the + // EXISTS gate keeps it off the surface. This is the case a partial load actually produces. + withDb((dbPath) => { + seedRungs(dbPath); + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => r.official === 'Безпечатен Тестов')).toBe(false); + expect(rows(dbPath, lit(OFFICIAL_SQL, 'person:none'))).toHaveLength(0); + expect( + rows(dbPath, lit(COMPANY_SQL, '111')).some((r) => r.official === 'Безпечатен Тестов'), + ).toBe(false); + expect(rows(dbPath, lit(LINK_CONTRACTS_SQL, 'person:none|111'))).toHaveLength(0); + }); + }); + + it('the company page shows ONLY the sealed office-holders of that winner', () => { + // Four withheld links and one confirmed link all point at eik 111, alongside the base fixture's + // Иван. A gate that let any withheld rung through would show up here as an extra name. + withDb((dbPath) => { + seedRungs(dbPath); + const names = rows(dbPath, lit(COMPANY_SQL, '111')).map((r) => r.official); + expect(names.sort()).toEqual(['Иван Минев', 'Потвърден Тестов']); + }); + }); + }); +}); + +// The read-time contract join and the WRITER that fills contract_count/contract_value_eur must use the +// same join shape. On the read side `tenders`/`authorities` are never projected, so both joins read as +// dead and a reviewer will eventually propose deleting them (one did — cefothe on #309). Deleting them +// there alone widens the read past the stored aggregate: contemporaneous counts could exceed +// contract_count, and the EXISTS gate would surface links the zero-contract gate excluded. Nothing but a +// test can hold a TypeScript template string and a .mjs prepared statement to the same shape. +describe('the read-time contract join matches the writer that stored the aggregate', () => { + const writer = readFileSync(resolve(root, 'scripts/cacbg/load.mjs'), 'utf8'); + // The writer's per-winner contract query, located by its projection rather than by line number. + const writerJoin = /FROM contracts c JOIN tenders t ON[^"]*?WHERE b\.eik_normalized/.exec( + writer, + )?.[0]; + + it('the writer still joins contracts→tenders→authorities→bidders', () => { + expect(writerJoin, 'load.mjs per-winner contract query not found').toBeTruthy(); + for (const rel of ['tenders', 'authorities', 'bidders']) expect(writerJoin).toContain(rel); + }); + + it('CONTRACT_JOIN joins the same four relations, in the same direction', () => { + // Compared by RELATION and join condition, not by text: the two are written in different languages + // and alias differently (cc/tt/aa/bb vs c/t/a/b), so only the shape is comparable. + const flat = LINK_SELECT.replace(/\s+/g, ' '); + expect(flat).toContain('FROM contracts cc'); + expect(flat).toContain('JOIN tenders tt ON tt.id = cc.tender_id'); + expect(flat).toContain('JOIN authorities aa ON aa.id = tt.authority_id'); + expect(flat).toContain('JOIN bidders bb ON bb.id = cc.bidder_id'); + // …and the writer's equivalents, so a change to either side fails here. + expect(writerJoin).toContain('JOIN tenders t ON t.id=c.tender_id'); + expect(writerJoin).toContain('JOIN authorities a ON a.id=t.authority_id'); + expect(writerJoin).toContain('JOIN bidders b ON b.id=c.bidder_id'); + }); +}); + +// The evidence-seal gate has FOUR copies, and they must say the same thing. Reviewers found the fourth +// (the company-search badge) checking `status='published'` alone, so search advertised a свързани-лица +// claim about a named official that the detail page correctly withheld. `refresh-slice.sql` matters for a +// different reason: it runs on the 6-hourly cron, so drift there silently keeps officials in the search +// index whose links no longer surface. Nothing but a test binds SQL in three languages and two file types. +describe('the evidence-seal gate is identical in all four places it is written', () => { + const sources: [string, string][] = [ + ['related-persons.ts (SURFACED_OWNERSHIP)', SURFACED_OWNERSHIP], + ['search.ts (company badge)', SEARCH_HITS_SQL], + ['precompute.sql', readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8')], + ['refresh-slice.sql', readFileSync(resolve(root, 'scripts/refresh-slice.sql'), 'utf8')], + ]; + + it.each(sources)('%s requires a publishing evidence seal', (_label, sql) => { + // Whitespace-insensitive: the four are formatted for their own file, and a line break is not drift. + const flat = sql.replace(/\s+/g, ' '); + expect(flat).toMatch( + /EXISTS \( ?SELECT 1 FROM interest_link_evidence e WHERE e\.link_key = il\.link_key AND e\.evidence_kind IN \('document','confirmed'\) ?\)/, + ); + }); + + it.each(sources)('%s admits exactly the two publishing rungs', (_label, sql) => { + // The rung list is the gate. A future rung added to one copy and not the others either leaks an + // unproven claim or silently drops a proven one, depending on which copy gained it. + const kinds = [...sql.matchAll(/evidence_kind IN \(([^)]*)\)/g)].map((m) => + m[1]!.replace(/\s|'/g, ''), + ); + expect(kinds.length).toBeGreaterThan(0); + for (const k of kinds) expect(k).toBe('document,confirmed'); + }); }); diff --git a/packages/db/src/search-sql.test.ts b/packages/db/src/search-sql.test.ts index 58c25205d..59f9a95c8 100644 --- a/packages/db/src/search-sql.test.ts +++ b/packages/db/src/search-sql.test.ts @@ -16,6 +16,7 @@ import { SEARCH_HITS_SQL, SEARCH_HITS_SQL_NO_CONFLICT } from './queries/search'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function readScript(dbPath: string, path: string): void { execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${path}\n`, stdio: 'pipe' }); @@ -44,17 +45,26 @@ INSERT INTO bidders (id, name, eik_normalized, eik_valid, kind) VALUES ('eik:333','ГАМА ООД','333333333',1,'company'), ('eik:444','ДЕЛТА ООД','444444444',1,'company'), ('eik:555','ЕПСИЛОН ООД','555555555',1,'company'), - ('eik:666','ЗЕТА ООД','666666666',1,'company'); + ('eik:666','ЗЕТА ООД','666666666',1,'company'), + -- #279: two winners with a PUBLISHED link that must not badge. ЙОТА's link carries NO evidence seal at + -- all (a legacy row, a partial run, or a loader bug); КАПА's seal is a WITHHOLDING rung. The detail page + -- refuses both via SURFACED_OWNERSHIP; the badge has to agree, or search advertises an unproven claim + -- about a named official on a page that then shows nothing. + ('eik:777','ЙОТА ООД','777777777',1,'company'), + ('eik:888','КАПА ООД','888888888',1,'company'); INSERT INTO company_totals (bidder_id, name, kind, eik, eik_valid, won_eur, contracts, authorities) VALUES ('eik:111','АЛФА ООД','company','111111111',1,1000000,1,1), ('eik:222','БЕТА ООД','company','222222222',1,500000,1,1), ('eik:333','ГАМА ООД','company','333333333',1,200000,1,1), ('eik:444','ДЕЛТА ООД','company','444444444',1,300000,1,1), ('eik:555','ЕПСИЛОН ООД','company','555555555',1,700000,1,1), - ('eik:666','ЗЕТА ООД','company','666666666',1,900000,1,1); + ('eik:666','ЗЕТА ООД','company','666666666',1,900000,1,1), + ('eik:777','ЙОТА ООД','company','777777777',1,400000,1,1), + ('eik:888','КАПА ООД','company','888888888',1,600000,1,1); INSERT INTO persons (id, name) VALUES ('person:ИВАН МИНЕВ','Иван Минев'),('person:ГЕОРГИ ПЕТРОВ','Георги Петров'), - ('person:ДАНА ФАМ','Дана Фам'),('person:БОРИС БОРД','Борис Борд'),('person:ДВОЕН ТЕСТ','Двоен Тест'); + ('person:ДАНА ФАМ','Дана Фам'),('person:БОРИС БОРД','Борис Борд'),('person:ДВОЕН ТЕСТ','Двоен Тест'), + ('person:БЕЗ ПЕЧАТ','Без Печат'),('person:ЗАДЪРЖАН ПЕЧАТ','Задържан Печат'); INSERT INTO declarations (id, person_id, xml_file, control_hash, folder_year, declared_year, template, category, institution, position, source_url) VALUES ('decl:i','person:ИВАН МИНЕВ','i.xml','H1','2024','2023','assets','','ОБЩИНА РУСЕ','', 'https://register.cacbg.bg/2024/i.xml'), ('decl:g','person:ГЕОРГИ ПЕТРОВ','g.xml','H2','2024','2023','assets','','МИНИСТЕРСТВО Х','', 'https://register.cacbg.bg/2024/g.xml'), @@ -75,15 +85,30 @@ INSERT INTO interest_links -- so the redundant-family collapse (ADR-0032) DROPS the family link from the index; the winner's €50k counts -- ONCE, not €100k — no de-anonymization vector, no double-count. ('il:ds','person:ДВОЕН ТЕСТ|666','person:ДВОЕН ТЕСТ','eik:666','666666666','ЗЕТА ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',0,'none',1,'2020','2023',1,50000,'2021','2021','published'), - ('il:dfam','person:ДВОЕН ТЕСТ|666|family','person:ДВОЕН ТЕСТ','eik:666','666666666','ЗЕТА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2023',1,50000,'2021','2021','published'); + ('il:dfam','person:ДВОЕН ТЕСТ|666|family','person:ДВОЕН ТЕСТ','eik:666','666666666','ЗЕТА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2023',1,50000,'2021','2021','published'), + -- published, ownership class, live contracts — and NO seal. Everything the badge used to check, passed. + ('il:no','person:БЕЗ ПЕЧАТ|777','person:БЕЗ ПЕЧАТ','eik:777','777777777','ЙОТА ООД','exact_name_key','v1','document','owns','private_ownership',0,'none',1,'2020','2023',1,400000,'2021','2021','published'), + -- published, and sealed with a rung that WITHHOLDS (ADR-0035). A seal existing is not a seal permitting. + ('il:un','person:ЗАДЪРЖАН ПЕЧАТ|888','person:ЗАДЪРЖАН ПЕЧАТ','eik:888','888888888','КАПА ООД','exact_name_key','v1','document_uncorroborated','owns','private_ownership',0,'none',1,'2020','2023',1,600000,'2021','2021','published'); +INSERT INTO interest_link_evidence (link_key, evidence_kind, lookup_date, rules_version, live_status) VALUES + ('person:ИВАН МИНЕВ|111','document','2026-08-12','tr-rules-1','live'), + ('person:ИВАН МИНЕВ|333','confirmed','2026-08-12','tr-rules-1','live'), + ('person:ДАНА ФАМ|444','confirmed','2026-08-12','tr-rules-1','live'), + ('person:БОРИС БОРД|555','document','2026-08-12','tr-rules-1','live'), + ('person:ДВОЕН ТЕСТ|666','document','2026-08-12','tr-rules-1','live'), + ('person:ДВОЕН ТЕСТ|666|family','document','2026-08-12','tr-rules-1','live'), + ('person:ЗАДЪРЖАН ПЕЧАТ|888','document_uncorroborated','2026-08-12','tr-rules-1','live'); INSERT INTO authorities (id, name) VALUES ('a:1','ВЕДОМСТВО ТЕСТ'); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES - ('t:1','s1','Т1','a:1','open'),('t:3','s3','Т3','a:1','open'),('t:4','s4','Т4','a:1','open'),('t:6','s6','Т6','a:1','open'); + ('t:1','s1','Т1','a:1','open'),('t:3','s3','Т3','a:1','open'),('t:4','s4','Т4','a:1','open'), + ('t:6','s6','Т6','a:1','open'),('t:7','s7','Т7','a:1','open'),('t:8','s8','Т8','a:1','open'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES ('c:1','t:1','eik:111',1000000,'EUR','2021-05-01','N1',1000000), ('c:3','t:3','eik:333',200000,'EUR','2021-05-01','N3',200000), ('c:4','t:4','eik:444',300000,'EUR','2021-05-01','N4',300000), - ('c:6','t:6','eik:666',50000,'EUR','2021-05-01','N6',50000); + ('c:6','t:6','eik:666',50000,'EUR','2021-05-01','N6',50000), + ('c:7','t:7','eik:777',400000,'EUR','2021-05-01','N7',400000), + ('c:8','t:8','eik:888',600000,'EUR','2021-05-01','N8',600000); `; // Search-index population — a STRUCTURAL proxy for scripts/precompute.sql's officials block: it exercises the @@ -116,6 +141,7 @@ function withDb(fn: (dbPath: string) => void): void { try { readScript(dbPath, migration0); readScript(dbPath, migration2); + readScript(dbPath, migration9); exec(dbPath, FIXTURE); exec(dbPath, POPULATE_INDEX); fn(dbPath); @@ -151,6 +177,27 @@ describe('search свързани-лица SQL', () => { }); }); + it('the badge requires an evidence SEAL, not merely status=published (#279)', () => { + // The detail page gate is belt-and-braces — status AND a publishing seal (SURFACED_OWNERSHIP). The + // badge checked status alone, so an evidence-less published row advertised a свързани-лица claim on a + // named official in search while the page it links to correctly withheld it. Search is the wider + // surface of the two: it is what a reader sees before deciding to look. + withDb((dbPath) => { + // NO seal at all — the legacy/partial-run/loader-bug shape. + const iota = rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'йота*', 10)); + expect(iota).toHaveLength(1); + expect(iota[0]!.has_conflict).toBe(0); + // Sealed, but with a rung that WITHHOLDS. A seal existing is not a seal permitting. + const kapa = rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'капа*', 10)); + expect(kapa).toHaveLength(1); + expect(kapa[0]!.has_conflict).toBe(0); + // POSITIVE CONTROL — a properly sealed link still badges, on both publishing rungs. Without this a + // gate that rejected everything would satisfy the two assertions above. + expect(rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'алфа*', 10))[0]!.has_conflict).toBe(1); + expect(rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'гама*', 10))[0]!.has_conflict).toBe(1); + }); + }); + it('indexes family_ownership (ADR-0032) but NEVER management_role/ex-officio (interest_class gate)', () => { withDb((dbPath) => { // family_ownership now reaches the index identically to self (ADR-0032): Дана (a relative's stake) IS diff --git a/packages/db/src/seed-surface.test.ts b/packages/db/src/seed-surface.test.ts new file mode 100644 index 000000000..78a70aaa9 --- /dev/null +++ b/packages/db/src/seed-surface.test.ts @@ -0,0 +1,95 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { COMPANY_SQL, LEADERBOARD_SQL, OFFICIAL_SQL } from './queries/related-persons'; + +// The dev seed exists for ONE reason: `/conflicts` renders an empty surface without it, so the whole +// feature is unverifiable in a browser. That makes the seed a promise about the read gate, and a promise +// nothing checked — it was written against SURFACED_OWNERSHIP as it stood, then #279 added the evidence +// seal requirement to that same predicate and the fixture silently stopped surfacing. A fresh dev DB +// rendered empty, which is exactly the state the seed was created to prevent. +// +// So: run the REAL migrations, the REAL seed.sql, and the REAL exported queries. Any future change to +// the publishing gate that the seed does not keep up with now fails here rather than in someone's +// browser, days later, as „the local app looks broken". + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migrations = [ + 'packages/db/migrations/0000_init.sql', + 'packages/db/migrations/0003_related_persons_foundation.sql', + 'packages/db/migrations/0009_interest_link_evidence.sql', +].map((m) => resolve(root, m)); +const seed = resolve(root, 'scripts/seed.sql'); + +function seededDb(fn: (dbPath: string) => T): T { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-seed-')); + const dbPath = resolve(dir, 'seed.sqlite'); + try { + for (const file of [...migrations, seed]) + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${file}\n`, stdio: 'pipe' }); + return fn(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} +function rows(dbPath: string, sql: string): Record[] { + const out = execFileSync('sqlite3', ['-json', dbPath], { input: sql, encoding: 'utf8' }).trim(); + return out ? JSON.parse(out) : []; +} +function lit(sql: string, ...vals: (string | number)[]): string { + let i = 0; + return sql.replace(/\?/g, () => { + const v = vals[i++]; + return typeof v === 'number' ? String(v) : `'${String(v).replace(/'/g, "''")}'`; + }); +} + +describe('scripts/seed.sql produces a surface a developer can actually see', () => { + it('the leaderboard is NOT empty — the seed survives every gate in SURFACED_OWNERSHIP', () => { + seededDb((dbPath) => { + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.length).toBeGreaterThan(0); + // Both published outcomes the seed is built around: the official's own stake and a relative's. + expect(board.map((r) => r.relation).sort()).toEqual(['owns', 'related']); + }); + }); + + it('every surfaced seed link carries a real publishing seal, not a coincidence', () => { + seededDb((dbPath) => { + for (const r of rows(dbPath, lit(LEADERBOARD_SQL, 100))) + expect(['document', 'confirmed']).toContain(r.evidence_kind); + }); + }); + + it('the held joint-stock link stays off the surface — the seed keeps its negative case', () => { + // ГАМА ИНВЕСТ АД is seeded as a withheld link precisely so a change to the publishing rule shows up + // as a before/after rather than as an empty page either way. If it ever surfaces, the materiality + // bar (ADR-0022) or the seal gate has quietly stopped working. + seededDb((dbPath) => { + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => String(r.company).includes('ГАМА ИНВЕСТ'))).toBe(false); + expect(rows(dbPath, lit(COMPANY_SQL, '204556676'))).toHaveLength(0); + }); + }); + + it('the official and company pages render too — not just the leaderboard', () => { + seededDb((dbPath) => { + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + const own = board.find((r) => r.relation === 'owns')!; + expect(rows(dbPath, lit(OFFICIAL_SQL, String(own.person_id))).length).toBeGreaterThan(0); + expect(rows(dbPath, lit(COMPANY_SQL, String(own.eik))).length).toBeGreaterThan(0); + }); + }); + + it('re-running the seed is idempotent — INSERT OR IGNORE, no duplicated links', () => { + seededDb((dbPath) => { + const before = rows(dbPath, lit(LEADERBOARD_SQL, 100)).length; + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${seed}\n`, stdio: 'pipe' }); + expect(rows(dbPath, lit(LEADERBOARD_SQL, 100))).toHaveLength(before); + }); + }); +}); diff --git a/packages/db/src/value-flag-annex-step-sql.test.ts b/packages/db/src/value-flag-annex-step-sql.test.ts index c48721778..8c52bd892 100644 --- a/packages/db/src/value-flag-annex-step-sql.test.ts +++ b/packages/db/src/value-flag-annex-step-sql.test.ts @@ -26,6 +26,10 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script +// only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it +// ever reaches the value_flag CASE these tests are about. +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); // #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); @@ -65,6 +69,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); diff --git a/packages/db/src/value-flag-lot-stotinki-sql.test.ts b/packages/db/src/value-flag-lot-stotinki-sql.test.ts index 388d31729..67fea3526 100644 --- a/packages/db/src/value-flag-lot-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-lot-stotinki-sql.test.ts @@ -28,6 +28,8 @@ const migration3Path = resolve(root, 'packages/db/migrations/0003_related_person const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +// #279/ADR-0033: refresh-slice.sql's свързани-лица block reads interest_link_evidence, so 0009 too. +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -60,6 +62,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); + readScript(dbPath, migration9Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/value-flag-stotinki-sql.test.ts b/packages/db/src/value-flag-stotinki-sql.test.ts index a13808aab..49ab5e6fe 100644 --- a/packages/db/src/value-flag-stotinki-sql.test.ts +++ b/packages/db/src/value-flag-stotinki-sql.test.ts @@ -20,6 +20,10 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script +// only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it +// ever reaches the value_flag CASE these tests are about. +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); // #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); @@ -54,6 +58,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); readScript(dbPath, migration6Path); readScript(dbPath, migration7Path); readScript(dbPath, migration8Path); diff --git a/packages/ingest/src/refresh-officials.test.ts b/packages/ingest/src/refresh-officials.test.ts index 55170ef21..2c0ebb8f8 100644 --- a/packages/ingest/src/refresh-officials.test.ts +++ b/packages/ingest/src/refresh-officials.test.ts @@ -20,6 +20,9 @@ const migrations = [ '0000_init.sql', '0001_flow_pairs_bidder_index.sql', '0003_related_persons_foundation.sql', + // The officials batch now joins the Trade Register evidence seal (#279, ADR-0033) — without 0006 the + // real refresh-slice.sql this test executes cannot parse. + '0009_interest_link_evidence.sql', ].map((f) => resolve(root, 'packages/db/migrations', f)); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); @@ -67,6 +70,11 @@ INSERT INTO interest_links -- her — dropping the s.status='published' guard would collapse it behind an invisible self stake (false neg). ('il:ys','person:ЯНА|444','person:ЯНА','eik:444','444','ДЕЛТА ООД','exact_name_key','v1','C_hold','owns','private_ownership',0,'none',1,'2020','2023',1,20000,'2021','2021','held'), ('il:yf','person:ЯНА|444|family','person:ЯНА','eik:444','444','ДЕЛТА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',1,'none',1,'2020','2023',1,20000,'2021','2021','published'); +-- Every link carries an evidence seal (#279, ADR-0033): the officials batch requires a publishing rung, +-- so a fixture without seals indexes NOBODY and every assertion below passes vacuously. Derived from +-- interest_links, so a row added later is sealed automatically. +INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links; `; describe('ETL refresh-slice officials batch (the live parse-then-execute path)', () => { diff --git a/scripts/cacbg/audit.mjs b/scripts/cacbg/audit.mjs index 50ff29334..6ca5f3a75 100644 --- a/scripts/cacbg/audit.mjs +++ b/scripts/cacbg/audit.mjs @@ -7,11 +7,14 @@ import { DatabaseSync } from 'node:sqlite'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { nameDistinctiveness } from './classify.mjs'; +import fs from 'node:fs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; +import { RULES_VERSION, isSealedFact } from '../tr/evidence.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); +const STAGING = process.env.CACBG_STAGING || path.join(ROOT, 'scratch/cacbg/staging'); +const SNAPSHOT = path.join(STAGING, 'published-snapshot.json'); const { companyNameKey } = await import('../../packages/shared/src/company-name-key.ts'); const db = new DatabaseSync(DB, { readOnly: true }); @@ -31,12 +34,34 @@ const published = db ` SELECT il.id, il.link_key, il.person_id, il.eik, il.entity_key, il.match_method, il.publish_tier, il.bidder_id, il.relation, il.contemporaneous, il.contract_value_eur, - b.name AS bidder_name, b.eik_normalized AS bidder_eik, b.eik_valid AS bidder_eik_valid + b.name AS bidder_name, b.eik_normalized AS bidder_eik, b.eik_valid AS bidder_eik_valid, + -- LEFT JOIN, deliberately: a published link with NO seal is the finding, so it must reach the + -- loop rather than be filtered out of it. + e.evidence_kind, e.registry_role, e.matched_fact FROM interest_links il JOIN bidders b ON b.id = il.bidder_id + LEFT JOIN interest_link_evidence e ON e.link_key = il.link_key WHERE il.status = 'published'`, ) .all(); +// Every link's CURRENT status, for the monotonicity gate below. A link that left the published set +// because it was SUPPRESSED is still in this table — the takedown is a status flip, not a delete — and +// distinguishing that from a link that simply stopped being built is the difference between a declared +// removal and a silent recall regression. Read here, while the handle is open. +const statusNow = new Map( + db + .prepare('SELECT link_key, status FROM interest_links') + .all() + .map((r) => [r.link_key, r.status]), +); + +// The only two evidence rungs that publish (ADR-0033 decision 1). Everything else withholds. +const PUBLISHING_EVIDENCE = new Set(['document', 'confirmed']); +// The closed vocabulary for a sealed matched_fact lives in evidence.mjs, next to the code that WRITES +// it — one definition, so the gate can never permit a shape the writer has stopped emitting (or, worse, +// the other way round). `isSealedFact` bounds the seat leg to a settlement's one or two tokens, which is +// what stops a three-part Bulgarian name riding through behind the legitimate `seat:` prefix. + const findings = []; const flag = (link, axis, detail) => findings.push({ axis, link_key: link.link_key, eik: link.eik, detail }); @@ -50,12 +75,16 @@ for (const l of published) { // l.eik is a valid winner BEARING this name key (a stray/mis-attached ЕИК is still caught). The ЕИК+name // double-lock itself is re-proven in the provenance pass below (A_eik_no_provenance). if (!rec) flag(l, 'A_key_missing', `entity_key ${l.entity_key} not found in live bidder set`); - else if (l.publish_tier === 'A_eik') { + else if (l.match_method === 'declared_eik') { + // Identity here is the declarant-provided ЕИК, not the name (ADR-0028), so a name shared by more + // than one winner is legitimate — the ЕИК picks exactly one. Require instead that l.eik is a valid + // winner BEARING this name key, which still catches a stray or mis-attached ЕИК. The ЕИК+name + // double-lock is re-proven independently in the provenance pass below. if (!rec.valid.has(l.eik)) flag( l, 'A_eik_not_winner', - `A_eik published ${l.eik}, not among key ${l.entity_key}'s valid winners {${[...rec.valid].join(',')}}`, + `declared_eik published ${l.eik}, not among key ${l.entity_key}'s valid winners {${[...rec.valid].join(',')}}`, ); } else if (rec.valid.size !== 1) flag( @@ -72,12 +101,34 @@ for (const l of published) { if (!l.bidder_eik_valid) flag(l, 'B_eik_invalid', `published on eik_valid=0 bidder ${l.bidder_name}`); - // C. Tier honesty: B_distinctive must actually be distinctive by the same classifier that gated it. - if (l.publish_tier === 'B_distinctive' && nameDistinctiveness(l.entity_key) !== 'distinctive') + // C. Evidence honesty (#279, ADR-0033). The tier IS the evidence kind now, so the axis that used to + // re-derive name distinctiveness re-derives the publishing rule instead: a published link must + // carry a seal, and that seal must be one of the two rungs that publish. `nameDistinctiveness` + // no longer gates publication at all — it survives only as a withholding filter inside the + // loader — so re-checking it here would assert a rule that is no longer in force. + if (!l.evidence_kind) + flag(l, 'C_no_evidence', `published with no Trade Register evidence seal (${l.bidder_name})`); + else if (!PUBLISHING_EVIDENCE.has(l.evidence_kind)) + flag( + l, + 'C_withholding_evidence', + `published on evidence_kind='${l.evidence_kind}', which withholds (${l.bidder_name})`, + ); + else if (l.evidence_kind !== l.publish_tier) + flag( + l, + 'C_tier_evidence_mismatch', + `publish_tier='${l.publish_tier}' but sealed evidence_kind='${l.evidence_kind}'`, + ); + + // C2. The PII rail, audited rather than assumed: matched_fact is a CLOSED vocabulary and can never + // carry a name. The registry deed's names are read only to produce a boolean and must never reach + // a served column (#279 §9, ADR-0033 decision 5). A schema cannot enforce this; this does. + if (!isSealedFact(l.matched_fact)) flag( l, - 'C_not_distinctive', - `tier B_distinctive but nameDistinctiveness=${nameDistinctiveness(l.entity_key)} (${l.bidder_name})`, + 'C_matched_fact_shape', + `matched_fact='${l.matched_fact}' is outside the closed vocabulary — a name may have leaked`, ); } @@ -126,6 +177,63 @@ for (const l of nonExact) { db.close(); +// D. Monotonicity — ADR-0033 decision 6, the correction of #279 §8. +// +// §8 asked for a seal kept „forever" and strictly-additive recomputation. That is unachievable: labels +// flip, the deed cache expires, and a court can annul an entry (чл. 29 ЗТРРЮЛНЦ) with no rules change +// at all. So the seal is NOT a store — it is re-derived every run — and monotonicity is enforced HERE, +// as a gate, against the export load.mjs writes immediately before it drops the CACBG tables. +// +// The rule: a link that was published last run and is not published now is a REGRESSION unless the +// rules themselves changed. Under an unchanged rules_version nothing licensed the removal, so it is a +// hard finding. Under a changed one it is an intentional event and degrades to a printed diff. +// +// ship-related-persons.mjs's count floor cannot do this job: it compares a COUNT, so a one-for-one +// swap — one true link silently dropped, one gained — leaves it perfectly quiet. +const publishedNow = new Set(published.map((l) => l.link_key)); +let priorPublished = null; +try { + priorPublished = JSON.parse(fs.readFileSync(SNAPSHOT, 'utf8')); +} catch (e) { + // ENOENT is the legitimate first run — there is no prior surface to regress from. Anything else + // (unreadable, malformed) must not be swallowed into a silent pass of the gate. + if (e.code !== 'ENOENT') throw e; +} +const vanished = (priorPublished ?? []).filter((p) => !publishedNow.has(p.link_key)); + +// The three grounds decision 6 sanctions for a removal. Anything else that vanished is a regression. +// +// Getting this set right is what stops the gate from deadlocking the mechanisms it points at. The +// rules bump is the obvious one. The other two are not optional extras — each is the ONLY expressible +// form of a removal the ADR names in prose: +// +// suppressed — the ADR-0031 takedown path, which decision 6 wires the court-annulled entry +// (чл. 29 ЗТРРЮЛНЦ) to by name. It flips status published → suppressed with rules_version +// untouched, so without this branch the one removal the ADR explicitly licenses hard-fails. +// corrected — "a correction of wrong input". Suppression cannot express it: correcting the input +// unbuilds the link, and load.mjs's B3 unused-suppression gate then fails the build for a +// fingerprint that matched nothing. So the two sanctioned removals would fail in OPPOSITE +// directions, leaving a real correction with no path at all. The flag is set by load.mjs from the +// version-controlled, fingerprinted corrections list — a human decision recorded in git, not a +// condition the audit can infer. +// +// Both are declared, both are reviewed, and both are still PRINTED below: a withdrawn public claim is +// never silent, it just is not a build failure. +const declaredRemoval = (p) => + p.rules_version !== RULES_VERSION || + p.corrected === true || + statusNow.get(p.link_key) === 'suppressed'; +const regressions = vanished.filter((p) => !declaredRemoval(p)); +for (const p of regressions) + findings.push({ + axis: 'D_monotonicity', + link_key: p.link_key, + eik: p.link_key.split('|')[1] ?? '', + // The link_key is named explicitly: it is the only handle a human has to go and look at which + // claim disappeared, and the shared axis report prints the ЕИК alone. + detail: `${p.link_key} published last run under rules_version ${p.rules_version} (unchanged) and is not published now — nothing licensed this removal`, + }); + // Report const byAxis = {}; for (const f of findings) (byAxis[f.axis] ??= []).push(f); @@ -137,6 +245,29 @@ for (const [axis, fs] of Object.entries(byAxis)) { for (const f of fs.slice(0, 20)) console.log(` - [${f.eik}] ${f.detail}`); console.log(''); } +if (priorPublished === null) { + console.log(`## monotonicity — no prior export at ${SNAPSHOT}; treating this as a first run\n`); +} else { + const declared = vanished.filter(declaredRemoval); + console.log( + `## monotonicity — ${priorPublished.length} published last run, ${publishedNow.size} now; ` + + `${regressions.length} regression(s), ${declared.length} declared removal(s)\n`, + ); + // Declared removals are not findings, but they are never silent: withdrawing a named public claim is + // exactly when a human should read WHICH claims went and on what ground. The ground is printed too — + // "removed" alone would flatten a court annulment, a corrected parse and a rules bump into one line. + for (const p of declared) { + const ground = + p.rules_version !== RULES_VERSION + ? `a rules change (${p.rules_version} → ${RULES_VERSION})` + : statusNow.get(p.link_key) === 'suppressed' + ? 'a suppression (ADR-0031 takedown path)' + : 'an acknowledged input correction'; + console.log(` - ${p.link_key} removed under ${ground}`); + } + if (declared.length) console.log(''); +} + console.log(`## non-exact provenance (${provenance.length}) — verify each cross-check by eye`); for (const p of provenance) { console.log(` - ${p.method} [${p.eik}] winner="${p.winner}"`); diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index df36baa84..e0f8af4bb 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -26,10 +26,15 @@ const dirs = []; // Build a fixture DB (bidders + declarations + declared_interests + interest_links), run audit.mjs against // it as a subprocess, and return { threw, out } — threw=true iff the audit exited non-zero (a hard finding). -function buildAndAudit({ bidders, decls = [], dis = [], links }) { +function buildAndAudit({ bidders, decls = [], dis = [], links, seals = [], snapshot = null }) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-audit-')); dirs.push(dir); const DB = path.join(dir, 'fixture.sqlite'); + const staging = path.join(dir, 'staging'); + fs.mkdirSync(staging, { recursive: true }); + // The pre-wipe export load.mjs writes before it drops the CACBG tables. Absent === a first run. + if (snapshot) + fs.writeFileSync(path.join(staging, 'published-snapshot.json'), JSON.stringify(snapshot)); const db = new DatabaseSync(DB); db.exec(` CREATE TABLE bidders(id TEXT PRIMARY KEY, name TEXT, eik_normalized TEXT, eik_valid INT); @@ -38,10 +43,16 @@ function buildAndAudit({ bidders, decls = [], dis = [], links }) { CREATE TABLE interest_links( id TEXT PRIMARY KEY, link_key TEXT, person_id TEXT, eik TEXT, entity_key TEXT, match_method TEXT, publish_tier TEXT, bidder_id TEXT, relation TEXT, contemporaneous INT, contract_value_eur REAL, status TEXT); + -- The evidence seal (#279, migration 0006). The audit LEFT JOINs it, so it must exist even when a + -- case deliberately leaves a link unsealed — an unsealed published link is itself a finding. + CREATE TABLE interest_link_evidence( + link_key TEXT PRIMARY KEY, evidence_kind TEXT, registry_role TEXT, matched_fact TEXT, + entry_number TEXT, entry_date TEXT, lookup_date TEXT, rules_version TEXT, live_status TEXT); ${bidders.map((b) => `INSERT INTO bidders VALUES (${b});`).join('\n')} ${decls.map((d) => `INSERT INTO declarations VALUES (${d});`).join('\n')} ${dis.map((d) => `INSERT INTO declared_interests(declaration_id, entity_raw) VALUES (${d});`).join('\n')} ${links.map((l) => `INSERT INTO interest_links VALUES (${l});`).join('\n')} + ${seals.map((e) => `INSERT INTO interest_link_evidence(link_key,evidence_kind,registry_role,matched_fact,lookup_date,rules_version,live_status) VALUES (${e});`).join('\n')} `); db.close(); @@ -51,7 +62,12 @@ function buildAndAudit({ bidders, decls = [], dis = [], links }) { out = execFileSync( 'node', ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'audit.mjs')], - { cwd: ROOT, env: { ...process.env, CACBG_DB: DB }, encoding: 'utf8', stdio: 'pipe' }, + { + cwd: ROOT, + env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: staging }, + encoding: 'utf8', + stdio: 'pipe', + }, ); } catch (e) { threw = true; // execFileSync throws on non-zero exit (a hard finding fired) @@ -73,8 +89,9 @@ test('A_eik behind a colliding name, backed by a real ЕИК+name double-lock, P // the declarant wrote BOTH the ЕИК and the фирма → the double-lock the loader required dis: [`'d1','„ОБЩ" ЕООД, ЕИК 100000001'`], links: [ - `'il1','p1|100000001','p1','100000001','${KEY}','declared_eik','A_eik','b1','owns',0,1000,'published'`, + `'il1','p1|100000001','p1','100000001','${KEY}','declared_eik','confirmed','b1','owns',0,1000,'published'`, ], + seals: [`'p1|100000001','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], }); assert.equal( threw, @@ -93,8 +110,9 @@ test('A_eik published on a ЕИК that is NOT a winner bearing the name key → dis: [`'d1','„ОБЩ" ЕООД, ЕИК 300000003'`], // entity_key claims the colliding name, but the published eik/bidder is the unrelated 300000003 links: [ - `'il1','p1|300000003','p1','300000003','${KEY}','declared_eik','A_eik','b3','owns',0,1000,'published'`, + `'il1','p1|300000003','p1','300000003','${KEY}','declared_eik','confirmed','b3','owns',0,1000,'published'`, ], + seals: [`'p1|300000003','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], }); assert.equal(threw, true, 'a stray-ЕИК A_eik link must fail the gate'); assert.equal( @@ -110,8 +128,9 @@ test('A_eik with no declaration carrying its ЕИК+name → A_eik_no_provenance decls: [`'d1','p1'`], dis: [`'d1','нещо съвсем друго без ЕИК'`], // no ЕИК, no фирма → the double-lock cannot be re-proven links: [ - `'il1','p1|100000001','p1','100000001','${K('„ОБЩ" ЕООД')}','declared_eik','A_eik','b1','owns',0,1000,'published'`, + `'il1','p1|100000001','p1','100000001','${K('„ОБЩ" ЕООД')}','declared_eik','confirmed','b1','owns',0,1000,'published'`, ], + seals: [`'p1|100000001','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], }); assert.equal( threw, @@ -130,9 +149,325 @@ test('a NON-A_eik (name-based) colliding link STILL fails A_multi_eik — the na bidders: COLLIDING_BIDDERS, // exact_name_key published on a name that maps to 2 ЕИК — exactly what the gate must keep rejecting links: [ - `'il1','p1|100000001','p1','100000001','${KEY}','exact_name_key','B_distinctive','b1','owns',0,1000,'published'`, + `'il1','p1|100000001','p1','100000001','${KEY}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'`, ], }); assert.equal(threw, true, 'a name-based colliding published link must still fail'); assert.equal(/A_multi_eik/.test(out), true, 'invariant A still fires for non-A_eik links'); }); + +// ── C: evidence honesty (#279, ADR-0033) ───────────────────────────────────────────────────────── +// The axis that used to re-derive name distinctiveness now re-derives the PUBLISHING rule, because the +// publishing rule changed: identity rests on a registry fact, and nameDistinctiveness survives only as +// a withholding filter inside the loader. Re-checking distinctiveness here would assert a rule that is +// no longer in force — a test that passes while guarding nothing. +test('a published link with NO evidence seal is a hard finding', () => { + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [], // the whole point: nothing sealed it + }); + assert.equal(threw, true, 'publishing without evidence must fail the gate'); + assert.equal(/C_no_evidence/.test(out), true, out); +}); + +test('a link published on a WITHHOLDING rung is a hard finding', () => { + // bar_joint_stock / unknown / outside_tr never publish. If one ever reaches status='published', the + // loader and the seal disagree and a claim is on the surface that no evidence supports. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','unknown','b1','owns',0,1000,'published'`, + ], + seals: [`'p1|100000001','unknown',NULL,NULL,'2026-08-05','tr-rules-1','live'`], + }); + assert.equal(threw, true); + assert.equal(/C_withholding_evidence/.test(out), true, out); +}); + +test('a tier that disagrees with its own seal is a hard finding', () => { + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [`'p1|100000001','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], + }); + assert.equal(threw, true); + assert.equal(/C_tier_evidence_mismatch/.test(out), true, out); +}); + +test('a matched_fact outside the closed vocabulary is a hard finding — the name-leak rail', () => { + // #279 §9: the deed's names are read only to produce a boolean and must never reach a served column. + // A schema cannot enforce a vocabulary, so the audit does. This is the shape a leak would take. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','ИВАН ПЕТРОВ ТЕСТОВ','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'a name in matched_fact must fail the gate'); + assert.equal(/C_matched_fact_shape/.test(out), true, out); +}); + +// …and the rail must catch the shape a leak would ACTUALLY take. `seat:` is a legitimate member +// of the vocabulary, so an unbounded `seat:[\p{Lu} -]+` admits `seat:ИВАН ПЕТРОВ ГЕОРГИЕВ` — a full +// three-part Bulgarian name (ЗГР чл. 9) wearing the prefix of a fact we allow. That is precisely the +// value the rail exists to reject, and it is the one a mis-split of the seat field would produce. +// A Bulgarian settlement is one or two tokens („СОФИЯ", „ВЕЛИКО ТЪРНОВО", „ГЕНЕРАЛ ТОШЕВО"); the +// three-part name is exactly three. The bound is deliberately tight: a rarer 3-token seat trips the +// audit and a human adjudicates, which is the correct direction for a rail whose failure mode is +// publishing somebody's name. +test('a THREE-TOKEN seat is a name shape, not a settlement — the rail must catch it', () => { + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','confirmed','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','confirmed',NULL,'seat:ИВАН ПЕТРОВ ГЕОРГИЕВ','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'a three-part name behind seat: must fail the gate'); + assert.equal(/C_matched_fact_shape/.test(out), true, out); +}); + +test('a real two-token settlement still passes — the bound must not empty the seat rung', () => { + const { threw } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','confirmed','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','confirmed',NULL,'seat:ВЕЛИКО ТЪРНОВО','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, false, 'ВЕЛИКО ТЪРНОВО is a settlement and must survive the rail'); +}); + +test('a well-formed seal passes every C axis (positive control)', () => { + // Without this, all four negatives above would still pass if the axes fired unconditionally. + const { threw } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, false, 'a correctly sealed link must not fail the gate'); +}); + +// ── D. Monotonicity (ADR-0033 decision 6) ──────────────────────────────────────────────────────── +// #279 §8 asked for a seal kept „forever" and strictly-additive recomputation. ADR-0033 showed that is +// false (labels flip, the cache expires, a court can annul an entry) and replaced the STORE with a +// GATE: seals are re-derived every run, and the audit compares the current published set against the +// pre-wipe export. A link that vanishes under an UNCHANGED rules_version is a hard finding — nothing +// about the rules changed, so its disappearance is a regression, not a decision. +// +// Why this needs its own gate and ship's count floor cannot serve: assertShipFloor checks a COUNT. +// A one-for-one swap — one link lost, one gained — leaves the count identical and the floor silent +// while a true published link is dropped. Only a per-key comparison sees it. +const SEALED = [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'`, +]; +const ONE_LINK = { + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`, `'b2','ВТОРА ФИРМА ЕООД','200000002',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: SEALED, +}; + +test('D — a previously published link that vanished under an UNCHANGED rules_version is a hard finding', () => { + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, // published last run, gone now + ], + }); + assert.equal(threw, true, 'a silent recall regression must fail the build'); + assert.match(out, /D_monotonicity/); + assert.match(out, /p9\|200000002/); +}); + +test('D — the same disappearance under a CHANGED rules_version is a printed diff, not a finding', () => { + // Removal stays an intentional event: bumping the rules version is how you declare one. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [{ link_key: 'p9|200000002', rules_version: 'tr-rules-0' }], + }); + assert.equal(threw, false, 'a declared rules change must not fail the build'); + assert.match(out, /p9\|200000002/, 'but it must still be reported'); +}); + +test('D — no snapshot (a first run) neither fires nor crashes', () => { + const { threw } = buildAndAudit(ONE_LINK); + assert.equal(threw, false); +}); + +// The gate must also let the two removals ADR-0033 decision 6 actually SANCTIONS through, or it +// deadlocks the mechanisms it points at. Both are declared events with a reviewed paper trail in git; +// neither is a silent recall regression, and the gate exists to tell those apart. + +test('D — a link taken down through the ADR-0031 suppression path is a declared removal, not a regression', () => { + // Decision 6 names the court-annulled entry (чл. 29 ЗТРРЮЛНЦ) and wires it to ADR-0031. That path + // flips status published → suppressed, so the link leaves the published set with rules_version + // unchanged — firing the gate on the ONE removal the ADR explicitly licenses. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + links: [ + ...ONE_LINK.links, + `'il2','p9|200000002','p9','200000002','${K('ВТОРА ФИРМА ЕООД')}','exact_name_key','document','b2','owns',0,1000,'suppressed'`, + ], + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, + ], + }); + assert.equal(threw, false, 'the sanctioned takedown path must not fail the build'); + assert.match(out, /p9\|200000002/, 'but a withdrawn public claim is never silent'); +}); + +test('D — a snapshot entry acknowledged as a corrected input is a declared removal', () => { + // The other sanctioned ground: the link should never have been published because its INPUT was + // wrong. Suppression cannot express it — correcting the input unbuilds the link, and load.mjs's B3 + // gate then fails the build for a suppression that matched nothing. Without this the two sanctioned + // removals fail in opposite directions and there is no way to clear either. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1', corrected: true }, + ], + }); + assert.equal(threw, false, 'an acknowledged correction must not fail the build'); + assert.match(out, /p9\|200000002/); +}); + +test('D — neither escape hatch fires on its own: an unacknowledged, unsuppressed drop still hard-fails', () => { + // The mutation control for the two tests above. A gate that accepted every disappearance would pass + // both of them, and this is the assertion that says it did not. + const { threw } = buildAndAudit({ + ...ONE_LINK, + links: [ + ...ONE_LINK.links, + `'il2','p9|200000002','p9','200000002','${K('ВТОРА ФИРМА ЕООД')}','exact_name_key','document','b2','owns',0,1000,'held'`, + ], + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, + ], + }); + assert.equal( + threw, + true, + 'held is not a sanctioned removal — the evidence simply stopped licensing it', + ); +}); + +test('D positive control — an unchanged published set produces no monotonicity finding', () => { + // Without this, a gate that never fires would pass both negatives above. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [{ link_key: 'p1|100000001', rules_version: 'tr-rules-1' }], + }); + assert.equal(threw, false); + assert.doesNotMatch(out, /D_monotonicity/); +}); + +// The four axes below fire on shapes that no test exercised. Each is a hard finding — the audit is the +// last gate before a named claim ships — so an axis that silently stopped firing would be invisible. + +test('A_key_missing: a link whose entity_key is in NO live bidder → hard finding', () => { + // The key resolved when the link was built and does not now: the winner was renamed, re-keyed or + // dropped from the corpus. The link still names an official against a company we can no longer find, + // so it must stop the run rather than ship pointing at nothing. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('ИЗЧЕЗНАЛ ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'an unresolvable entity_key must fail the gate'); + assert.equal(/A_key_missing/.test(out), true); +}); + +test('A_eik_mismatch: a name-resolved key pointing at a DIFFERENT ЕИК than it resolves to → hard', () => { + // The libel case in its purest form: the name resolves to exactly one winner, and the link published a + // different company against it. Everything on the card — contracts, money, the ЕИК link — would be the + // wrong company's, under a real official's name. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`, `'b2','ДРУГ ЕООД','200000002',1`], + links: [ + `'il1','p1|200000002','p1','200000002','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b2','owns',0,1000,'published'`, + ], + seals: [ + `'p1|200000002','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true); + assert.equal(/A_eik_mismatch/.test(out), true); +}); + +test('B_bidder_eik: the stored bidder row disagrees with the link ЕИК → hard finding', () => { + // Row integrity. The card renders the BIDDER's name and money but links out on the link's ЕИК, so a + // disagreement means the reader is shown one company and sent to another. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`, `'b2','ДРУГ ЕООД','200000002',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b2','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true); + assert.equal(/B_bidder_eik/.test(out), true); +}); + +test('B_eik_invalid: publishing against a checksum-INVALID ЕИК → hard finding', () => { + // eik_valid=0 means the ЕИК failed its control digit, so it identifies no company at all. This axis is + // the rail that keeps such a row off the public surface, and nothing else re-checks it downstream. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',0`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'an invalid-ЕИК publish must fail the gate'); + assert.equal(/B_eik_invalid/.test(out), true); +}); + +test('the four axes above are a BOUND: a clean, valid, sealed link passes them all', () => { + // POSITIVE CONTROL for the whole block. Four assertions that something fails prove nothing unless the + // correct shape passes — an audit that flagged everything would satisfy every test above. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, false, `a clean link must pass: ${out}`); + for (const axis of ['A_key_missing', 'A_eik_mismatch', 'B_bidder_eik', 'B_eik_invalid']) + assert.equal(new RegExp(axis).test(out), false, `${axis} must not fire on a clean link`); +}); diff --git a/scripts/cacbg/classify.mjs b/scripts/cacbg/classify.mjs index fe979345f..d066a3fa9 100644 --- a/scripts/cacbg/classify.mjs +++ b/scripts/cacbg/classify.mjs @@ -16,6 +16,7 @@ const FORM_TOKENS = new Set([ 'КД', 'СД', 'АДСИЦ', + 'КДА', 'КООПЕРАЦИЯ', 'ФОНДАЦИЯ', 'СДРУЖЕНИЕ', @@ -32,6 +33,24 @@ const FORM_TOKENS = new Set([ // generic/withhold — the safe side), never fabricates a joint-stock exclusion. Operates on UPPERCASE input. const SEAT_MARKER = /\s+(?:ГР|С|ОБЩ|ОБЛ|Ж\.К)\.\s*\S[^,]*$/u; const hasFormToken = (s) => s.split(/[^А-ЯЁ]+/).some((t) => FORM_TOKENS.has(t)); + +// The legal forms that TERMINATE a фирма (ЗТРРЮЛНЦ writes them as a suffix), so anything after one is a +// seat or a qualifier — never part of the name. ЕТ/СД/КД/КООПЕРАЦИЯ/ФОНДАЦИЯ/СДРУЖЕНИЕ are deliberately +// ABSENT: those PRECEDE the фирма („ЕТ Алекс Петров Димитров"), and truncating after them would eat the +// name itself — turning a distinctive ЕТ into a generic one and withholding a true link. +const SUFFIX_FORMS = new Set(['ЕООД', 'ООД', 'ЕАД', 'АД', 'АДСИЦ', 'КДА', 'ДЗЗД']); + +// Cut everything after the LAST фирма-terminating form token. This is the case the comma-peel and the +// marker strip both miss: „ТРЕЙС ГРУП ХОЛД АД София" has neither a comma nor a „гр." dot, so it survived +// both, stopped ending in its form, and defeated every end-anchored form test downstream. Token-exact +// (never a substring), so „КАДИЕВ ГЛОБАЛ ЕООД" and „АД-ХОК ЕООД" are untouched. +function stripAfterSuffixForm(s) { + const re = /[А-ЯЁ]+/gu; + let last = null; + for (let m = re.exec(s); m !== null; m = re.exec(s)) if (SUFFIX_FORMS.has(m[0])) last = m; + return last === null ? s : s.slice(0, last.index + last[0].length).trim(); +} + function stripSeatSuffix(upper) { let s = String(upper).trim(); // Peel trailing comma-clauses right-to-left while the clause bears no legal form (i.e. it's a seat, not @@ -44,7 +63,7 @@ function stripSeatSuffix(upper) { ) { s = m[1].trim(); } - return s.replace(SEAT_MARKER, '').trim(); + return stripAfterSuffixForm(s.replace(SEAT_MARKER, '').trim()); } /** @@ -71,49 +90,55 @@ const norm = (s) => .replace(/[\s.\-–—]+/g, ' ') .trim(); -// Joint-stock / listed legal form (АД / ЕАД / АДСИЦ) as the TRAILING form token. In BG company names the +// Joint-stock / share-issuing legal form (АД / ЕАД / АДСИЦ / КДА) as the TRAILING form token. In BG company names the // legal form is always the suffix, so anchor to the end (optionally followed by quotes/whitespace); a whole // token bounded on the left by string edge, whitespace or quotes — NOT hyphens/dots, so „АД-ХОК ЕООД" (a // hyphenated ООД name) is not misread. Anchoring to the suffix is what stops „АД ГРУП ООД" (an ООД whose // NAME begins with the token „АД") being wrongly excluded as joint-stock — the form there is ООД. -const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ)[\s"„“”«»]*$/u; +// Twinned byte-for-byte by JOINT_SUFFIX in scripts/tr/deed.mjs — the TR parser cannot import from this +// directory without closing a cacbg↔tr cycle. A test in deed.test.mjs pins the two identical; change one +// and that test fails rather than the two silently diverging on a legal form. +export const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; +// The same four forms as whole tokens. Kept in step with JOINT_STOCK above by classify.test.mjs, and with +// deed.mjs's JOINT_SUFFIX twin by deed.test.mjs — three spellings of one rule, all three pinned. +const JOINT_STOCK_FORMS = new Set(['АД', 'ЕАД', 'АДСИЦ', 'КДА']); /** * Materiality by legal form. The public ownership surface is CLOSELY-HELD companies only (ООД/ЕООД/ЕТ/ - * КД/СД/ДЗЗД or a form-unspecified name from the closely-held table). Joint-stock forms (АД/ЕАД/АДСИЦ) are + * КД/СД/ДЗЗД or a form-unspecified name from the closely-held table). Joint-stock forms (АД/ЕАД/АДСИЦ/КДА, + * the last a командитно дружество с акции — it issues shares, so it belongs with them) are * public-float securities — a declared parcel of listed shares is NOT a material ownership conflict, and * presenting it as one defames (the „11 Trace shares → €88M" trap). Excludes only an explicit АД-form token, * so it withholds rather than fabricates. @returns {boolean} true ⇒ material/closely-held. */ export function closelyHeldForm(name) { - return !JOINT_STOCK.test( - stripSeatSuffix( - String(name ?? '') - .normalize('NFC') - .toUpperCase(), - ), - ); + // LAST-FORM-TOKEN-WINS, not an end anchor. The anchor asked „does the name END in a joint-stock form?", + // which a declarant-typed cell can defeat just by appending a seat — and `stripSeatSuffix` cannot be + // trusted to have removed every shape of one. Asking instead „which legal form is the name's LAST?" + // is position-independent: a trailing seat, a stray qualifier, or nothing at all leaves the verdict + // unchanged, while „АД" leading („АД ГРУП ООД") or glued („АД-ХОК ЕООД") still isn't the form. + // + // This is why the predicate no longer uses JOINT_STOCK directly while deed.mjs's twin still does: that + // twin reads the deed envelope's `fullName` — a REGISTRY-clean name that genuinely ends in its form — + // whereas this one reads a free-text cell a human typed. Same rule, different input hygiene. + const tokens = stripSeatSuffix( + String(name ?? '') + .normalize('NFC') + .toUpperCase(), + ) + .split(/[^А-ЯЁ]+/u) + .filter(Boolean); + const lastForm = tokens.filter((t) => FORM_TOKENS.has(t)).at(-1); + // No form token at all ⇒ nothing says joint-stock ⇒ material, exactly as the anchor behaved. The bar + // only ever fires on an EXPLICIT joint-stock form, so it withholds rather than fabricates. + return lastForm === undefined || !JOINT_STOCK_FORMS.has(lastForm); } -/** Seat proof: declared seat and winner settlement both present and equal ⇒ same entity (deterministic). */ -export function seatConfirmed(declSeat, winnerSettlement) { - const a = norm(declSeat); - const b = norm(winnerSettlement); - return a.length > 0 && b.length > 0 && a === b; -} - -/** - * Publish tier for a single-winner-ЕИК match: - * 'A_eik' — a declarant-provided ЕИК resolved the winner: the national unique identifier, - * deterministic even behind a generic/colliding name. Assigned by the loader for - * declared_eik matches (not this function — it needs the match method); ADR-0016. - * 'A_seat' — seat-confirmed: deterministic, publishable even for generic names. - * 'B_distinctive' — single-ЕИК + structurally distinctive name: publishable (disclosed heuristic). - * 'C_hold' — generic name, no seat/ЕИК proof: withhold pending TR name-census. - */ -export function publishTier({ seatOk, distinctiveness }) { - if (seatOk) return 'A_seat'; - return distinctiveness === 'distinctive' ? 'B_distinctive' : 'C_hold'; -} +// seatConfirmed() and publishTier() lived here until #279. The publish tiers they produced +// (A_seat / B_distinctive / C_hold) are superseded by the Trade Register evidence ladder in +// scripts/tr/evidence.mjs — identity now rests on a checkable registry fact rather than on the shape +// of the declared name (ADR-0033, superseding ADR-0009). The seat comparison moved with it, because a +// declared seat is now matched against the REGISTERED seat rather than against the winner row's +// settlement column. nameDistinctiveness above survives, narrowed to an AND-gate on the weakest rung. /** * Temporal relation of a contract to the years a stake was declared (asset decls are annual snapshots). diff --git a/scripts/cacbg/classify.test.mjs b/scripts/cacbg/classify.test.mjs index 7d2bddd15..f0f971ba3 100644 --- a/scripts/cacbg/classify.test.mjs +++ b/scripts/cacbg/classify.test.mjs @@ -2,11 +2,10 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { nameDistinctiveness, - seatConfirmed, - publishTier, temporalStatus, localityToken, closelyHeldForm, + JOINT_STOCK, } from './classify.mjs'; test('nameDistinctiveness: numbers / Latin / ≥3 words are distinctive; bare 1-2 word Cyrillic is generic', () => { @@ -32,19 +31,6 @@ test('nameDistinctiveness: numbers / Latin / ≥3 words are distinctive; bare 1- assert.equal(nameDistinctiveness('„ДОМИНО" ЕООД'), 'generic'); // quoted single word + form }); -test('seatConfirmed: equal non-empty seats confirm; empty or mismatched do not', () => { - assert.equal(seatConfirmed('Шумен', 'ШУМЕН'), true); - assert.equal(seatConfirmed('София', 'Пловдив'), false); - assert.equal(seatConfirmed('', 'София'), false); // sparse winner/declared seat never confirms - assert.equal(seatConfirmed('София', ''), false); -}); - -test('publishTier: seat proof wins; else distinctiveness decides publish vs hold', () => { - assert.equal(publishTier({ seatOk: true, distinctiveness: 'generic' }), 'A_seat'); - assert.equal(publishTier({ seatOk: false, distinctiveness: 'distinctive' }), 'B_distinctive'); - assert.equal(publishTier({ seatOk: false, distinctiveness: 'generic' }), 'C_hold'); -}); - test('temporalStatus: contract within declared-year span is contemporaneous', () => { assert.equal(temporalStatus([2020, 2021, 2022], 2021), 'contemporaneous'); assert.equal(temporalStatus([2020, 2022], 2024), 'after_last_decl'); @@ -71,6 +57,25 @@ test('closelyHeldForm: ООД/ЕООД/ЕТ material; АД/ЕАД/АДСИЦ (li assert.equal(closelyHeldForm('АД СТИЛ ЕООД'), true); }); +test('closelyHeldForm: КДА (командитно дружество с акции) is joint-stock and must be excluded', () => { + // КДА issues shares like an АД — the shareholder book is not public, so a declared parcel is neither + // verifiable nor necessarily material, which is the whole basis of the exclusion. It was missing from + // both JOINT_STOCK and FORM_TOKENS, so a КДА read as closely-held and its holder could be published + // as an owner. #279 rung 1 names it explicitly alongside АД and ЕАД. + assert.equal(closelyHeldForm('ФИНАНС КДА'), false); + assert.equal(closelyHeldForm('"АЛФА ИНВЕСТ" КДА'), false); + assert.equal(closelyHeldForm('АЛФА КДА, гр. София'), false); // seat suffix must not rescue it + // …and the mirror: „КДА" inside a word or as a leading token is not the form. + assert.equal(closelyHeldForm('КДА-ТРЕЙД ЕООД'), true); + assert.equal(closelyHeldForm('КДА ГРУП ООД'), true); +}); + +test('nameDistinctiveness: КДА counts as a legal form, not a content word', () => { + // FORM_TOKENS feeds the content-word count. A form token counted as content inflates distinctiveness, + // which is the direction that publishes prematurely. + assert.equal(nameDistinctiveness('ФИНАНС КДА'), 'generic'); +}); + test('closelyHeldForm: a trailing седалище after the form does not flip an АД to closely-held (libel)', () => { // The declarant appended the seat to the name cell. Without stripping it, the end-anchored form test // misses the АД and returns closely-held=true → a listed-АД parcel presented as a material conflict. @@ -86,6 +91,56 @@ test('closelyHeldForm: a trailing седалище after the form does not flip assert.equal(closelyHeldForm('СТРОЙ, ИНВЕСТ ООД'), true); // trailing clause has ООД → closely-held }); +test('the token set and the JOINT_STOCK regex name the SAME four forms', () => { + // closelyHeldForm now decides on the last form TOKEN while deed.mjs's envelope test still uses the + // end-anchored REGEX. Two spellings of one rule drift silently — the day one gains a form the other + // lacks, a joint-stock company passes one gate and is barred by the other. Pin them to each other. + // (deed.mjs's third spelling is pinned to the regex by deed.test.mjs:378.) + for (const form of ['АД', 'ЕАД', 'АДСИЦ', 'КДА']) { + assert.equal(JOINT_STOCK.test(`ФИРМА ${form}`), true, `regex misses ${form}`); + assert.equal(closelyHeldForm(`ФИРМА ${form}`), false, `token set misses ${form}`); + // …and the mirror: every form the regex accepts must be one the token set bars, seat or no seat. + assert.equal(closelyHeldForm(`ФИРМА ${form} София`), false, `token set misses ${form} + seat`); + } + // Every FORM_TOKEN that is NOT one of the four must read as closely-held. + for (const form of ['ЕООД', 'ООД', 'ЕТ', 'ДЗЗД', 'КД', 'СД']) { + assert.equal(closelyHeldForm(`ФИРМА ${form}`), true, `${form} wrongly barred`); + } +}); + +test('closelyHeldForm: a seat with NO comma and NO „гр." marker still cannot flip an АД (libel)', () => { + // The gap the marker/comma rules leave open. `SEAT_MARKER` requires a literal dot and the comma-peel + // requires a comma, so „ТРЕЙС ГРУП ХОЛД АД София" — the plainest way a declarant writes it — survives + // both, no longer ENDS in the form, and the end-anchored JOINT_STOCK test misses it. A listed АД then + // reads as closely-held: the „11 акции на Trace → €88M" trap, from the one input shape nothing strips. + assert.equal(closelyHeldForm('ТРЕЙС ГРУП ХОЛД АД София'), false); + assert.equal(closelyHeldForm('ТРЕЙС ГРУП ХОЛД АД СОФИЯ'), false); + assert.equal(closelyHeldForm('Транспроект ЕАД Пловдив'), false); + assert.equal(closelyHeldForm('НЕС АДСИЦ Варна'), false); + assert.equal(closelyHeldForm('АЛФА КДА София'), false); + // POSITIVE CONTROLS — the bar must stay a bound, not become a blanket. A predicate that always returned + // false would pass every assertion above; these are what distinguish the fix from that (ADR-0027). + assert.equal(closelyHeldForm('Вамос ООД Русе'), true); // dot-less seat on a closely-held form + assert.equal(closelyHeldForm('ЕНЕРДЖИ СЪПЛАЙ ЕООД Бургас'), true); + assert.equal(closelyHeldForm('АД ГРУП ООД'), true); // leading „АД" is not the form + assert.equal(closelyHeldForm('АД-ХОК ЕООД'), true); // „АД" glued by a hyphen is not a form token + assert.equal(closelyHeldForm('КДА ГРУП ООД'), true); + assert.equal(closelyHeldForm('КАДИЕВ ГЛОБАЛ ЕООД'), true); // „АД" inside a word + assert.equal(closelyHeldForm('ЕТ Алекс'), true); // ЕТ leads the фирма — nothing after it is a seat +}); + +test('nameDistinctiveness: a dot-less trailing city is not counted as a content word either', () => { + // Same blind spot, and here it fails toward PUBLISHING: an uncounted seat token inflates the content-word + // count to 3 ⇒ 'distinctive'. Since #279 rung 2 gates an uncorroborated „Документ" publish on exactly this + // predicate (ADR-0035), a seat read as a content word is a false company-identity claim, not just noise. + assert.equal(nameDistinctiveness('СТРОЙ ИНВЕСТ ООД София'), 'generic'); + assert.equal(nameDistinctiveness('НИКАС КОМЕРС ЕООД Пловдив'), 'generic'); + // POSITIVE CONTROLS: a genuinely ≥3-content-word фирма stays distinctive, and a leading-form ЕТ name + // keeps its content words — nothing after „ЕТ" is a seat, so the strip must not reach them. + assert.equal(nameDistinctiveness('ХИДРО СТРОЙ МОНТАЖ ЕООД София'), 'distinctive'); + assert.equal(nameDistinctiveness('ЕТ АЛЕКС ПЕТРОВ ДИМИТРОВ'), 'distinctive'); +}); + test('nameDistinctiveness: a trailing city is not counted as a content word (no premature publish)', () => { // The exact over-publish case: 2 real content words + a seat token would read as 3 → distinctive. assert.equal(nameDistinctiveness('СТРОЙ ИНВЕСТ ООД, СОФИЯ'), 'generic'); // was distinctive via +СОФИЯ diff --git a/scripts/cacbg/guard.mjs b/scripts/cacbg/guard.mjs index 97a326bda..9502ac963 100644 --- a/scripts/cacbg/guard.mjs +++ b/scripts/cacbg/guard.mjs @@ -9,8 +9,15 @@ import { fileURLToPath } from 'node:url'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); export const SCRATCH = path.join(ROOT, 'scratch', 'cacbg'); -export function assertScratchIgnored() { - const probe = path.join('scratch', 'cacbg', '.probe'); +/** + * Assert that a `scratch/` tree is git-ignored, before anything writes PII into it. + * Parameterised rather than copied: the Trade Register leg needs the identical rail for its deed + * cache (owner names, company addresses — ADR-0033 decision 5), and a second copy of a safety rail + * drifts from the original. Existing no-argument callers are unaffected. + * @param {string} [subdir] directory under scratch/ to probe + */ +export function assertScratchIgnored(subdir = 'cacbg') { + const probe = path.join('scratch', subdir, '.probe'); try { execFileSync('git', ['check-ignore', '-q', probe], { cwd: ROOT }); } catch { diff --git a/scripts/cacbg/link-corrections.jsonl b/scripts/cacbg/link-corrections.jsonl new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/cacbg/load-ambiguous.test.mjs b/scripts/cacbg/load-ambiguous.test.mjs index 336cfb307..01cf8f7ad 100644 --- a/scripts/cacbg/load-ambiguous.test.mjs +++ b/scripts/cacbg/load-ambiguous.test.mjs @@ -45,12 +45,33 @@ function buildAndLoad(bidderRows) { fs.writeFileSync(path.join(STAGING, 'holdings.jsonl'), ''); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + // An EMPTY Trade Register cache, pointed at explicitly (#279, ADR-0033). This fixture declares no + // holdings, so it resolves no links and the coverage gate is satisfied by an empty candidate set — + // but the cache FILE must still exist, because a missing one refuses the whole load. + // + // Explicit rather than defaulted, and that matters: load.mjs falls back to the repo's real + // scratch/tr/tr-cache.sqlite, so a test that omits TR_CACHE_DB silently runs against whatever the + // developer's last live crawl left behind. That is exactly how this test passed locally and failed + // in CI, where no such file exists. + const trDb = path.join(dir, 'tr-cache.sqlite'); + new DatabaseSync(trDb).close(); + let threw = false; try { execFileSync( 'node', ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], - { cwd: ROOT, env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, stdio: 'pipe' }, + { + cwd: ROOT, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: trDb, + TR_RAW_DIR: path.join(dir, 'tr-deeds'), + }, + stdio: 'pipe', + }, ); } catch { threw = true; // execFileSync throws on a non-zero exit code (a fired hard-gate) diff --git a/scripts/cacbg/load-collision.test.mjs b/scripts/cacbg/load-collision.test.mjs index 32991193b..43fb1df44 100644 --- a/scripts/cacbg/load-collision.test.mjs +++ b/scripts/cacbg/load-collision.test.mjs @@ -16,7 +16,7 @@ import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad() { execFileSync( @@ -24,15 +24,93 @@ function runLoad() { ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], { cwd: ROOT, - env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + }, stdio: 'pipe', }, ); } + +/** + * Minimal Trade Register evidence for this fixture (#279, ADR-0033). Publishing now rests on a registry + * fact, so a loader test without a cache would only ever exercise the fail-closed path. Each winner's + * deed names its own declarant as съдружник, which is the „Документ" rung. + */ +function buildTrCache(owners) { + fs.mkdirSync(TR_RAW, { recursive: true }); + const cache = new DatabaseSync(TR_DB); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + for (const [eik, spec] of Object.entries(owners)) { + const { name, seat } = spec; + const deed = { + uic: eik, + fullName: '"ФИКС" ЕООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: `

${name}

`, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + // The REGISTERED seat, matching what this official declared. Both фирми here are + // generic („КОМПАНИЯ ЕДНО/ДВЕ" — two content words), so under ADR-0035 a name match + // alone cannot establish which company was declared; the agreeing seat is what does. + // Without it this fixture would exercise the withholding path instead of the + // cross-folder attribution it exists to test. + { + nameCode: 'CR_F_5_L', + htmlData: `

Държава: БЪЛГАРИЯ
Населено място: гр. ${seat}

`, + fieldEntryNumber: '20110502101008', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], + }; + fs.writeFileSync(path.join(TR_RAW, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code,legal_form_verdict,latest_own_entry_date) VALUES(?,?,?,?,?,?,?,?)', + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + 4, + 'closely_held', + '2011-05-02', + ); + } + cache.close(); +} const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-collision-')); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); fs.mkdirSync(STAGING, { recursive: true }); @@ -45,7 +123,7 @@ before(() => { CREATE TABLE contracts(id TEXT PRIMARY KEY, tender_id TEXT, bidder_id TEXT, signed_at TEXT, amount_eur REAL); INSERT INTO authorities VALUES ('auth:1','ВЕДОМСТВО ТЕСТ'); INSERT INTO tenders VALUES ('t1','auth:1'),('t2','auth:1'); - -- Two distinct seat-confirmed single-ЕИК winners → both publish (A_seat), no ambiguity. + -- Two distinct single-ЕИК winners, each named in its own deed AND seat-corroborated → both publish. INSERT INTO bidders VALUES ('eik:100000001','КОМПАНИЯ ЕДНО ЕООД','100000001',1,'София'); INSERT INTO bidders VALUES ('eik:200000002','КОМПАНИЯ ДВЕ ЕООД','200000002',1,'Пловдив'); INSERT INTO contracts VALUES ('c1','t1','eik:100000001','2021-05-01',50000); @@ -92,6 +170,11 @@ before(() => { holdings.map((h) => JSON.stringify(h)).join('\n') + '\n', ); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + + buildTrCache({ + 100000001: { name: 'ИВАН ПЪРВИ ТЕСТОВ', seat: 'София' }, + 200000002: { name: 'ПЕТЪР ВТОРИ ПРОБЕН', seat: 'Пловдив' }, + }); }); after(() => fs.rmSync(dir, { recursive: true, force: true })); diff --git a/scripts/cacbg/load-divestment-nonwinner.test.mjs b/scripts/cacbg/load-divestment-nonwinner.test.mjs index 226be07c4..fac967b58 100644 --- a/scripts/cacbg/load-divestment-nonwinner.test.mjs +++ b/scripts/cacbg/load-divestment-nonwinner.test.mjs @@ -17,7 +17,7 @@ import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad() { execFileSync( @@ -25,15 +25,86 @@ function runLoad() { ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], { cwd: ROOT, - env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + }, stdio: 'pipe', }, ); } + +/** + * Minimal Trade Register evidence for this fixture (#279, ADR-0033). Publishing now rests on a registry + * fact, so a loader test without a cache would only ever exercise the fail-closed path. Each winner's + * deed names its own declarant as съдружник — the „Документ" rung. + */ +function buildTrCache(owners) { + fs.mkdirSync(TR_RAW, { recursive: true }); + const cache = new DatabaseSync(TR_DB); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + for (const [eik, names] of Object.entries(owners)) { + const html = [] + .concat(names) + .map((n) => `

${n}

`) + .join(`
`); + const deed = { + uic: eik, + fullName: '"ФИКС" ЕООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: html, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], + }; + fs.writeFileSync(path.join(TR_RAW, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code,legal_form_verdict,latest_own_entry_date) VALUES(?,?,?,?,?,?,?,?)', + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + 4, + 'closely_held', + '2011-05-02', + ); + } + cache.close(); +} + const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-divest-')); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); fs.mkdirSync(STAGING, { recursive: true }); @@ -64,7 +135,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Диан Дивестов', + person: 'Диан Иванов Дивестов', position: '', entity: 'ДИВ ТЕХ 5 ЕООД', kind: 'shares', @@ -80,7 +151,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Диан Дивестов', + person: 'Диан Иванов Дивестов', position: '', entity: 'НЕПОБЕДИМ КОМПАНИ ООД', kind: 'shares', @@ -99,7 +170,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Верен Държателев', + person: 'Верен Иванов Държателев', position: '', entity: 'ДРУГ ВИН 6 ЕООД', kind: 'shares', @@ -114,6 +185,13 @@ before(() => { holdings.map((h) => JSON.stringify(h)).join('\n') + '\n', ); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + + buildTrCache({ + // Диан DIVESTED, so the live deed must name somebody else — otherwise §7's reconciliation + // correctly overturns his declared termination and the case stops testing what it is for. + 100000001: 'НОВ ИВАНОВ СОБСТВЕНИК', + 200000002: 'ВЕРЕН ИВАНОВ ДЪРЖАТЕЛЕВ', + }); // filings.jsonl — one record per declaration (as extract.mjs emits it), carrying the declaration type. The // divest horizon is built from this: Диан's 2022 assets declaration (listing only the non-winner) advances // his assets horizon to 2022 → the 2019 ДИВ ТЕХ 5 winner stake is withdrawn. Верен has only a 2019 filing. @@ -143,8 +221,8 @@ test('a later NON-winner ownership filing still withdraws a divested winner stak ) .get(eik, person); - const dian = link('100000001', 'Диан Дивестов'); - const veren = link('200000002', 'Верен Държателев'); + const dian = link('100000001', 'Диан Иванов Дивестов'); + const veren = link('200000002', 'Верен Иванов Държателев'); // The divested winner stake is dated to its last declaration and excluded from the public surface. assert.equal(dian.status, 'withdrawn'); diff --git a/scripts/cacbg/load-homonym.test.mjs b/scripts/cacbg/load-homonym.test.mjs index 395b62edb..7f38b12bf 100644 --- a/scripts/cacbg/load-homonym.test.mjs +++ b/scripts/cacbg/load-homonym.test.mjs @@ -18,19 +18,94 @@ import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad() { execFileSync( 'node', ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], - { cwd: ROOT, env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, stdio: 'pipe' }, + { + cwd: ROOT, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + }, + stdio: 'pipe', + }, ); } + +/** + * Minimal Trade Register evidence for this fixture (#279, ADR-0033). Publishing now rests on a registry + * fact, so a loader test without a cache would only ever exercise the fail-closed path. Each winner's + * deed names its own declarant as съдружник — the „Документ" rung. + */ +function buildTrCache(owners) { + fs.mkdirSync(TR_RAW, { recursive: true }); + const cache = new DatabaseSync(TR_DB); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + for (const [eik, names] of Object.entries(owners)) { + const html = [] + .concat(names) + .map((n) => `

${n}

`) + .join(`
`); + const deed = { + uic: eik, + fullName: '"ФИКС" ЕООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: html, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], + }; + fs.writeFileSync(path.join(TR_RAW, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code,legal_form_verdict,latest_own_entry_date) VALUES(?,?,?,?,?,?,?,?)', + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + 4, + 'closely_held', + '2011-05-02', + ); + } + cache.close(); +} + const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-homonym-')); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); fs.mkdirSync(STAGING, { recursive: true }); @@ -59,7 +134,7 @@ before(() => { template: 'assets', category: '', institution: 'ОБЩИНА СОФИЯ', - person: 'Георги Иванов', + person: 'Георги Иванов Петров', position: 'Кмет', entity: 'ВИН ЕДНО 5 ЕООД', kind: 'shares', @@ -77,7 +152,7 @@ before(() => { template: 'assets', category: '', institution: 'МИНИСТЕРСТВО НА ТЕСТА', - person: 'Георги Иванов', + person: 'Георги Иванов Петров', position: 'Директор', entity: 'ВИН ДВЕ 6 ЕООД', kind: 'shares', @@ -95,7 +170,7 @@ before(() => { template: 'assets', category: '', institution: 'ОБЩИНА СОФИЯ', - person: 'Георги Иванов', + person: 'Георги Иванов Петров', position: 'Кмет', entity: 'ВИН ЕДНО 5 ЕООД', kind: 'shares', @@ -110,6 +185,11 @@ before(() => { holdings.map((h) => JSON.stringify(h)).join('\n') + '\n', ); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + + buildTrCache({ + 100000001: 'ГЕОРГИ ИВАНОВ ПЕТРОВ', + 200000002: 'ГЕОРГИ ИВАНОВ ПЕТРОВ', + }); }); after(() => fs.rmSync(dir, { recursive: true, force: true })); @@ -120,7 +200,7 @@ test('same-named officials at different institutions do NOT merge into one perso // Two DISTINCT persons named „Георги Иванов" — one per institution — never one merged identity. const persons = db - .prepare("SELECT id, name FROM persons WHERE name = 'Георги Иванов' ORDER BY id") + .prepare("SELECT id, name FROM persons WHERE name = 'Георги Иванов Петров' ORDER BY id") .all(); assert.equal(persons.length, 2, 'two distinct namesake officials, not one merged person'); assert.notEqual(persons[0].id, persons[1].id); diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 4dc360b28..ca7c030c5 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -14,24 +14,65 @@ import { DatabaseSync } from 'node:sqlite'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +// nameDistinctiveness is deliberately NOT imported: the evidence ladder replaced it in the publish +// path (ADR-0033). It survives in classify.mjs for the review queue, not for a publishing decision. import { - nameDistinctiveness, - seatConfirmed, - publishTier, temporalStatus, localityToken, closelyHeldForm, + nameDistinctiveness, } from './classify.mjs'; +import { openCache, readDeed, coverage } from '../tr/cache.mjs'; +import { TR_DB, TR_RAW, deedPath } from '../tr/paths.mjs'; +import { + evidenceVerdict, + isSealedFact, + reconcileTermination, + RULES_VERSION, +} from '../tr/evidence.mjs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; -import { fingerprint, loadSuppressions, SUPPRESSION_KEY_VERSION } from './suppressions.mjs'; +import { + fingerprint, + loadCorrections, + loadSuppressions, + SUPPRESSION_KEY_VERSION, +} from './suppressions.mjs'; import { canonicalInstitution } from './institutions.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); const STAGING = process.env.CACBG_STAGING || path.join(ROOT, 'scratch/cacbg/staging'); const MIGRATION = path.join(ROOT, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// 0006 attaches the Trade Register evidence seal (#279, ADR-0033). Applied here as well as 0003 +// because this loader rebuilds the CACBG tables from the migrations on every run — a seal table +// missing from the work DB would make every evidence write fail at ship time instead of at load. +const MIGRATION_EVIDENCE = path.join( + ROOT, + 'packages/db/migrations/0009_interest_link_evidence.sql', +); const REPORT = path.join(STAGING, 'findings.md'); -const MATCHER_VERSION = 'cnk-1+classify-1'; // bump when the normalizer or classify logic changes +// Bumped for #279: classify-2 (КДА added to the joint-stock bar) + tr-1 (identity now rests on a +// Trade Register fact, not on name distinctiveness). RULES_VERSION versions the EVIDENCE rules +// separately — §8's monotonicity gate keys on that one, not on this. +const MATCHER_VERSION = 'cnk-1+classify-2+tr-1'; +const TR_CACHE_DB = process.env.TR_CACHE_DB || TR_DB; +const TR_RAW_DIR = process.env.TR_RAW_DIR || TR_RAW; +// A deliberate, logged override for the coverage gate below. Without it a single permanently +// unreachable ЕИК would deadlock the pipeline forever; with it, the operator states that they know. +const ALLOW_PARTIAL_TR = process.argv.includes('--allow-partial-tr'); +// Bootstrap mode: write the crawl's input list and stop, successfully. The decision run and the register +// crawl now share one job (they must — the raw deeds hold third-party names and cannot travel between +// runners), and that job has to be able to start from nothing: the list is derived from the resolved +// corpus, so only this script can produce it, but the full run refuses without the very cache the list +// is used to fill. Ignoring the refusal's exit code instead would erase the difference between „no cache +// yet" and „this run is broken". +// +// POINT THIS AT A SCRATCH COPY OF THE WORK DB. It is not a read-only pass: reaching the candidate list +// means rebuilding the corpus tables, so it drops and repopulates persons/declarations/declared_interests +// and leaves interest_links EMPTY. Empty is the safe end state (the ship floor refuses it, and no link +// can be published without evidence it never gathered), but it is not the state a subsequent real run +// should inherit. The one thing it must never touch either way is the monotonicity snapshot — see below. +const EMIT_CANDIDATES_ONLY = process.argv.includes('--emit-candidates'); const { companyNameKey, isMatchableKey } = await import('../../packages/shared/src/company-name-key.ts'); @@ -55,7 +96,24 @@ const readJsonl = (f) => .map((l) => JSON.parse(l)) : []; -const db = new DatabaseSync(DB); +// Bootstrap mode works on a THROWAWAY COPY, and that is not a convenience — it is the correctness of the +// monotonicity gate. Reaching the candidate list means rebuilding the corpus tables, which drops +// interest_links; the pass itself never publishes, so it would leave the table EMPTY. The next real run +// would then read that empty table as its prior-published set, write an empty snapshot, and the gate — +// whose only job is to notice a published claim disappearing — would pass unconditionally, for ever. +// Copying here rather than asking the caller to do it keeps the flag safe wherever it is invoked from. +const WORK_DB = EMIT_CANDIDATES_ONLY ? `${DB}.bootstrap` : DB; +if (EMIT_CANDIDATES_ONLY) { + for (const suffix of ['', '-wal', '-shm']) { + // -wal/-shm may legitimately be absent (a cleanly closed DB has neither); anything else must surface. + try { + fs.copyFileSync(`${DB}${suffix}`, `${WORK_DB}${suffix}`); + } catch (e) { + if (e.code !== 'ENOENT') throw e; + } + } +} +const db = new DatabaseSync(WORK_DB); db.exec('PRAGMA foreign_keys=ON'); // Suppressions live in a VERSION-CONTROLLED, HMAC-fingerprinted list (ADR-0031), NOT a DB table — so a // takedown survives a fresh-CI-runner rebuild and never ships the „who was taken down" signal to prod. @@ -68,9 +126,91 @@ const suppressedFp = new Set(suppEntries.map((e) => e.fp)); // B3 unused-suppression gate: every listed fingerprint MUST match exactly one built link. Track which get // used; a fingerprint that matched nothing (a stale/mis-keyed takedown) fails the build after the load loop. const usedSuppressions = new Set(); +// Export the CURRENT published surface before anything is dropped — ADR-0033 decision 6. +// +// The rebuild below is total, so the previous run's published set exists only in this instant. The +// audit compares against this file and hard-fails on a link that was published last run and is not +// published now under an UNCHANGED rules_version: nothing licensed that removal, so it is a silent +// recall regression. rules_version travels per key, because a link that vanished under a rules BUMP is +// an intentional removal and must degrade to a printed diff instead. +// +// Held and withdrawn links are deliberately excluded: they were never a public claim, so their absence +// next run is not a regression. On a first run the tables do not exist yet and the export is an empty +// set — written, not skipped, so a missing file means „the loader never ran", not „nothing published". +const priorPublished = (() => { + try { + return db + .prepare( + `SELECT il.link_key AS link_key, e.rules_version AS rules_version + FROM interest_links il + LEFT JOIN interest_link_evidence e ON e.link_key = il.link_key + WHERE il.status = 'published'`, + ) + .all() + .map((r) => ({ link_key: r.link_key, rules_version: r.rules_version ?? RULES_VERSION })); + } catch (e) { + // A first run: interest_links does not exist yet. Any OTHER failure must surface — swallowing it + // would turn a broken export into a permanently silent gate. + if (!/no such table/i.test(e.message)) throw e; + return []; + } +})(); +// Decision 6's SECOND sanctioned removal: „a correction of wrong input". A link whose input was wrong +// should never have been published, but correcting the input UNBUILDS it — so a suppression on it +// would match no built link and trip the B3 gate above, while doing nothing leaves a permanent hard +// finding. The acknowledgement is therefore recorded here, against the set the gate actually compares: +// each prior-published key whose fingerprint is listed is exported flagged, and audit.mjs reads the +// flag as a declared removal. Fingerprinted for ADR-0031's reason — `pid|eik` in git would record which +// named official was tied to which company for ever. +const CORRECTIONS_LIST = + process.env.CACBG_CORRECTIONS_LIST || path.join(ROOT, 'scripts/cacbg/link-corrections.jsonl'); +const correctedFp = new Set( + loadCorrections(CORRECTIONS_LIST, SUPP_SALT, SUPPRESSION_KEY_VERSION).map((e) => e.fp), +); +const usedCorrections = new Set(); +const snapshot = priorPublished.map((p) => { + if (correctedFp.size === 0) return p; + const fp = fingerprint(p.link_key, SUPP_SALT); + if (!correctedFp.has(fp)) return p; + usedCorrections.add(fp); + return { ...p, corrected: true }; +}); +// The B3 rail, mirrored — and it matters MORE here. A stale suppression silently un-suppresses; a +// stale acknowledgement silently pre-clears a FUTURE disappearance of that same link, which is exactly +// the regression the gate exists to catch, with nobody having decided it. An acknowledgement is +// one-shot by construction: once the corrected link stops being published it also stops appearing in +// the prior set, so the entry must be deleted from the list in the same change that lands the fix. +if (!EMIT_CANDIDATES_ONLY) { + const unusedCorr = [...correctedFp].filter((fp) => !usedCorrections.has(fp)); + if (unusedCorr.length > 0) { + db.close(); + throw new Error( + `${unusedCorr.length} correction(s) matched NO previously published link — a stale acknowledgement ` + + `would clear a future disappearance of that link, which is the regression this gate exists to ` + + `catch. Delete the entry once its fix has shipped (an acknowledgement is one-shot). ` + + `Unmatched fingerprints: ${unusedCorr.map((f) => f.slice(0, 12) + '…').join(', ')}`, + ); + } +} +// Not written by the bootstrap pass, which works on a throwaway copy and has no business restating what +// the real run is about to record. +// +// tmp + rename, not a bare write: a crash mid-write leaves a truncated file, and audit.mjs deliberately +// does NOT swallow a parse failure here (only ENOENT is a legitimate first run). A torn snapshot would +// therefore wedge every subsequent audit until a human cleared the file by hand. The raw deeds already +// land this way; the gate's own input deserves the same. +if (!EMIT_CANDIDATES_ONLY) { + const snapPath = path.join(STAGING, 'published-snapshot.json'); + const snapTmp = `${snapPath}.tmp`; + fs.writeFileSync(snapTmp, JSON.stringify(snapshot, null, 2) + '\n'); + fs.renameSync(snapTmp, snapPath); +} + // Full idempotent rebuild that also picks up schema changes: drop the CACBG tables (children first — // FK-safe) and re-apply the migration. Nothing to preserve — suppressions are external now. for (const t of [ + // FIRST: interest_link_evidence references interest_links, so it must go before its parent. + 'interest_link_evidence', 'interest_link_authorities', 'interest_links', 'declared_interests', @@ -80,6 +220,7 @@ for (const t of [ ]) db.exec(`DROP TABLE IF EXISTS ${t}`); db.exec(fs.readFileSync(MIGRATION, 'utf8')); +db.exec(fs.readFileSync(MIGRATION_EVIDENCE, 'utf8')); // A link is suppressed when its fingerprint is in the list. Only compute the HMAC when the list is // non-empty (size>0 ⇒ salt present, else the loader above threw), so the empty common path skips crypto. const isSuppressed = (linkKey) => { @@ -219,14 +360,41 @@ const familyMaterialByTemplate = new Map(); // declare a stake ONLY in the interests declaration). One record per declaration incl. empty / no-material ones, // so a same-type divest-to-ZERO still advances the horizon. An absent/typeless file just yields no match ⇒ the // link is kept (fail-safe: never withdraw on missing evidence). +// +// A filing whose is unreadable falls back to its FOLDER year (#279 §1.3). Dropping the record +// instead — the previous behaviour — is not the fail-safe it resembles: an undated filing that vanishes +// never advances the horizon, so `divested` stays false and a stake the official has since SOLD keeps +// naming them on the public surface. That is a stale claim about a real person, which is the failure this +// surface can least afford. The folder year is an APPROXIMATION — it is the publication year and runs +// ahead of the declared year (migration 0003) — so it can advance the horizon by up to a year early. That +// errs toward WITHDRAWING a claim we are no longer sure of, which is the safe direction here. +// +// A filing datable by NEITHER field is still ignored: the fallback dates a filing, it does not invent one. const filingMaxByPersonType = new Map(); +let filingFolderDated = 0, + filingUndatable = 0; for (const f of readJsonl(path.join(STAGING, 'filings.jsonl'))) { if (!isMatchableKey(companyNameKey(f.person))) continue; - const fy = yr(f.year); - if (!Number.isFinite(fy)) continue; + let fy = yr(f.year); + if (!Number.isFinite(fy)) { + fy = yr(f.folder); + if (!Number.isFinite(fy)) { + // Counted, not silently dropped. A horizon we failed to advance is invisible in the output — the + // link simply stays up — so without this the only symptom of a corpus-wide date regression would be + // a surface that quietly stopped withdrawing anything. + filingUndatable++; + continue; + } + filingFolderDated++; + } const k = `${personId(f.person, f.institution)}|${f.template ?? ''}`; filingMaxByPersonType.set(k, Math.max(filingMaxByPersonType.get(k) ?? fy, fy)); } +if (filingFolderDated > 0 || filingUndatable > 0) { + console.log( + ` filings: ${filingFolderDated} dated by FOLDER (unreadable ), ${filingUndatable} undatable (ignored — no horizon)`, + ); +} db.exec('BEGIN'); for (const h of readJsonl(path.join(STAGING, 'holdings.jsonl'))) { @@ -384,6 +552,12 @@ for (const r of readJsonl(path.join(STAGING, 'related.jsonl'))) { db.exec('COMMIT'); // --- enrich each (person,eik) → interest_links (+ per-authority breakdown) ----------------------- +// THE WRITER of contract_count / contract_value_eur. Its join shape (contracts→tenders→authorities→ +// bidders) is mirrored by CONTRACT_JOIN in packages/db/src/queries/related-persons.ts, so that the +// read-time subset can never exceed what was stored. `authorities` here IS projected (the per-authority +// breakdown needs the name), unlike on the read side where it looks dead — that asymmetry is why the two +// have to be pinned to each other rather than reasoned about separately. related-persons-sql.test.ts +// asserts they agree; change one only by changing both, and re-baseline ADR-0033 §10 when you do. const contractStmt = db.prepare( "SELECT strftime('%Y', c.signed_at) yr, a.id auth_id, a.name authority, c.amount_eur eur FROM contracts c JOIN tenders t ON t.id=c.tender_id JOIN authorities a ON a.id=t.authority_id JOIN bidders b ON b.id=c.bidder_id WHERE b.eik_normalized=?", ); @@ -393,6 +567,12 @@ const insLink = db.prepare( const insILA = db.prepare( 'INSERT OR IGNORE INTO interest_link_authorities(link_key,authority_id,authority_name,contract_count,value_eur,own) VALUES(?,?,?,?,?,?)', ); +// The evidence seal (#279 §8, migration 0006). Written for EVERY link, not only published ones — the +// seals on held and withdrawn links are what let the review queue explain itself. `matched_fact` is a +// closed vocabulary and must NEVER carry a name; the audit enforces that with a pattern check. +const insEvidence = db.prepare( + 'INSERT OR REPLACE INTO interest_link_evidence(link_key,evidence_kind,registry_role,matched_fact,entry_number,entry_date,lookup_date,rules_version,live_status) VALUES(?,?,?,?,?,?,?,?,?)', +); // classify one authority (whose name may be a ';'-joined blob) against the official's institutions. // exact = deterministic name equality; name_contains/locality = DISCLOSED heuristics (candidate, not proof). const OWN_RANK = { exact: 3, name_contains: 2, locality: 1, none: 0 }; @@ -416,6 +596,90 @@ function authOwn(authorityName, instNorms, instNormsLong, locTokens) { } // Distinct officials who declared each company (ЕИК). A private interest has ONE owner-declarant; a // public body's board is declared by MANY rotating members — the deterministic ex-officio tell (ADR-0019). +// ── Trade Register evidence: the candidate set, the fail-closed gate, and the deed reader ───────── +// Identity now rests on a checkable registry fact rather than on the shape of the declared name +// (#279, ADR-0033). Two consequences the loader has to enforce, both fail-closed: +// +// 1. NO cache ⇒ throw. Publishing without evidence is precisely what this change abolishes. +// 2. PARTIAL cache ⇒ throw. This is the silent one. An 80%-restored cache yields roughly 80 +// published links, which is ABOVE ship-related-persons.mjs's floor of 50 — so it would sail +// through that guard, ship a decimated surface, and wipe the rest of the live links. +// +// The candidate set is every resolved ЕИК across ALL aggregates, not just the ones that end up +// published: a link held for want of evidence still needs its deed to say so. +const candidateEiks = [...new Set([...agg.values()].map((r) => r.eik))].sort(); +fs.writeFileSync(path.join(STAGING, 'candidate-eiks.txt'), candidateEiks.join('\n') + '\n'); + +if (EMIT_CANDIDATES_ONLY) { + // Stop BEFORE the TR gate and before anything is written to the domain. A bootstrap pass that built + // links would leave a surface resting on no evidence at all, and a failure between this pass and the + // real one would leave that surface sitting in the work DB, shippable. + console.log( + `${candidateEiks.length} candidate ЕИК written for the crawler; stopping (--emit-candidates)`, + ); + db.close(); + for (const suffix of ['', '-wal', '-shm']) fs.rmSync(`${WORK_DB}${suffix}`, { force: true }); // the copy has served its purpose + process.exit(0); +} + +if (!fs.existsSync(TR_CACHE_DB)) { + db.close(); + throw new Error( + `REFUSE TO LOAD: no Trade Register cache at ${TR_CACHE_DB}. Every publishing decision now rests ` + + `on a registry fact (ADR-0033); without the cache there is no evidence to rest on. Run ` + + `scripts/tr/fetch-deeds.mjs --eiks-file ${path.join(STAGING, 'candidate-eiks.txt')} first.`, + ); +} +const trCache = openCache(TR_CACHE_DB); +const trCoverage = coverage(trCache, candidateEiks); +console.log( + `TR cache: ${trCoverage.covered}/${trCoverage.wanted} covered ` + + `(fetched ${trCoverage.fetched}, outside ТР ${trCoverage.outsideTr}, missing ${trCoverage.missing})`, +); +if (trCoverage.missing > 0 && !ALLOW_PARTIAL_TR) { + trCache.close(); + db.close(); + throw new Error( + `REFUSE TO LOAD: the Trade Register cache covers ${trCoverage.covered} of ${trCoverage.wanted} ` + + `candidate ЕИК. A partial cache does not fail loudly downstream — it publishes a decimated ` + + `surface that still clears the ship floor and then wipes the rest of the live links. Finish ` + + `the crawl, or pass --allow-partial-tr to state that a smaller surface is intended.`, + ); +} + +// Deeds are read from git-ignored scratch and cached in memory for the run. The parsed deed carries +// third-party names; they are used ONLY inside evidenceVerdict's boolean comparisons and never reach +// a column, a log line or the report (ADR-0033 decision 5). +// The lookup date sealed on every link: when the evidence was gathered, not when it was interpreted. +// It is the freshness bound the methodology page has to state, so it comes from the cache rather than +// from `now` — a re-run over an unchanged cache must not make the evidence look fresher than it is. +const trLookupDate = (() => { + const row = trCache.prepare('SELECT MAX(fetched_at) m FROM deeds').get(); + return row?.m ? String(row.m).slice(0, 10) : new Date().toISOString().slice(0, 10); +})(); + +const deedCache = new Map(); +function deedFor(eik) { + if (deedCache.has(eik)) return deedCache.get(eik); + const row = readDeed(trCache, eik); + let entry; + if (!row) entry = { deed: null, outsideTr: false, missing: true }; + else if (row.status === 'outside_tr') entry = { deed: null, outsideTr: true, missing: false }; + else { + // RE-DERIVED from the ЕИК through safeEik, never the stored raw_path. The cache index is written by + // the crawler but travels between runs — and once it does, a stored path is attacker-influenced + // input joined straight onto a filesystem root, which is a traversal read. purgeExpired already + // re-derives for exactly this reason; this was the one read that did not. deedPath also throws on a + // malformed ЕИК rather than quietly reading some other company's deed (R8). + const file = deedPath(eik, TR_RAW_DIR); + entry = fs.existsSync(file) + ? { deed: JSON.parse(fs.readFileSync(file, 'utf8')), outsideTr: false, missing: false } + : { deed: null, outsideTr: false, missing: true }; + } + deedCache.set(eik, entry); + return entry; +} + const declarantsByEik = new Map(); for (const rec of agg.values()) { if (rec.scope !== 'self') continue; // ex-officio tell counts SELF declarants of a public board only @@ -460,7 +724,20 @@ for (const rec of agg.values()) { a.count++; if (r.eur != null) a.value += r.eur; } - const seatOk = [...rec.seats].some((s) => seatConfirmed(s, rec.bidder.settlement)); + // ── the evidence ladder replaces the publish tiers (ADR-0033 decision 1) ──────────────────────── + // rec.seats is keyed on `pid|eik|scope`, so it already holds ONLY the seats this person declared for + // THIS company — which is what #279 §5 rung 3 requires: 4.9% of company-name keys carry more than one + // distinct declared seat, so a company-only key would let one person's seat confirm another's link. + const { deed, outsideTr, missing } = deedFor(rec.eik); + if (missing && !ALLOW_PARTIAL_TR) { + // Unreachable via the coverage gate above; kept as a belt-and-braces refusal so a future change + // that loosens the gate cannot silently publish a link with no evidence behind it. + trCache.close(); + db.close(); + throw new Error( + `no cached deed for ЕИК ${rec.eik} — the coverage gate should have caught this`, + ); + } // A declarant-provided ЕИК is the national unique identifier (ЗТРРЮЛНЦ) — it resolves the winner // deterministically even behind a generic or winner-colliding name, so a declared_eik match publishes // on its own basis (A_eik), never held for name-genericness. This is at least as certain as the seat @@ -468,14 +745,60 @@ for (const rec of agg.values()) { // Name-only methods (exact_name_key / extracted_name) still ride the distinctiveness/seat gate below: // a globally non-unique winner name (e.g. „Водоснабдяване и канализация ЕАД" → 2 valid ЕИК in different // towns) can never be name-distinctive, so it publishes only if the declared SEAT disambiguates, else held. + // The filters that can only WITHHOLD are retained as an AND-gate on the weakest rung only + // (ADR-0033 decision 2): a nationally shared company name cannot ride „Потвърдено". The stronger + // „Документ" rung is deliberately not gated — the register named this person in THIS company, which + // makes the name key moot. Near-zero recall cost, and it preserves ADR-0017's outcome. + // ADR-0017 carried forward, and NARROWED to what it actually held: a name backing more than one valid + // ЕИК cannot support a name-derived identity claim. It gates the SEAT leg of rung 3 only — never the + // declared-ЕИК leg (ADR-0028: the ЕИК is the identity), and never rung 2 (the register named this + // person in THIS company). nameDistinctiveness is deliberately NOT part of this gate: the seat rung + // exists precisely to rescue a generic name, so requiring distinctiveness would empty it. const nameUnique = nameGloballyUnique(rec.key); - const tier = - rec.method === 'declared_eik' - ? 'A_eik' - : publishTier({ - seatOk, - distinctiveness: nameUnique ? nameDistinctiveness(rec.key) : 'generic', - }); + // „Неизвестна" — the withholding verdict, used for every way of ending up with no usable evidence. + const noEvidence = () => ({ + kind: 'unknown', + publishable: false, + registryRole: null, + matchedFact: null, + entryNumber: null, + entryDate: null, + rulesVersion: RULES_VERSION, + }); + // With --allow-partial-tr the operator has accepted an incomplete cache. An uncached ЕИК then yields + // no evidence at all, which is „Неизвестна" — held. It must never be read as a reason to publish. + let verdict; + if (missing) verdict = noEvidence(); + else { + try { + verdict = evidenceVerdict({ + deed, + outsideTr, + declarantName: rec.person, + declaredSeats: [...rec.seats], + declaredEik: rec.method === 'declared_eik', + firstDeclaredYear: declYears.length ? Math.min(...declYears) : null, + scope: rec.scope, + nameGloballyUnique: nameUnique, + // ADR-0035. The resolver picked this winner by фирма; `nameGloballyUnique` above only says no OTHER + // WINNER shares that name, which says nothing about the register at large. So an uncorroborated + // rung 2 additionally asks whether the фирма is one a national twin is unlikely to share. + // `rec.key` is the WINNER's registered name — the canonical spelling of the company we actually + // looked up, which is the identity in question. + companyNameDistinctive: nameDistinctiveness(rec.key) === 'distinctive', + }); + } catch (err) { + // A deed we cannot parse is a deed we cannot reason about — so this link withholds, exactly as an + // uncached one does, and the run continues. Failing the whole load instead would let one malformed + // deed out of ~400 decide the fate of every other link, and the ship floor would then refuse the + // reduced surface — turning a single bad payload into a total outage. Loud, per link, fail-closed. + console.error( + ` ${rec.eik}: evidence UNREADABLE — ${err instanceof Error ? err.message : err} (link held)`, + ); + verdict = noEvidence(); + } + } + const tier = verdict.kind; const contemporaneous = [...years].some( (cy) => temporalStatus(declYears, cy) === 'contemporaneous', ) @@ -537,15 +860,33 @@ for (const rec of agg.values()) { // non-surfaced class ('internal'), not 'published'. cValue can legitimately be 0 with cCount>0 (contracts // whose amount is unknown/NULL) — that is a real conflict of unknown value, so gate on COUNT, not value. const surfaces = (iClass === 'private_ownership' || iClass === 'family_ownership') && cCount > 0; + // §7: „terminated" is an inference from SILENCE, and its commonest cause is a finished mandate, not a + // sale. Before ADR-0021 E11's withdrawal takes effect, a terminated OWN stake is reconciled against the + // live deed: a person still registered as an owner has not divested. Family stakes are never reconciled + // — the registered owner there is the relative, whose name we neither store nor check. + // PHASE 1 uses only `terminated`; the „и към днешна дата" label is computed and deliberately not + // rendered (ADR-0033 decision 4 — it asserts a present tense behind an LIA addendum). + const recon = + divested && rec.scope === 'self' + ? reconcileTermination({ deed, declarantName: rec.person, scope: rec.scope }) + : { terminated: divested, label: null }; + const terminatedEffective = recon.terminated; + + // Order matters and differs from the old ladder: `internal` is now decided BEFORE `published`, so a + // non-surfaced class (ex-officio board, management-only, zero-contract) can never land in `held`. + // `held` is the REVIEW QUEUE, and its population is exactly the evidence rungs that withhold — + // bar_joint_stock, unknown and outside_tr. const status = isSuppressed(linkKey) ? 'suppressed' - : divested - ? 'withdrawn' - : tier === 'C_hold' - ? 'held' - : surfaces - ? 'published' - : 'internal'; + : verdict.kind === 'refuted' + ? 'withdrawn' // §5.4 — own stakes only; evidence.mjs refuses to refute a family stake + : terminatedEffective + ? 'withdrawn' + : !surfaces + ? 'internal' + : verdict.publishable + ? 'published' + : 'held'; const yrs = [...years]; insLink.run( `il:${linkKey}`, @@ -570,6 +911,35 @@ for (const rec of agg.values()) { yrs.length ? String(Math.max(...yrs)) : null, status, ); + // live_status is RE-DERIVED every run and never treated as part of the permanent seal (ADR-0033 R3): + // it asserts a present tense whose freshness is bounded by the cache refresh cycle. + const liveStatus = !terminatedEffective + ? recon.label === 'owner_today' + ? 'terminated_owner_still' + : 'live' + : recon.label === 'manager_today' + ? 'terminated_manager_still' + : 'terminated'; + // Refuse at WRITE time, not only in the post-hoc audit. The audit runs after the whole domain is + // built; by then the name is already in a table, and a run that dies on a suppressed-by-default axis + // could ship it. `matched_fact` is the one sealed column derived from a deed's text, so it is the one + // place a third-party name can reach a served row — the rail belongs where the value is produced. + if (!isSealedFact(verdict.matchedFact)) + throw new Error( + `REFUSE TO SEAL: matched_fact for ${linkKey} is outside the closed vocabulary — a name may have ` + + `leaked out of a deed (#279 §9, ADR-0033 decision 5)`, + ); + insEvidence.run( + linkKey, + verdict.kind, + verdict.registryRole, + verdict.matchedFact, + verdict.entryNumber, + verdict.entryDate, + trLookupDate, + verdict.rulesVersion, + liveStatus, + ); for (const [auth_id, a] of perAuth) insILA.run(linkKey, auth_id, a.name, a.count, a.value || null, a.own); } @@ -686,6 +1056,21 @@ const S = { r.n, ]), ), + // Every evidence rung's count, publishing and withholding alike (ADR-0033 decision 1). `publish_tier` + // IS the verdict kind since #279. + by_evidence_kind: Object.fromEntries( + q('SELECT publish_tier, COUNT(*) n FROM interest_links GROUP BY publish_tier').map((r) => [ + r.publish_tier, + r.n, + ]), + ), + // ADR-0035's residual: links where the register named this person in the resolved company but nothing + // established that the company is the declared one. This is the number F8's hand-labelled sample reads + // to decide whether the distinctiveness gate tightens to strict ЕИК/seat corroboration. Reported + // separately from the map above because it is a DECISION input, not just a tally. + document_uncorroborated: one( + "SELECT COUNT(*) n FROM interest_links WHERE publish_tier='document_uncorroborated'", + ).n, ambiguous_name_keys: ambiguousKeys.length, ambiguous_name_key_examples: ambiguousKeys, noMatch, diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index d795bf598..987cbdd8a 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -10,11 +10,13 @@ import path from 'node:path'; import os from 'node:os'; import { fileURLToPath } from 'node:url'; import { fingerprint } from './suppressions.mjs'; +// The seal vocabulary is asserted with the PRODUCTION predicate, imported from the module that writes it. +import { isSealedFact } from '../tr/evidence.mjs'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); const SUPP_SALT = 'test-salt-9f3a'; // stand-in for the CI secret SUPPRESSION_SALT -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad(extraEnv = {}) { execFileSync( @@ -22,17 +24,105 @@ function runLoad(extraEnv = {}) { ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], { cwd: ROOT, - env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING, ...extraEnv }, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + ...extraEnv, + }, stdio: 'pipe', }, ); } + +/** + * Build a Trade Register cache + raw deeds covering the fixture's winners. + * + * `spec[eik]` describes one deed: `owners` / `managers` are full names placed in SEPARATE registry + * entities (so the entity-boundary rule is exercised end to end), `form` is the numeric legalForm and + * `suffix` the ЗТРРЮЛНЦ form on fullName. Anything omitted from `spec` is still cached — as a deed + * naming somebody else — because the loader must FAIL CLOSED on a cache that does not cover every + * candidate, and a test that silently left ЕИК uncovered would exercise that path by accident. + */ +function buildTrCache(dbFile, rawDir, spec = {}, { omit = [] } = {}) { + fs.mkdirSync(rawDir, { recursive: true }); + const cache = new DatabaseSync(dbFile); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + const src = new DatabaseSync(DB, { readOnly: true }); + const eiks = src + .prepare('SELECT eik_normalized e FROM bidders WHERE eik_normalized IS NOT NULL') + .all() + .map((r) => r.e); + src.close(); + + const container = (t) => + `

${t}

`; + const joinEntities = (names) => names.map(container).join(`
`); + + for (const eik of eiks) { + if (omit.includes(eik)) continue; + const d = spec[eik] ?? {}; + if (d.outsideTr) { + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,fetched_at,outside_reason) VALUES(?,?,?,?)', + ) + .run(eik, 'outside_tr', '2026-08-05T00:00:00Z', 'HTTP 200, empty body'); + continue; + } + const fields = []; + const push = (nameCode, names, entryDate) => + names?.length && + fields.push({ + nameCode, + htmlData: joinEntities(names), + fieldEntryNumber: '20110502101007', + fieldEntryDate: `${entryDate ?? '2011-05-02'}T00:00:00`, + }); + push('CR_F_19_L', d.owners ?? ['НЯКОЙ ДРУГ СОБСТВЕНИК'], d.ownEntryDate); + push('CR_F_7_L', d.managers, d.ownEntryDate); + if (d.seat) push('CR_F_5_L', [`Населено място: ${d.seat}`], d.seatEntryDate ?? d.ownEntryDate); + const deed = { + uic: eik, + fullName: `"ФИКС" ${d.suffix ?? 'ООД'}`, + legalForm: d.form ?? 4, + sections: [{ subDeeds: [{ groups: [{ fields }] }] }], + }; + fs.writeFileSync(path.join(rawDir, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + `INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code, + legal_form_verdict,seat_normalized,latest_own_entry_date) + VALUES(?,?,?,?,?,?,?,?,?)`, + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + d.form ?? 4, + d.suffix && /АД|КДА/.test(d.suffix) ? 'joint_stock' : 'closely_held', + d.seat ? d.seat.replace(/^гр\.\s*/, '').toUpperCase() : null, + d.ownEntryDate ?? '2011-05-02', + ); + } + cache.close(); +} const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-load-')); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); fs.mkdirSync(STAGING, { recursive: true }); // minimal slice of the winner schema that load.mjs joins @@ -82,6 +172,22 @@ before(() => { -- stake at all. The empty filing advances his horizon past 2019 → the 2019 stake is withdrawn. INSERT INTO bidders VALUES ('eik:101010104','ДИВЕСТ ЗЕРО 4 ЕООД','101010104',1,'София'); INSERT INTO contracts VALUES ('c11','t1','eik:101010104','2019-05-01',150000); + -- §1.3 unparseable filing YEAR: Безгодин owns ДИВЕСТ БЕЗГОД (2019), then files a later declaration whose + -- is unreadable while its FOLDER carries 2023. The folder must supply the horizon, or the filing + -- is dropped, the horizon never advances, and a sold stake stays published — a stale public claim. + INSERT INTO bidders VALUES ('eik:212121218','ДИВЕСТ БЕЗГОД 9 ЕООД','212121218',1,'София'); + INSERT INTO contracts VALUES ('c17','t1','eik:212121218','2019-05-01',120000); + -- §1.3 positive control: Дрънкан's later filing is datable by NEITHER NOR folder. It must be + -- ignored, never guessed — an undatable filing is no evidence of a sale, so this stake stays published. + INSERT INTO bidders VALUES ('eik:232323231','ДРЪНКАН ТЕХ 10 ЕООД','232323231',1,'София'); + INSERT INTO contracts VALUES ('c18','t1','eik:232323231','2019-05-01',110000); + -- ADR-0035 winner-vs-non-winner homonym: „ХОМОНИМ ТРЕЙД" is a GENERIC фирма (two content words) and the + -- sole WINNER holding it. Хомоним Иванов Тестов declared a stake in a company of that name — but the one + -- he owns never bid, so the resolver lands on this winner instead. This winner's deed happens to name a + -- HOMONYM (identical three tokens), which under a name-only rung 2 „proves" a link false in both halves. + -- Nothing corroborates the company (no declared ЕИК, no declared seat), so it must be withheld. + INSERT INTO bidders VALUES ('eik:242424248','ХОМОНИМ ТРЕЙД ЕООД','242424248',1,'София'); + INSERT INTO contracts VALUES ('c19','t1','eik:242424248','2023-05-01',95000); -- N10 canonicalization: Канонов owns КАНОН ТЕХ 5, filing „МВР" one year and the full ministry name the -- next → ONE identity, ONE link (not a split). Distinctive name (number) → tier B publishable. INSERT INTO bidders VALUES ('eik:131313136','КАНОН ТЕХ 5 ЕООД','131313136',1,'София'); @@ -99,6 +205,12 @@ before(() => { -- not twice — the load-side sibling of the UI conflictHeadline per-ЕИК money dedup. INSERT INTO bidders VALUES ('eik:181818187','ПАРТНЬОРИ 5 ЕООД','181818187',1,'София'); INSERT INTO contracts VALUES ('c15','t1','eik:181818187','2022-06-01',600000); + -- FAMILY positive control (#279): identical in shape to Кмет's case but the official DECLARED the + -- company's seat, which is what confirms the company's identity when the registered owner is the + -- relative whose name we never hold. Without this case „family published: 0" would be + -- indistinguishable from a structurally dead path (ADR-0027's false-zero lesson). + INSERT INTO bidders VALUES ('eik:191919199','СЕМЕЕН ДОМ ЕООД','191919199',1,'Русе'); + INSERT INTO contracts VALUES ('c16','t4','eik:191919199','2023-07-01',180000); `); db.close(); @@ -128,7 +240,7 @@ before(() => { template: 'assets', category: '', institution: 'X', - person: 'Мария Иванова', + person: 'Мария Иванова Петрова', position: '', entity: '"ГЕНЕРИК" ООД', kind: 'shares', @@ -145,7 +257,7 @@ before(() => { template: 'assets', category: '', institution: 'Y', - person: 'Петър Николов', + person: 'Петър Иванов Николов', position: '', entity: 'СИЙ ЕООД', kind: 'shares', @@ -162,7 +274,7 @@ before(() => { template: 'assets', category: '', institution: 'Z', - person: 'Георги Стоянов', + person: 'Георги Иванов Стоянов', position: '', entity: 'СИЙ ЕООД', kind: 'shares', @@ -181,7 +293,7 @@ before(() => { template: 'assets', category: '', institution: 'W', - person: 'Стефан Колев', + person: 'Стефан Иванов Колев', position: '', entity: '"ГЕНЕРИК" ООД, ЕИК 222222229', kind: 'shares', @@ -198,7 +310,7 @@ before(() => { template: 'assets', category: '', institution: 'V', - person: 'Радка Илиева', + person: 'Радка Иванова Илиева', position: '', entity: '"ГЕНЕРИК" ООД, ЕИК 333333338', kind: 'shares', @@ -215,7 +327,7 @@ before(() => { template: 'interests', category: '', institution: 'U', - person: 'Борис Манолов', + person: 'Борис Иванов Манолов', position: 'член на съвет', entity: 'ХОЛДИНГ 9 ЕАД', kind: 'management', @@ -231,7 +343,7 @@ before(() => { template: 'interests', category: '', institution: 'U', - person: 'Виктор Асенов', + person: 'Виктор Иванов Асенов', position: 'член на съвет', entity: 'ХОЛДИНГ 9 ЕАД', kind: 'management', @@ -249,7 +361,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Николай Дивестов', + person: 'Николай Иванов Дивестов', position: '', entity: 'ДИВЕСТ 1 ЕООД', kind: 'shares', @@ -265,7 +377,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Николай Дивестов', + person: 'Николай Иванов Дивестов', position: '', entity: 'ДИВЕСТ 2 ЕООД', kind: 'shares', @@ -283,7 +395,7 @@ before(() => { template: 'assets', category: '', institution: 'ОБЩИНА ТЕСТ', - person: 'Кмет Тестов', + person: 'Кмет Иванов Тестов', position: 'кмет', entity: 'ЕВРОСТРОЙ 21 ЕООД', kind: 'shares', @@ -293,6 +405,24 @@ before(() => { holderRelation: 'related', controlHash: 'H11', }, + // FAMILY positive control: same as Кмет, but with the seat declared → rung 3 confirms the company. + { + folder: '2024', + xmlFile: 'K2.xml', + year: '2023', + template: 'assets', + category: '', + institution: 'ОБЩИНА ТЕСТ', + person: 'Кметица Иванова Втора', + position: 'кмет', + entity: 'СЕМЕЕН ДОМ ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: 'Русе', + holderRelation: 'related', + controlHash: 'H11b', + }, // SECURITIES: Акционер holds LISTED joint-stock shares (kind securities) → excluded, no ownership link. { folder: '2024', @@ -301,7 +431,7 @@ before(() => { template: 'assets', category: '', institution: 'S', - person: 'Акционер Тестов', + person: 'Акционер Иванов Тестов', position: '', entity: 'ЛИСТЕД ТЕСТ АД', kind: 'securities', @@ -320,7 +450,7 @@ before(() => { template: 'assets', category: '', institution: 'N2', - person: 'Нула Тестов', + person: 'Нула Иванов Тестов', position: '', entity: 'НУЛА ТЕХ 3 ЕООД', kind: 'shares', @@ -340,7 +470,7 @@ before(() => { template: 'assets', category: '', institution: 'T2', - person: 'Пълен Дивестов', + person: 'Пълен Иванов Дивестов', position: '', entity: 'ДИВЕСТ ЗЕРО 4 ЕООД', kind: 'shares', @@ -350,6 +480,61 @@ before(() => { holderRelation: 'self', controlHash: 'H14', }, + // §1.3 unparseable filing year: Безгодин's 2019 stake. His later filing (in filings.jsonl, no holdings + // row) carries an unreadable but a 2023 FOLDER — the fallback that must withdraw this stake. + { + folder: '2020', + xmlFile: 'BG0.xml', + year: '2019', + template: 'assets', + category: '', + institution: 'T3', + person: 'Безгодин Иванов Дивестов', + position: '', + entity: 'ДИВЕСТ БЕЗГОД 9 ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: 'гр. София', + holderRelation: 'self', + controlHash: 'H20', + }, + // §1.3 positive control: Дрънкан's 2019 stake, whose only later filing is undatable (see filings.jsonl). + { + folder: '2020', + xmlFile: 'DR0.xml', + year: '2019', + template: 'assets', + category: '', + institution: 'T4', + person: 'Дрънкан Иванов Тестов', + position: '', + entity: 'ДРЪНКАН ТЕХ 10 ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: '', + holderRelation: 'self', + controlHash: 'H21', + }, + // ADR-0035: Хомоним's declared stake in a generic-named company, with NO ЕИК and NO seat declared. + { + folder: '2024', + xmlFile: 'HOM.xml', + year: '2023', + template: 'assets', + category: '', + institution: 'T5', + person: 'Хомоним Иванов Тестов', + position: '', + entity: 'ХОМОНИМ ТРЕЙД ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: '', + holderRelation: 'self', + controlHash: 'H22', + }, // B4 UNKNOWN holder: the holder cell is neither confidently the declarant's own name nor a relative's // (an ambiguous 1-token-different cell). classifyHolder → 'unknown' → this forms NO link (counted // nowhere), so a phantom relative never enters a published family figure. @@ -360,7 +545,7 @@ before(() => { template: 'assets', category: '', institution: 'N3', - person: 'Двусмислен Тестов', + person: 'Двусмислен Иванов Тестов', position: '', entity: 'ДИСТИНКТ ТЕХ 7 ЕООД', kind: 'shares', @@ -379,7 +564,7 @@ before(() => { template: 'assets', category: '', institution: 'МВР', - person: 'Канонов Тестов', + person: 'Канонов Иванов Тестов', position: '', entity: 'КАНОН ТЕХ 5 ЕООД', kind: 'shares', @@ -396,7 +581,7 @@ before(() => { template: 'assets', category: '', institution: 'Министерство на вътрешните работи', - person: 'Канонов Тестов', + person: 'Канонов Иванов Тестов', position: '', entity: 'КАНОН ТЕХ 5 ЕООД', kind: 'shares', @@ -414,7 +599,7 @@ before(() => { template: 'assets', category: '', institution: '', - person: 'Безинст Тестов', + person: 'Безинст Иванов Тестов', position: '', entity: 'БЕЗИНСТ ТЕХ 6 ЕООД', kind: 'shares', @@ -434,7 +619,7 @@ before(() => { template: 'interests', category: '', institution: 'INT', - person: 'Интер Тестов', + person: 'Интер Иванов Тестов', position: '', entity: 'ИНТЕР ТЕХ 8 ЕООД', kind: 'shares', @@ -454,7 +639,7 @@ before(() => { template: 'assets', category: '', institution: 'ТЕСТ ВЕДОМСТВО', - person: 'Алфа Партньоров', + person: 'Алфа Иванов Партньоров', position: '', entity: 'ПАРТНЬОРИ 5 ЕООД', kind: 'shares', @@ -471,7 +656,7 @@ before(() => { template: 'assets', category: '', institution: 'ДРУГО ВЕДОМСТВО', - person: 'Бета Партньоров', + person: 'Бета Иванов Партньоров', position: '', entity: 'ПАРТНЬОРИ 5 ЕООД', kind: 'shares', @@ -503,7 +688,7 @@ before(() => { xmlFile: 'ZE1.xml', year: '2023', template: 'assets', // Пълен's later EMPTY ASSET filing — same type as his 2019 asset stake ⇒ divests it - person: 'Пълен Дивестов', + person: 'Пълен Иванов Дивестов', institution: 'T2', }); // #226 (Todor B1) cross-type: Интер declares his stake ONLY in an INTERESTS declaration (2020); his later @@ -515,13 +700,68 @@ before(() => { xmlFile: 'INTA.xml', year: '2023', template: 'assets', - person: 'Интер Тестов', + person: 'Интер Иванов Тестов', institution: 'INT', }); + // §1.3: the divesting filing whose is UNREADABLE. `folder` carries 2023 and is the only thing that + // can date it. Dropping the record leaves the horizon at 2019, `divested` false, and a sold stake on the + // public surface — the failure direction that matters here, since the claim names a real official. + filings.push({ + folder: '2023', + xmlFile: 'BG1.xml', + year: 'н/д', + template: 'assets', + person: 'Безгодин Иванов Дивестов', + institution: 'T3', + }); + // POSITIVE CONTROL for the same fallback: a filing datable by NEITHER field must still be ignored, not + // guessed at. Дрън's stake stays published — an undatable filing is no evidence of a sale. + filings.push({ + folder: 'архив', + xmlFile: 'DR1.xml', + year: '', + template: 'assets', + person: 'Дрънкан Иванов Тестов', + institution: 'T4', + }); fs.writeFileSync( path.join(STAGING, 'filings.jsonl'), filings.map((f) => JSON.stringify(f)).join('\n') + '\n', ); + + // The Trade Register evidence each link now has to rest on (#279, ADR-0033). Shaped so every + // existing case keeps the INTENT it was written for, under the new rule rather than the old one: + // • a person the register names as owner/manager → „Документ" + // • a declared seat matching the registered seat → „Потвърдено" + // • a declared ЕИК → „Потвърдено" (never name-gated, ADR-0028) + // • nobody we can match and nothing to confirm → „Неизвестна", held + buildTrCache(TR_DB, TR_RAW, { + 111111119: { managers: ['ИВАН ПЕТРОВ ТЕСТОВ'] }, // manages → document/manager (class keeps it internal) + 444444447: { seat: 'гр. Бургас' }, // Петър declared Бургас → confirmed; Георги declared none → held + 555555556: { + managers: ['БОРИС ИВАНОВ МАНОЛОВ', 'ВИКТОР ИВАНОВ АСЕНОВ'], + suffix: 'ЕАД', + form: 5, + }, + 666666665: { owners: ['СЪВСЕМ ДРУГ СОБСТВЕНИК'] }, // Николай absent → his divestment stands + 777777773: { owners: ['НИКОЛАЙ ИВАНОВ ДИВЕСТОВ'] }, // still the registered owner → document + 888888884: { owners: ['РОДНИНА КМЕТОВА'] }, // family: the RELATIVE owns it, not the official + 999999998: { suffix: 'АД', form: 5 }, + 101010104: { owners: ['ДРУГ СОБСТВЕНИК'] }, // Пълен absent → divest-to-zero stands + 131313136: { owners: ['КАНОНОВ ИВАНОВ ТЕСТОВ'] }, + 161616163: { owners: ['ИНТЕР ИВАНОВ ТЕСТОВ'] }, + 181818187: { owners: ['АЛФА ИВАНОВ ПАРТНЬОРОВ', 'БЕТА ИВАНОВ ПАРТНЬОРОВ'] }, + 121212129: { owners: ['НУЛА ИВАНОВ ТЕСТОВ'] }, + 191919199: { owners: ['РОДНИНА ВТОРА'], seat: 'гр. Русе' }, // family + declared seat → confirmed + // Безгодин is ABSENT from the deed (so §7 reconciliation cannot reverse the divestment) but his + // declared seat matches the registered one, so rung 3 says „Потвърдено" and the link PUBLISHES. + // Only the folder-dated divestment withdraws it — making the pre-fix failure the dangerous one. + 212121218: { owners: ['ДРУГ СОБСТВЕНИК СЪВСЕМ'], seat: 'гр. София' }, + 232323231: { owners: ['ДРЪНКАН ИВАНОВ ТЕСТОВ'] }, // still the owner → the undatable filing changes nothing + // The homonym: the deed names someone with Хомоним's exact three tokens. Rung 2 matches — and must + // still withhold, because nothing says this is the company he declared. + 242424248: { owners: ['ХОМОНИМ ИВАНОВ ТЕСТОВ'] }, + }); }); after(() => fs.rmSync(dir, { recursive: true, force: true })); @@ -540,7 +780,7 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // management_role never surfaces → status 'internal', NOT 'published' (a direct D1 reader must not see a // non-surfaced official+company row labelled published; the served query also filters by interest_class). assert.equal(ivan.status, 'internal'); - assert.equal(ivan.publish_tier, 'B_distinctive'); + assert.equal(ivan.publish_tier, 'document'); // the register names him a manager of this company assert.equal(ivan.relation, 'manages'); assert.equal(ivan.interest_class, 'management_role'); // manages, sole declarant → ambiguous, not headline assert.equal(ivan.own_institution, 'exact'); @@ -557,8 +797,8 @@ test('resolves publish/held/quarantine tiers deterministically', () => { assert.equal(blob.value_eur, 25000); // bare collision name (no ЕИК in text) → quarantined, Мария gets no link - assert.equal(link('222222229', 'Мария Иванова'), undefined); - assert.equal(link('333333338', 'Мария Иванова'), undefined); + assert.equal(link('222222229', 'Мария Иванова Петрова'), undefined); + assert.equal(link('333333338', 'Мария Иванова Петрова'), undefined); // the only links onto the colliding ЕИК come from declared_eik (Стефан/Радка), never exact_name_key assert.equal( db @@ -569,14 +809,14 @@ test('resolves publish/held/quarantine tiers deterministically', () => { 0, ); - const petar = link('444444447', 'Петър Николов'); - assert.equal(petar.publish_tier, 'A_seat'); // generic name rescued by seat match + const petar = link('444444447', 'Петър Иванов Николов'); + assert.equal(petar.publish_tier, 'confirmed'); // declared seat == registered seat assert.equal(petar.status, 'published'); assert.equal(petar.interest_class, 'private_ownership'); // declared a share → the headline conflict signal // two officials manage the SAME company → deterministically classed ex-officio (public board), not private - const boris = link('555555556', 'Борис Манолов'); - const viktor = link('555555556', 'Виктор Асенов'); + const boris = link('555555556', 'Борис Иванов Манолов'); + const viktor = link('555555556', 'Виктор Иванов Асенов'); assert.equal(boris.interest_class, 'ex_officio_board'); assert.equal(viktor.interest_class, 'ex_officio_board'); assert.equal(boris.relation, 'manages'); @@ -584,14 +824,14 @@ test('resolves publish/held/quarantine tiers deterministically', () => { assert.equal(boris.status, 'internal'); assert.equal(viktor.status, 'internal'); - const georgi = link('444444447', 'Георги Стоянов'); - assert.equal(georgi.publish_tier, 'C_hold'); // generic, no seat → held + const georgi = link('444444447', 'Георги Иванов Стоянов'); + assert.equal(georgi.publish_tier, 'unknown'); // same company, but he declared no seat → nothing confirms assert.equal(georgi.status, 'held'); // E11 divestment: Николай's 2019 stake in ДИВЕСТ 1 is superseded by a 2022 filing that omits it → withdrawn; // his current ДИВЕСТ 2 stake stays published. A later ownership filing that drops a company ends that link. - const gone = link('666666665', 'Николай Дивестов'); - const kept = link('777777773', 'Николай Дивестов'); + const gone = link('666666665', 'Николай Иванов Дивестов'); + const kept = link('777777773', 'Николай Иванов Дивестов'); assert.equal(gone.status, 'withdrawn'); // divested — excluded from the published surface assert.equal(gone.interest_class, 'private_ownership'); assert.equal(gone.last_declared_year, '2019'); // dated to its last declaration, never asserted "current" @@ -601,15 +841,15 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // certain ЕИК (declared_eik) behind a colliding name, no seat → the declarant-provided ЕИК is the // national unique identifier, so identity is deterministic → publishes as A_eik, NOT held for // name-genericness (ADR-0016; the ЕИК is at least as certain as the seat that rescues Радка below). - const stefan = link('222222229', 'Стефан Колев'); + const stefan = link('222222229', 'Стефан Иванов Колев'); assert.equal(stefan.match_method, 'declared_eik'); // ЕИК resolution IS certain - assert.equal(stefan.publish_tier, 'A_eik'); // ЕИК = unique identifier → deterministic, not name-gated + assert.equal(stefan.publish_tier, 'confirmed'); // the declared ЕИК confirms the company, never name-gated assert.equal(stefan.status, 'published'); // private_ownership (20% ООД share) → surfaces // same colliding name, resolved by her declared ЕИК (with seat as extra corroboration) → the ЕИК is the // identity, so A_eik (not A_seat); publishable. A_seat's own path stays covered by Петър above. - const radka = link('333333338', 'Радка Илиева'); + const radka = link('333333338', 'Радка Иванова Илиева'); assert.equal(radka.match_method, 'declared_eik'); - assert.equal(radka.publish_tier, 'A_eik'); + assert.equal(radka.publish_tier, 'confirmed'); assert.equal(radka.status, 'published'); // FAMILY: a close relative's declared stake in a winner that sold to the official's OWN institution. @@ -617,10 +857,17 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // relative's declared stake in a procurement winner is the same public-interest signal. class // family_ownership, relation 'related', own_institution exact. It has real contract money (€250k, cCount>0) // so the zero-contract gate keeps it. - const family = link('888888884', 'Кмет Тестов'); + const family = link('888888884', 'Кмет Иванов Тестов'); assert.equal(family.relation, 'related'); assert.equal(family.interest_class, 'family_ownership'); - assert.equal(family.status, 'published'); // ADR-0032: family surfaces like self + // #279 NARROWS the family surface, and this is where it shows. The registered owner of a family + // stake is the RELATIVE, whose name we deliberately never store (ADR-0010 item 4, ADR-0032 #2) — so + // rung 2 („Документ") can never fire for a family link, by construction. Its identity can only be + // confirmed by something the OFFICIAL declared: the seat, or the ЕИК. Кмет declared neither, so his + // link is now HELD rather than published. ADR-0032's decision is untouched — family publishes on the + // named surface exactly like self — but it now needs the same registry evidence as everything else. + assert.equal(family.status, 'held'); + assert.equal(family.publish_tier, 'unknown'); assert.equal(family.own_institution, 'exact'); // relative's company sold to the official's own institution assert.equal(family.contemporaneous, 1); assert.equal(family.contract_value_eur, 250000); @@ -645,8 +892,8 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // SUM(contract_value_eur) over published links double-counts that winner — the load-side twin of the UI // conflictHeadline bug. Prove the fixture actually exercises the collision (naive per-link sum strictly // exceeds the per-ЕИК-deduped sum), then assert the reported totals equal the deduped figure. - const alfa = link('181818187', 'Алфа Партньоров'); - const beta = link('181818187', 'Бета Партньоров'); + const alfa = link('181818187', 'Алфа Иванов Партньоров'); + const beta = link('181818187', 'Бета Иванов Партньоров'); assert.equal(alfa.status, 'published'); assert.equal(beta.status, 'published'); assert.equal(alfa.interest_class, 'private_ownership'); @@ -683,14 +930,14 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // ZERO-CONTRACT gate (I5): a distinctive winner with NO contracts is collected but never published — the // card would read „0 договори · 0 €", which is no procurement conflict. status 'internal', not 'published'. - const zero = link('121212129', 'Нула Тестов'); + const zero = link('121212129', 'Нула Иванов Тестов'); assert.equal(zero.contract_count, 0); assert.equal(zero.interest_class, 'private_ownership'); // it IS own material ownership … - assert.equal(zero.publish_tier, 'B_distinctive'); // … and tier-B by name … + assert.equal(zero.publish_tier, 'document'); // … with registry evidence … assert.equal(zero.status, 'internal'); // … but the zero-contract gate withholds it from the surface // SECURITIES/materiality: a self holding of LISTED joint-stock shares forms NO ownership link. - assert.equal(link('999999998', 'Акционер Тестов'), undefined); + assert.equal(link('999999998', 'Акционер Иванов Тестов'), undefined); // but it is still recorded as a declared interest (census), tagged kind securities assert.equal( db.prepare("SELECT kind FROM declared_interests WHERE entity_raw='ЛИСТЕД ТЕСТ АД'").get().kind, @@ -700,7 +947,7 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // B1 divest-to-ZERO: Пълен owned ДИВЕСТ ЗЕРО in 2019, then filed an EMPTY declaration in 2023 (no holdings // row — only a filings.jsonl entry). The empty filing advances his horizon to 2023, so the 2019 stake is // WITHDRAWN. Without the filing horizon (pre-B1) his scope-max would be 2019 and this would stay published. - const divZero = link('101010104', 'Пълен Дивестов'); + const divZero = link('101010104', 'Пълен Иванов Дивестов'); assert.equal(divZero.interest_class, 'private_ownership'); assert.equal(divZero.last_declared_year, '2019'); // dated to its last declaration, never asserted current assert.equal(divZero.status, 'withdrawn'); // caught by the empty later filing (B1) @@ -710,18 +957,47 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // that asset-declaration silence as a sale and WITHDRAWS the stake; the per-type horizon must not — no later // INTERESTS filing omits the company. This link must stay PUBLISHED. (Guards against dropping a true link: // 13% of holders declare a stake only in the interests declaration.) - const crossType = link('161616163', 'Интер Тестов'); + // ADR-0035 — the CRITICAL, end to end. Хомоним declared a stake in „ХОМОНИМ ТРЕЙД ЕООД"; the company he + // actually owns never bid, so `resolveEntity` resolved his declaration to the same-named WINNER, whose + // deed names a person with his exact three tokens. Rung 2 matches. It must NOT publish: the register + // proves someone of that name owns THIS company, not that this is the company he declared. `nameGlobally- + // Unique` cannot catch it — it ranges over bidders, and this winner is the only bidder with the name. + const homonym = link('242424248', 'Хомоним Иванов Тестов'); + assert.equal(homonym.publish_tier, 'document_uncorroborated'); + assert.notEqual(homonym.status, 'published'); + // The seal must record WHY it was withheld, and must not carry the role the rung refused to assert. + const homonymSeal = db + .prepare( + 'SELECT evidence_kind, registry_role, matched_fact FROM interest_link_evidence WHERE link_key=?', + ) + .get(homonym.link_key); + assert.equal(homonymSeal.evidence_kind, 'document_uncorroborated'); + assert.equal(homonymSeal.registry_role, null); + assert.equal(homonymSeal.matched_fact, null); + + // §1.3 unparseable filing YEAR: Безгодин's later declaration has an unreadable ('н/д') but a 2023 + // FOLDER. Dropping that record — the pre-fix behaviour — leaves his horizon at 2019, so `divested` stays + // false and a stake he no longer holds keeps naming him on the public surface. The folder must date it. + const noYear = link('212121218', 'Безгодин Иванов Дивестов'); + assert.equal(noYear.interest_class, 'private_ownership'); + assert.equal(noYear.status, 'withdrawn'); + // POSITIVE CONTROL: datable by NEITHER field ⇒ ignored, not guessed. The fallback must not become a + // licence to invent a horizon — an undatable filing is no evidence of a sale, so this link stays up. + const noDate = link('232323231', 'Дрънкан Иванов Тестов'); + assert.equal(noDate.status, 'published'); + + const crossType = link('161616163', 'Интер Иванов Тестов'); assert.equal(crossType.interest_class, 'private_ownership'); assert.equal(crossType.status, 'published'); // NOT withdrawn — the later asset filing is a different type // B4 UNKNOWN holder: an ambiguous holder cell forms NO link at all (counted nowhere) — it must never // reach the leaderboard, self or family (ADR-0032). Двусмислен gets no interest_link. - assert.equal(link('111111119', 'Двусмислен Тестов'), undefined); + assert.equal(link('111111119', 'Двусмислен Иванов Тестов'), undefined); // but the person + declared_interest are still recorded (census), and it is neither self nor family. assert.equal( db .prepare( - "SELECT COUNT(*) n FROM interest_links il JOIN persons p ON p.id=il.person_id WHERE p.name='Двусмислен Тестов'", + "SELECT COUNT(*) n FROM interest_links il JOIN persons p ON p.id=il.person_id WHERE p.name='Двусмислен Иванов Тестов'", ) .get().n, 0, @@ -738,10 +1014,10 @@ test('resolves publish/held/quarantine tiers deterministically', () => { .n, 1, ); - const kanon = link('131313136', 'Канонов Тестов'); + const kanon = link('131313136', 'Канонов Иванов Тестов'); assert.equal(kanon.status, 'published'); // distinctive, private ownership, has a contract // N10 empty-institution: an empty institution cannot distinguish homonyms → Безинст forms NO link. - assert.equal(link('141414141', 'Безинст Тестов'), undefined); + assert.equal(link('141414141', 'Безинст Иванов Тестов'), undefined); db.close(); }); @@ -772,14 +1048,17 @@ test('re-run is idempotent and honors the suppression list (contested link stays 'suppressed', ); // idempotent: still exactly the same number of links + persons after a clean rebuild. - // 16 links: 12 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero - // 'withdrawn' + Интер's per-type-kept published link) + 1 family + Канонов's canonicalized single link + + // 20 links: 15 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero + // 'withdrawn' + Безгодин's folder-dated 'withdrawn' + Дрънкан's undatable-filing 'published' + + // Хомоним's ADR-0035 'document_uncorroborated' hold + Интер's per-type-kept published link) + 2 family (Кмет's, now held for want of registry evidence, + // and Кметица's seat-confirmed one) + Канонов's canonicalized single link + // Алфа & Бета (two officials on one winner, ПАРТНЬОРИ 5); Мария (quarantined), Акционер (securities), // Двусмислен (unknown holder) & Безинст (empty institution) none. - assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 16); - // 19 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, - // zero-contract Нула and the two ПАРТНЬОРИ co-owners; Канонов's two institution-variant filings fold to ONE. - assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 19); + assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 20); + // 23 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, + // zero-contract Нула, the two ПАРТНЬОРИ co-owners, the two §1.3 filing-date cases and Хомоним; + // Канонов's two institution-variant filings fold to ONE. + assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 23); db.close(); }); @@ -879,3 +1158,362 @@ test('a suppression on a ROTATED key_version FAILS the build (B3 no silent salt (err) => /key_version/.test(String(err.stderr ?? '') + String(err.message ?? '')), ); }); + +// ── the fail-closed evidence gates (ADR-0033 decision 7) ───────────────────────────────────────── +// Both directions of "no evidence ⇒ no publish", because the partial case is the dangerous one: it does +// NOT fail loudly downstream. An 80%-restored cache yields roughly 80 published links, which clears +// ship-related-persons.mjs's floor of 50 — so it would ship a decimated surface and then wipe the rest +// of the live links. The loader is the only place that can still tell the difference. +test('a MISSING Trade Register cache refuses the whole load', () => { + const gone = path.join(dir, 'no-such-cache.sqlite'); + assert.throws( + () => runLoad({ TR_CACHE_DB: gone }), + /REFUSE TO LOAD[\s\S]*no Trade Register cache/, + ); +}); + +test('a PARTIAL cache refuses the load rather than publishing a decimated surface', () => { + const partialDb = path.join(dir, 'partial-cache.sqlite'); + const partialRaw = path.join(dir, 'partial-deeds'); + // Cover everything EXCEPT two winners — the shape a resumed-but-unfinished crawl leaves behind. + buildTrCache( + partialDb, + partialRaw, + { 111111119: { managers: ['ИВАН ПЕТРОВ ТЕСТОВ'] } }, + { omit: ['444444447', '777777773'] }, + ); + assert.throws( + () => runLoad({ TR_CACHE_DB: partialDb, TR_RAW_DIR: partialRaw }), + /REFUSE TO LOAD[\s\S]*covers \d+ of \d+/, + ); +}); + +test('--allow-partial-tr is the deliberate, stated override', () => { + // Without an override a single permanently unreachable ЕИК would deadlock the pipeline forever. + const partialDb = path.join(dir, 'partial-ok.sqlite'); + const partialRaw = path.join(dir, 'partial-ok-deeds'); + buildTrCache(partialDb, partialRaw, {}, { omit: ['121212129'] }); + assert.doesNotThrow(() => + execFileSync( + 'node', + [ + '--import', + path.join(HERE, 'register-ts.mjs'), + path.join(HERE, 'load.mjs'), + '--allow-partial-tr', + ], + { + cwd: ROOT, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: partialDb, + TR_RAW_DIR: partialRaw, + }, + stdio: 'pipe', + }, + ), + ); + runLoad(); // restore the full-cache state for any later reader +}); + +test('the candidate ЕИК list is written for the crawler, covering held links too', () => { + runLoad(); + const listed = fs + .readFileSync(path.join(STAGING, 'candidate-eiks.txt'), 'utf8') + .split('\n') + .filter(Boolean); + const db = open(); + const all = db + .prepare('SELECT DISTINCT eik FROM interest_links') + .all() + .map((r) => r.eik); + db.close(); + // Every resolved ЕИК, not just the published ones: a link held for want of evidence still needs a + // deed to explain why it is held. + for (const e of all) assert.ok(listed.includes(e), `${e} missing from candidate-eiks.txt`); + assert.equal(new Set(listed).size, listed.length, 'no duplicates — each ЕИК costs one request'); +}); + +test('every link carries an evidence seal, and no seal carries a name', () => { + runLoad(); + const db = open(); + const links = db.prepare('SELECT link_key, publish_tier, status FROM interest_links').all(); + const seals = db.prepare('SELECT * FROM interest_link_evidence').all(); + assert.equal(seals.length, links.length, 'a seal for EVERY link, held and withdrawn included'); + + // The PRODUCTION predicate, imported from the module that WRITES the vocabulary — a restated regex + // here was looser than the real one and would have passed a seat token carrying a full name. + for (const s of seals) { + assert.ok(isSealedFact(s.matched_fact), s.matched_fact); + assert.ok(s.rules_version.length > 0); + assert.match(s.lookup_date, /^\d{4}-\d{2}-\d{2}$/); + } + // The rail, stated as an assertion rather than a convention. Scope it precisely: `link_key` carries + // the OFFICIAL's own identity by design — they are named on the public surface from their own + // declaration — so the rail is about everyone else. No name that exists only inside a registry deed + // (a co-owner, a manager, the relative who actually holds a family stake) may reach a sealed column. + const evidenceOnly = seals.map(({ link_key: _ignored, ...rest }) => rest); + const blob = JSON.stringify(evidenceOnly).toUpperCase(); + for (const thirdParty of [ + 'РОДНИНА', // the relative who owns the family company + 'СЪВСЕМ ДРУГ СОБСТВЕНИК', // a registry co-owner nobody declared + 'НЯКОЙ ДРУГ', + 'ДРУГ СОБСТВЕНИК', + ]) + assert.ok(!blob.includes(thirdParty), `a third-party name reached the seal: ${thirdParty}`); + // …and the declarant's name must not be duplicated into the evidence fields either. + for (const surname of ['ТЕСТОВ', 'ДИВЕСТОВ', 'ПАРТНЬОРОВ']) + assert.ok(!blob.includes(surname), `a declarant name reached an evidence column: ${surname}`); + db.close(); +}); + +test('a family stake publishes ONLY when the official confirmed the company themselves', () => { + // The positive control for the narrowest published path. A family link can never earn „Документ" — + // the registered owner is the relative, whose name we never hold — so it stands or falls on the seat + // or ЕИК the OFFICIAL declared. Without this case „family published: 0" would be indistinguishable + // from a structurally dead path (ADR-0027). + runLoad(); + const db = open(); + const withSeat = db + .prepare( + "SELECT il.status, il.publish_tier FROM interest_links il JOIN persons p ON p.id=il.person_id WHERE il.eik='191919199' AND p.name='Кметица Иванова Втора'", + ) + .get(); + assert.equal(withSeat.status, 'published'); + assert.equal(withSeat.publish_tier, 'confirmed'); + const seal = db + .prepare( + "SELECT matched_fact FROM interest_link_evidence WHERE link_key LIKE '%191919199|family'", + ) + .get(); + assert.equal(seal.matched_fact, 'seat:РУСЕ'); + db.close(); +}); + +test('the published surface is exported BEFORE the wipe, so the audit can gate monotonicity', () => { + // ADR-0033 decision 6. The loader rebuilds the CACBG tables from scratch every run, so the previous + // published set exists only in the instant before the DROP. Without this export the audit has + // nothing to compare against and the monotonicity gate can never fire — which is precisely the state + // review found: rules_version was written and never read. + // A FIRST run, on its own database: interest_links does not exist yet, so the export must be an + // empty set that is WRITTEN rather than skipped — a missing file has to keep meaning „the loader + // never ran", not „nothing was published". + const fresh = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-load-first-')); + fs.mkdirSync(path.join(fresh, 'staging'), { recursive: true }); + for (const f of fs.readdirSync(STAGING)) + fs.copyFileSync(path.join(STAGING, f), path.join(fresh, 'staging', f)); + // The base DB (bidders, contracts) comes from the main pipeline and is a precondition of the load — + // a genuinely empty file is not a first run, it is a broken one. So: copy the base and drop the + // CACBG tables, which is exactly the state before the loader has ever run against it. + const firstDb = path.join(fresh, 'first.sqlite'); + fs.copyFileSync(DB, firstDb); + const fdb = new DatabaseSync(firstDb); + for (const t of [ + 'interest_link_evidence', + 'interest_link_authorities', + 'interest_links', + 'declared_interests', + 'related_persons_internal', + 'declarations', + 'persons', + ]) + fdb.exec(`DROP TABLE IF EXISTS ${t}`); + fdb.close(); + runLoad({ CACBG_DB: firstDb, CACBG_STAGING: path.join(fresh, 'staging') }); + assert.deepEqual( + JSON.parse(fs.readFileSync(path.join(fresh, 'staging', 'published-snapshot.json'), 'utf8')), + [], + ); + fs.rmSync(fresh, { recursive: true, force: true }); + + const SNAP = path.join(STAGING, 'published-snapshot.json'); + runLoad(); + const db = open(); + const published = db + .prepare("SELECT link_key FROM interest_links WHERE status='published'") + .all() + .map((r) => r.link_key); + db.close(); + assert.ok(published.length > 0, 'fixture must publish something for this test to mean anything'); + + runLoad(); + const snap = JSON.parse(fs.readFileSync(SNAP, 'utf8')); + assert.deepEqual( + snap.map((s) => s.link_key).sort(), + [...published].sort(), + 'the export must hold exactly what was published before the wipe', + ); + // rules_version travels WITH each key: the gate distinguishes „vanished under unchanged rules" + // (a regression) from „vanished under a rules bump" (an intentional removal), and it cannot do that + // from the current version alone. + for (const s of snap) + assert.ok(s.rules_version?.length > 0, 'each exported key carries its rules'); + // Held and withdrawn links must NOT be exported — they were never a public claim, so their absence + // next run is not a regression to gate on. + assert.equal(snap.length, published.length); +}); + +// The crawl input and the crawl's consumer are the same script, which used to make the two workflows +// deadlock: the decision run refused without a deed cache, the refresh run refused without a candidate +// list, and each produced only what the other needed. Merged into one job, the job still has to be able +// to BOOTSTRAP — produce the candidate list on a runner that has no cache yet — and it cannot do that by +// running the full load and ignoring a non-zero exit, because that exit is also how a genuinely broken +// run reports itself. Hence an explicit mode that stops at the list and succeeds. +test('--emit-candidates writes the crawl list and exits 0 with NO Trade Register cache', () => { + const gone = path.join(dir, 'bootstrap-absent.sqlite'); + const listFile = path.join(STAGING, 'candidate-eiks.txt'); + fs.rmSync(listFile, { force: true }); + assert.doesNotThrow(() => + execFileSync( + 'node', + [ + '--import', + path.join(HERE, 'register-ts.mjs'), + path.join(HERE, 'load.mjs'), + '--emit-candidates', + ], + { + cwd: ROOT, + env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING, TR_CACHE_DB: gone }, + stdio: 'pipe', + }, + ), + ); + const listed = fs.readFileSync(listFile, 'utf8').split('\n').filter(Boolean); + assert.ok(listed.length > 0, 'the bootstrap must actually produce candidates'); + assert.equal(new Set(listed).size, listed.length); + runLoad(); // restore the full built state for any later reader +}); + +function emitCandidates(trCacheDb) { + execFileSync( + 'node', + [ + '--import', + path.join(HERE, 'register-ts.mjs'), + path.join(HERE, 'load.mjs'), + '--emit-candidates', + ], + { + cwd: ROOT, + env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING, TR_CACHE_DB: trCacheDb }, + stdio: 'pipe', + }, + ); +} + +test('a bootstrap pass leaves the REAL work DB untouched — it runs on a throwaway copy', () => { + // Reaching the candidate list means rebuilding the corpus tables, and the pass never publishes, so + // against the real DB it would leave interest_links empty. That is the damage: not the pass itself, + // but what the NEXT run then reads. + runLoad(); + const db = open(); + const before = db.prepare('SELECT COUNT(*) AS n FROM interest_links').get().n; + db.close(); + assert.ok(before > 0, 'the fixture must actually publish something'); + + emitCandidates(path.join(dir, 'bootstrap-absent-2.sqlite')); + + const after = open(); + assert.equal( + after.prepare('SELECT COUNT(*) AS n FROM interest_links').get().n, + before, + 'the bootstrap pass must not empty the domain it was pointed at', + ); + after.close(); + assert.equal(fs.existsSync(`${DB}.bootstrap`), false, 'the throwaway copy must be cleaned up'); +}); + +test('the monotonicity gate still sees a prior surface AFTER a bootstrap pass', () => { + // The end-to-end shape of the merged workflow: real run → bootstrap (to produce the crawl list) → + // real run. If the bootstrap emptied interest_links, the second real run would export an EMPTY + // prior-published set, and the gate — whose only job is to notice a published claim disappearing — + // would pass unconditionally, for ever. + const snapshot = path.join(STAGING, 'published-snapshot.json'); + runLoad(); + emitCandidates(path.join(dir, 'bootstrap-absent-3.sqlite')); + runLoad(); + const prior = JSON.parse(fs.readFileSync(snapshot, 'utf8')); + assert.ok( + prior.length > 0, + 'the snapshot went empty — the gate is now vacuous and would never fire again', + ); +}); + +// ── the corrections list — ADR-0033 decision 6's second sanctioned removal ──────────────────────── +// Decision 6 licenses removal by „a rules-version bump, or a correction of wrong input". Only the +// first was expressible. Suppression cannot carry a correction: correcting the input UNBUILDS the +// link, and the B3 unused-suppression gate above then fails the build for a fingerprint that matched +// nothing — so the two sanctioned removals failed in opposite directions and a real correction had no +// path at all. The list is fingerprinted for the same reason ADR-0031's is: `pid|eik` in git would +// record which named official was linked to which company, for ever. + +test('a corrections entry marks its key in the snapshot, so the audit reads a declared removal', () => { + runLoad(); + const db = open(); + const key = db + .prepare("SELECT link_key FROM interest_links WHERE status='published' LIMIT 1") + .get().link_key; + db.close(); + + const corrFile = path.join(dir, 'corr.jsonl'); + fs.writeFileSync( + corrFile, + JSON.stringify({ + fp: fingerprint(key, SUPP_SALT), + key_version: '1', + reason: 'the declaration row was misparsed; the stake was never declared', + corrected_at: '2026-08-11', + }) + '\n', + ); + runLoad({ CACBG_CORRECTIONS_LIST: corrFile, SUPPRESSION_SALT: SUPP_SALT }); + + const snap = JSON.parse(fs.readFileSync(path.join(STAGING, 'published-snapshot.json'), 'utf8')); + const marked = snap.find((s) => s.link_key === key); + assert.ok(marked, 'the key must still be exported — the gate needs to SEE it leave'); + assert.equal(marked.corrected, true); + // Every other key stays unflagged: an acknowledgement is per-link, never a blanket amnesty. + assert.equal( + snap.filter((s) => s.corrected === true).length, + 1, + 'one acknowledgement must not clear the whole surface', + ); +}); + +test('a corrections entry matching NO previously published link fails the build', () => { + // The B3 rail, mirrored. A stale acknowledgement is worse than a missing one: it sits in the list + // and would clear a FUTURE disappearance of the same link — the exact regression the gate exists to + // catch — with nobody having decided that. + const corrFile = path.join(dir, 'corr-stale.jsonl'); + fs.writeFileSync( + corrFile, + JSON.stringify({ + fp: fingerprint('p-nobody|999999999', SUPP_SALT), + key_version: '1', + reason: 'stale', + corrected_at: '2026-08-11', + }) + '\n', + ); + runLoad(); + assert.throws( + () => runLoad({ CACBG_CORRECTIONS_LIST: corrFile, SUPPRESSION_SALT: SUPP_SALT }), + (err) => + /correction/i.test(String(err.stderr ?? '') + String(err.message ?? '')) && + /matched NO/i.test(String(err.stderr ?? '') + String(err.message ?? '')), + ); +}); + +test('corrections are fail-closed on a missing salt, exactly like suppressions', () => { + const corrFile = path.join(dir, 'corr-nosalt.jsonl'); + fs.writeFileSync( + corrFile, + JSON.stringify({ fp: 'deadbeef', key_version: '1', reason: 'x', corrected_at: '2026-08-11' }) + + '\n', + ); + assert.throws( + () => runLoad({ CACBG_CORRECTIONS_LIST: corrFile, SUPPRESSION_SALT: '' }), + (err) => /SUPPRESSION_SALT is unset/.test(String(err.stderr ?? '') + String(err.message ?? '')), + ); +}); diff --git a/scripts/cacbg/parse.mjs b/scripts/cacbg/parse.mjs index bf95f1548..6e89e2427 100644 --- a/scripts/cacbg/parse.mjs +++ b/scripts/cacbg/parse.mjs @@ -82,10 +82,18 @@ function cellsByNum(row) { } // find the @_Num of the first column whose @_Description matches `re` (labels live on the header row) function colNum(firstRow, re, fallback) { + const found = colNumOrNull(firstRow, re); + return found ?? fallback; +} +// The same resolution WITHOUT the fallback, for the one column where "I could not find it" and "I found it +// and it was blank" must stay distinguishable. `colNum` collapses them: an unresolvable column resolves to +// a Num that is not in the row, `by[col]` is undefined, and the caller reads an empty cell it never found. +// For the HOLDER column those two readings differ by who owns the stake (§1.4) — see its call site. +function colNumOrNull(firstRow, re) { for (const c of asArray(firstRow?.Cell)) { if (c?.['@_Num'] && re.test(String(c?.['@_Description'] ?? ''))) return c['@_Num']; } - return fallback; + return null; } const year4 = (s) => String(s ?? '').match(/\b(20\d{2})\b/)?.[1] ?? null; @@ -159,15 +167,20 @@ function parseAssets(pp) { ? colNum(rows[0], /наименование.*дружеств|фирма/i, '4') : colNum(rows[0], /емитент/i, '6'); const cSeat = colNum(rows[0], /седалище/i, '5'); - const cHolder = colNum(rows[0], /собствено.*фамил/i, isOod ? '7' : '8'); + // NO fallback here, unlike every other column (§1.4). A blank holder cell MEANS something — the stake is + // the declarant's own (classifyHolder) — so a column we failed to resolve must not imitate one. With a + // fallback, a renumbered table resolves the holder to a missing column, reads '' and calls a RELATIVE's + // stake the official's own, which then publishes as their private_ownership. Unresolvable ⇒ 'unknown', + // which forms no link at all: we did not read the holder, so we make no claim about them. + const cHolder = colNumOrNull(rows[0], /собствено.*фамил/i); const cEgn = colNum(rows[0], /^егн$/i, isOod ? '8' : '9'); for (const row of rows) { const by = cellsByNum(row); const company = by[cCompany] ?? ''; if (!company) continue; if ((by[cEgn] ?? '').length > 0) egnPresent = true; - const holder = by[cHolder] ?? ''; - const holderRelation = classifyHolder(holder, declarant); + const holderRelation = + cHolder === null ? 'unknown' : classifyHolder(by[cHolder] ?? '', declarant); if (holderRelation === 'related') familyHoldingCount += 1; const seat = isOod ? (by[cSeat] ?? '') : ''; interests.push({ diff --git a/scripts/cacbg/parse.test.mjs b/scripts/cacbg/parse.test.mjs index 30495481d..6a9b835e3 100644 --- a/scripts/cacbg/parse.test.mjs +++ b/scripts/cacbg/parse.test.mjs @@ -249,3 +249,35 @@ test('parseList: only values shaped like a declaration filename are announced', assert.equal(shape('Уведомление'), 0, 'a title slotted into the filename field'); assert.equal(shape('AAAA.pdf'), 0, 'not a declaration document'); }); + +test('§1.4: an UNRESOLVABLE holder column is unknown, not an own stake', () => { + // `colNum` always returns a fallback, so a table that renumbers the holder column away resolves it to a + // column that is not there: `by[cHolder]` is undefined → '' → classifyHolder('') → 'self'. A RELATIVE's + // stake then enters the OWN-only path and publishes as the official's private_ownership — the surface's + // worst failure, since the official did not declare that stake as theirs. + // + // The required distinction is column RESOLVABLE-BUT-EMPTY (→ self, a blank cell means the declarant) + // versus column NOT RESOLVABLE (→ unknown, we have not read the holder at all). + const renumbered = ` + 1 + "РЕНОМЕР" ЕООД + Мария Спасова Роднинска`; + const d = parseDeclaration(assetDecl({ rows: renumbered }), 'REN.xml'); + const it = d.interests.find((i) => i.entity === '"РЕНОМЕР" ЕООД'); + assert.equal(it.holderRelation, 'unknown'); + + // POSITIVE CONTROL 1 — the column resolves and the cell is genuinely EMPTY: still the declarant's own + // stake. Collapsing both cases to 'unknown' would silently drop every stake declared this way. + const blankHolder = ` + 1 + "ПРАЗНА" ЕООД + `; + const blank = parseDeclaration(assetDecl({ rows: blankHolder }), 'BLK.xml'); + assert.equal(blank.interests[0].holderRelation, 'self'); + + // POSITIVE CONTROL 2 — a resolvable column still classifies a relative and the declarant correctly, so + // the change bounds one case rather than blanketing the parser. + const ok = parseDeclaration(assetDecl({ rows: selfRow + familyRow }), 'OK.xml'); + assert.equal(ok.interests.find((i) => i.entity === '"ТЕСТ АГРО" ЕООД').holderRelation, 'self'); + assert.equal(ok.interests.find((i) => i.entity === '"ФАМИЛНА" ЕООД').holderRelation, 'related'); +}); diff --git a/scripts/cacbg/suppressions.mjs b/scripts/cacbg/suppressions.mjs index b1096c606..2c67c93d4 100644 --- a/scripts/cacbg/suppressions.mjs +++ b/scripts/cacbg/suppressions.mjs @@ -33,6 +33,25 @@ export function fingerprint(linkKey, salt) { * An absent or empty list needs no salt (nothing to fingerprint), so the common path stays friction-free. */ export function loadSuppressions(listPath, salt, keyVersion = SUPPRESSION_KEY_VERSION) { + return loadFingerprintedList(listPath, salt, keyVersion, 'suppression'); +} + +/** + * The MONOTONICITY-gate twin (ADR-0033 decision 6): links whose removal is licensed because their + * INPUT was wrong, not because the rules changed. + * + * A separate list from the suppressions, because the two are not the same act and cannot share one. + * A suppression keeps a built link OUT of the public surface; a correction says the link should never + * have been built at all — so correcting the input unbuilds it, and a suppression on it would then + * match nothing and trip the B3 unused gate. Same fingerprinting for the same reason: the raw + * `pid|eik` records which named official was tied to which company, which is precisely what must not + * live in git history. Consumed by load.mjs, which flags the matching keys in the pre-wipe snapshot. + */ +export function loadCorrections(listPath, salt, keyVersion = SUPPRESSION_KEY_VERSION) { + return loadFingerprintedList(listPath, salt, keyVersion, 'correction'); +} + +function loadFingerprintedList(listPath, salt, keyVersion, label) { const entries = existsSync(listPath) ? readFileSync(listPath, 'utf8') .split('\n') @@ -42,14 +61,14 @@ export function loadSuppressions(listPath, salt, keyVersion = SUPPRESSION_KEY_VE : []; if (entries.length > 0 && !salt) { throw new Error( - `${entries.length} suppression(s) in ${listPath} but SUPPRESSION_SALT is unset — refusing to build ` + + `${entries.length} ${label}(s) in ${listPath} but SUPPRESSION_SALT is unset — refusing to build ` + `(would silently un-suppress contested links). Set SUPPRESSION_SALT and retry.`, ); } for (const e of entries) { if (String(e.key_version ?? '') !== String(keyVersion)) { throw new Error( - `suppression ${String(e.fp ?? '').slice(0, 12)}… has key_version ${JSON.stringify(e.key_version)} ` + + `${label} ${String(e.fp ?? '').slice(0, 12)}… has key_version ${JSON.stringify(e.key_version)} ` + `but the current SUPPRESSION_KEY_VERSION is ${keyVersion} — refusing to build (a rotated salt ` + `would silently un-suppress it). Re-fingerprint every entry under the current salt + key_version.`, ); diff --git a/scripts/cacbg/tr-census.mjs b/scripts/cacbg/tr-census.mjs deleted file mode 100644 index e7b8c66d7..000000000 --- a/scripts/cacbg/tr-census.mjs +++ /dev/null @@ -1,195 +0,0 @@ -// TR name-uniqueness census (ADR-0015). Promotes tier-C held interest_links — generic company names -// with a single WINNER namesake — to 'published' iff the name is GLOBALLY unique in the Trade Register. -// -// Source: the Commercial Register open-data dump on data.egov.bg (DPA-safe: ЕГН/ЛНЧ hashed out, company -// name + ЕИК retained). We build companyNameKey(name) → {distinct ЕИК} over ALL active entities using -// the SAME normalizer as the matcher, so the key spaces are identical. Promotion is deterministic: -// key count == 1 AND that ЕИК == the matched winner ЕИК. Anything else stays held. No heuristic. -// -// Field-detection is structural (find the 9/13-digit ЕИК; find the company-name field), so it is robust -// to the exact open-data column names — confirm the mapping against the real dump before a production run. -// -// Run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/tr-census.mjs --dump -import { DatabaseSync } from 'node:sqlite'; -import fs from 'node:fs'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); -const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); -const arg = (n) => { - const i = process.argv.indexOf(`--${n}`); - return i >= 0 ? process.argv[i + 1] : undefined; -}; -const DUMP = arg('dump') || process.env.TR_DUMP; -const { companyNameKey } = await import('../../packages/shared/src/company-name-key.ts'); - -const isEik = (v) => typeof v === 'string' && /^\d{9}$|^\d{13}$/.test(v.trim()); - -// The register stores CompanyName WITHOUT the legal form and LegalForm as a code; our matcher keys on the -// name WITH its Bulgarian form (bidders.name = «…» ЕООД), so we must reconstruct it or every lookup misses. -// Codes seen in the real dump: EOOD OOD ASSOC CC K EAD ET AD KCHT FOUND KD SD DPK EDPK IAD IEAD. -const LEGAL_FORM = { - EOOD: 'ЕООД', - OOD: 'ООД', - EAD: 'ЕАД', - AD: 'АД', - ET: 'ЕТ', - KD: 'КД', - K: 'КД', - SD: 'СД', - KCHT: 'КДА', - IAD: 'ИАД', - IEAD: 'ИЕАД', - DPK: 'ПК', - EDPK: 'ЕПК', - CC: 'кооперация', - ASSOC: 'СНЦ', - FOUND: 'фондация', -}; -// A Bulgarian trade name reconstructed to match the matcher's key space. Unknown form → append the raw -// code (harmless: it just forms its own key that no bidder lookup will hit). -const bgName = (companyName, legalForm) => - `${String(companyName ?? '').trim()} ${LEGAL_FORM[legalForm] ?? String(legalForm ?? '').trim()}`.trim(); - -// Extract {eik, name} from ONE TR deed's attribute object ({CompanyName, LegalForm, UIC, DeedStatus, …}). -// Also tolerates a flat {eik/uic, name} record (test fixtures). ЕИК = the 9/13-digit UIC. -export function extractEntity(rec) { - const uic = rec.UIC ?? rec.uic ?? rec.eik ?? rec.EIK; - const eik = isEik(String(uic ?? '')) ? String(uic).trim() : null; - const name = - rec.CompanyName != null ? bgName(rec.CompanyName, rec.LegalForm) : (rec.name ?? null); - return { eik, name: name || null }; -} - -// Read a TR dump as deed-attribute records. The real data.egov.bg export nests them at -// Message[].Body[].Deeds[].Deed[], each carrying its fields under `$` (fast-xml/attr style). Falls back to -// a plain JSON array / {data:[…]} / JSONL of already-flat records so tests and other shapes still work. -function* records(dump) { - const raw = fs.readFileSync(dump, 'utf8'); - const trimmed = raw.trimStart(); - if (trimmed.startsWith('{') || trimmed.startsWith('[')) { - const parsed = JSON.parse(raw); - const deeds = parsed?.Message?.[0]?.Body?.[0]?.Deeds?.[0]?.Deed; - if (Array.isArray(deeds)) { - for (const d of deeds) yield d?.['$'] ?? d; - return; - } - const arr = Array.isArray(parsed) - ? parsed - : (parsed.data ?? parsed.records ?? parsed.result ?? []); - yield* arr; - } else { - for (const line of raw.split('\n')) { - const t = line.trim(); - if (t) yield JSON.parse(t); - } - } -} - -// Build companyNameKey → Set(ЕИК) over one or more dump files (accumulates — call with every daily file to -// approach the full register). Deduped by ЕИК within a key, so the same company appearing on many days -// (or a name→one company) counts once; two DISTINCT companies sharing a name yield size>1 = not unique. -export function buildCensus(dumps) { - const census = new Map(); - for (const dump of Array.isArray(dumps) ? dumps : [dumps]) { - for (const rec of records(dump)) { - const { eik, name } = extractEntity(rec); - if (!eik || !name) continue; - const key = companyNameKey(name); - if (!census.has(key)) census.set(key, new Set()); - census.get(key).add(eik); - } - } - return census; -} - -// Promote held tier-C links the census proves globally unique: name-key maps to EXACTLY one ЕИК AND that -// ЕИК is the matched winner's (a national namesake, even at a different ЕИК, keeps the link held). With -// dryRun, report the would-promote set without writing — used to validate against a PARTIAL census, which -// must never actually promote (an un-ingested namesake would make a false-unique claim = libel). -export function promote(db, census, { dryRun = false, minEik = null, forcePartial = false } = {}) { - // Partial-census guard (libel gate). A census smaller than the real ТР register can make a genuinely - // non-unique name look unique (`eiks.size === 1`) → a false, libelous attribution. A real promote must - // ASSERT coverage in code, not by convention: pass `--min-eik ≥ register size` (the census's distinct-ЕИК - // count must meet it) or the explicit `--force-partial` escape hatch. dryRun is exempt — it writes nothing - // and exists precisely to inspect a partial census. - if (!dryRun && !forcePartial) { - if (!Number.isInteger(minEik) || minEik < 1) { - throw new Error( - 'refusing to promote without --min-eik : a partial census silently fabricates ' + - 'false-unique attributions. Pass --min-eik ≥ the known ТР register size, or --force-partial to override.', - ); - } - const distinctEik = new Set(); - for (const s of census.values()) for (const e of s) distinctEik.add(e); - if (distinctEik.size < minEik) { - throw new Error( - `refusing to promote from a partial census: ${distinctEik.size} distinct ЕИК < --min-eik ${minEik}. ` + - 'Ingest more ТР dumps until coverage meets the register size, or pass --force-partial to override.', - ); - } - } - const held = db - .prepare( - "SELECT link_key, eik, entity_key FROM interest_links WHERE status='held' AND publish_tier='C_hold'", - ) - .all(); - const upd = db.prepare( - "UPDATE interest_links SET status='published', match_method='exact_name_key+tr_census' WHERE link_key=?", - ); - const would = []; - if (!dryRun) db.exec('BEGIN'); - for (const l of held) { - const eiks = census.get(l.entity_key); - if (eiks && eiks.size === 1 && eiks.has(l.eik)) { - would.push(l.link_key); - if (!dryRun) upd.run(l.link_key); - } - } - if (!dryRun) db.exec('COMMIT'); - return { promoted: would.length, stillHeld: held.length - would.length, would }; -} - -// Resolve --dump: a single file, a comma-list, or --dump-dir (every *.json/*.jsonl inside). -function dumpFiles() { - const dir = arg('dump-dir'); - if (dir) - return fs - .readdirSync(dir) - .filter((f) => /\.(json|jsonl)$/.test(f)) - .map((f) => path.join(dir, f)); - return DUMP - ? DUMP.split(',') - .map((s) => s.trim()) - .filter(Boolean) - : []; -} - -// CLI entry (guarded so importing this module in tests has no side effects) -const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); -if (isMain) { - const files = dumpFiles(); - const dryRun = process.argv.includes('--dry-run'); - const forcePartial = process.argv.includes('--force-partial'); - const minEik = arg('min-eik') !== undefined ? Number(arg('min-eik')) : null; - if (!files.length) { - console.log( - 'no --dump/--dump-dir provided; census not run. Provide TR open-data files to promote tier-C links.', - ); - } else { - const db = new DatabaseSync(DB); - const census = buildCensus(files); - const eikTotal = [...census.values()].reduce((n, s) => n + s.size, 0); - console.log( - `census: ${files.length} file(s) → ${census.size} distinct name-keys / ${eikTotal} ЕИК`, - ); - const { promoted, stillHeld, would } = promote(db, census, { dryRun, minEik, forcePartial }); - console.log( - `${dryRun ? 'DRY-RUN would promote' : 'tier-C promotions'}: ${promoted} published, ${stillHeld} still held (non-unique or namesake mismatch)`, - ); - if (dryRun && would.length) - console.log(' would-promote link_keys:\n ' + would.join('\n ')); - db.close(); - } -} diff --git a/scripts/cacbg/tr-census.test.mjs b/scripts/cacbg/tr-census.test.mjs deleted file mode 100644 index 76d2fbb0f..000000000 --- a/scripts/cacbg/tr-census.test.mjs +++ /dev/null @@ -1,121 +0,0 @@ -// TR census — real data.egov.bg deed shape + deterministic tier-C promotion. -// Run: node --import ./scripts/cacbg/register-ts.mjs --test scripts/cacbg/tr-census.test.mjs -import { test, before, after } from 'node:test'; -import assert from 'node:assert/strict'; -import { DatabaseSync } from 'node:sqlite'; -import fs from 'node:fs'; -import path from 'node:path'; -import os from 'node:os'; -import { extractEntity, buildCensus, promote } from './tr-census.mjs'; -const { companyNameKey } = await import('../../packages/shared/src/company-name-key.ts'); - -let dir, dump, dbPath; -const K = (s) => companyNameKey(s); - -before(() => { - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-census-')); - dump = path.join(dir, 'tr.json'); - // Real export shape: Message[].Body[].Deeds[].Deed[], each with its fields under `$`; CompanyName is - // bare and LegalForm is a code — the census must reconstruct «name + Bulgarian form» to match bidders. - const deeds = [ - { CompanyName: 'СИЙ', LegalForm: 'AD', UIC: '444444447' }, // globally unique → promote - { CompanyName: 'ОБЩА ФИРМА', LegalForm: 'OOD', UIC: '555555556' }, // one of two namesakes - { CompanyName: 'Обща Фирма', LegalForm: 'OOD', UIC: '666666663' }, // second namesake → NOT unique - { CompanyName: 'СИЙ', LegalForm: 'AD', UIC: '444444447' }, // same deed on another day → deduped - ]; - fs.writeFileSync( - dump, - JSON.stringify({ Message: [{ Body: [{ Deeds: [{ Deed: deeds.map((d) => ({ $: d })) }] }] }] }), - ); - - dbPath = path.join(dir, 'db.sqlite'); - const db = new DatabaseSync(dbPath); - db.exec(` - CREATE TABLE interest_links(link_key TEXT PRIMARY KEY, eik TEXT, entity_key TEXT, status TEXT, publish_tier TEXT, match_method TEXT); - INSERT INTO interest_links VALUES ('p1|444444447','444444447','${K('СИЙ АД')}','held','C_hold','exact_name_key'); - INSERT INTO interest_links VALUES ('p2|555555556','555555556','${K('ОБЩА ФИРМА ООД')}','held','C_hold','exact_name_key'); - `); - db.close(); -}); - -after(() => fs.rmSync(dir, { recursive: true, force: true })); - -test('extractEntity reconstructs «name + Bulgarian legal form» from a TR deed and reads UIC as ЕИК', () => { - assert.deepEqual(extractEntity({ CompanyName: 'СИЙ', LegalForm: 'AD', UIC: '444444447' }), { - eik: '444444447', - name: 'СИЙ АД', - }); - assert.deepEqual(extractEntity({ CompanyName: 'ТЕСТ', LegalForm: 'EOOD', UIC: '205057459' }), { - eik: '205057459', - name: 'ТЕСТ ЕООД', - }); - assert.equal(extractEntity({ CompanyName: 'X', LegalForm: 'AD', UIC: '2018060520' }).eik, null); // 10 digits ≠ ЕИК - assert.equal(extractEntity({ name: 'flat ООД', uic: '5555555560000' }).eik, '5555555560000'); // flat fallback + 13-digit -}); - -test('buildCensus indexes name-key → set of ЕИК over the nested export (dedup by ЕИК)', () => { - const c = buildCensus(dump); - assert.equal(c.get(K('СИЙ АД')).size, 1); // appears twice, one ЕИК → deduped - assert.equal(c.get(K('ОБЩА ФИРМА ООД')).size, 2); // two distinct ЕИК fold to one key → non-unique -}); - -test('promote publishes only globally-unique tier-C links; shared names stay held', () => { - const db = new DatabaseSync(dbPath); - // census covers 3 distinct ЕИК → assert coverage with --min-eik 3 (the partial-census gate). - const res = promote(db, buildCensus(dump), { minEik: 3 }); - assert.equal(res.promoted, 1); - const rows = new Map( - db - .prepare('SELECT link_key,status,match_method FROM interest_links') - .all() - .map((r) => [r.link_key, r]), - ); - assert.equal(rows.get('p1|444444447').status, 'published'); - assert.match(rows.get('p1|444444447').match_method, /tr_census/); - assert.equal(rows.get('p2|555555556').status, 'held'); // two namesakes → stays held - db.close(); -}); - -test('partial-census gate: a real promote refuses without --min-eik, and when coverage is below it', () => { - const db = new DatabaseSync(dbPath); - // isolate: p1 back to held (an earlier test may have published it — shared dbPath) - db.exec( - "UPDATE interest_links SET status='held', match_method='exact_name_key' WHERE link_key='p1|444444447'", - ); - const census = buildCensus(dump); // 3 distinct ЕИК - // no --min-eik → refuse (can't assert completeness → could fabricate a false-unique attribution) - assert.throws(() => promote(db, census, {}), /--min-eik/); - // --min-eik above the census's coverage → refuse (partial dump) - assert.throws(() => promote(db, census, { minEik: 4 }), /partial census|distinct ЕИК/); - // the refused runs published nothing — p1 is still held - assert.equal( - db.prepare("SELECT status FROM interest_links WHERE link_key='p1|444444447'").get().status, - 'held', - ); - db.close(); -}); - -test('partial-census gate: --force-partial is the explicit override', () => { - const db = new DatabaseSync(dbPath); - db.exec( - "UPDATE interest_links SET status='held', match_method='exact_name_key' WHERE link_key='p1|444444447'", - ); - const res = promote(db, buildCensus(dump), { forcePartial: true }); // no minEik, but forced - assert.equal(res.promoted, 1); - db.close(); -}); - -test('dry-run reports would-promote set without mutating the DB', () => { - const db = new DatabaseSync(dbPath); - // reset p1 back to held for an isolated dry-run check - db.exec( - "UPDATE interest_links SET status='held', match_method='exact_name_key' WHERE link_key='p1|444444447'", - ); - const res = promote(db, buildCensus(dump), { dryRun: true }); - assert.deepEqual(res.would, ['p1|444444447']); - assert.equal( - db.prepare("SELECT status FROM interest_links WHERE link_key='p1|444444447'").get().status, - 'held', - ); // untouched - db.close(); -}); diff --git a/scripts/precompute.sql b/scripts/precompute.sql index f57cbedd0..f93a0a790 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -246,6 +246,13 @@ FROM interest_links il JOIN persons p ON p.id = il.person_id -- rendering both re-identifies the relative via a ТР owner lookup, and the company is already surfaced -- by the self row. WHERE il.status = 'published' AND il.interest_class IN ('private_ownership', 'family_ownership') + -- …and the identity rests on a Trade Register fact (#279, ADR-0033). This predicate is the THIRD copy + -- of the surface gate — the other two are SURFACED_OWNERSHIP in packages/db/src/queries/related-persons.ts + -- and the sibling block in the other of precompute.sql / refresh-slice.sql. All three must move + -- together: this one feeds the officials search index, so omitting it would keep officials findable + -- whose links no longer surface. + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed')) AND EXISTS (SELECT 1 FROM contracts cc JOIN bidders bb ON bb.id = cc.bidder_id WHERE bb.eik_normalized = il.eik) AND NOT (il.interest_class = 'family_ownership' AND EXISTS ( diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 0f6fac957..60854cf5f 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -2501,6 +2501,13 @@ FROM interest_links il JOIN persons p ON p.id = il.person_id -- rendering both re-identifies the relative via a ТР owner lookup, and the company is already surfaced -- by the self row. WHERE il.status = 'published' AND il.interest_class IN ('private_ownership', 'family_ownership') + -- …and the identity rests on a Trade Register fact (#279, ADR-0033). This predicate is the THIRD copy + -- of the surface gate — the other two are SURFACED_OWNERSHIP in packages/db/src/queries/related-persons.ts + -- and the sibling block in the other of precompute.sql / refresh-slice.sql. All three must move + -- together: this one feeds the officials search index, so omitting it would keep officials findable + -- whose links no longer surface. + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed')) AND EXISTS (SELECT 1 FROM contracts cc JOIN bidders bb ON bb.id = cc.bidder_id WHERE bb.eik_normalized = il.eik) AND NOT (il.interest_class = 'family_ownership' AND EXISTS ( diff --git a/scripts/seed.sql b/scripts/seed.sql index 2ff25add1..3d26a0da1 100644 --- a/scripts/seed.sql +++ b/scripts/seed.sql @@ -16,3 +16,103 @@ INSERT OR IGNORE INTO bidders (id, name, bulstat) VALUES ('bidder-a', 'Алфа ЕООД', '111111111'), ('bidder-b', 'Бета АД', '222222222'), ('bidder-c', 'Гама ООД', '333333333'); + +-- ── свързани лица (conflict-of-interest) smoke fixture ──────────────────────────────────────────── +-- Without these the /conflicts routes render an empty surface locally, so the whole feature is +-- unverifiable in a browser. Every person and company below is INVENTED — no real declarant, no real +-- winner. The ЕИК pass the real 9-digit checksum (scripts/normalize-raw.sql's eik_valid), so the +-- fixture cannot teach a wrong ЕИК shape to anyone reading it. +-- +-- The rows are shaped by what the read gate actually requires (packages/db/src/queries/related-persons.ts): +-- • status='published' AND interest_class IN (private_ownership, family_ownership) — SURFACED_OWNERSHIP +-- • a LIVE contract for that ЕИК — the read-time zero-contract gate +-- • signed_at inside [first_declared_year, last_declared_year] — IN_WINDOW ⇒ the „в периода" chip +-- • no published SELF stake on the same (official, ЕИК) for a family row — NOT_REDUNDANT_FAMILY +-- • a Trade Register evidence seal of a PUBLISHING rung — the #279 seal gate +-- One link per outcome, including the non-surfaced ones, so a change to the publishing rule is +-- visible as a before/after rather than as an empty page either way. + +INSERT OR IGNORE INTO authorities (id, name, bulstat, region) VALUES + ('auth-plovdiv', 'Община Пример', '131223340', 'Пловдив'); + +INSERT OR IGNORE INTO tenders + (id, source_id, title, authority_id, cpv_code, estimated_value, currency, procedure_type, status, published_at, deadline_at) +VALUES + ('t-conflict-01', 'AOP-2022-0101', 'Строителен надзор на общински обекти', 'auth-plovdiv', '71520000', 900000, 'BGN', 'открита процедура', 'closed', '2022-01-10', '2022-02-10'), + ('t-conflict-02', 'AOP-2022-0102', 'Консултантски услуги по проект', 'auth-sofia', '79400000', 400000, 'BGN', 'открита процедура', 'closed', '2022-04-05', '2022-05-05'), + ('t-conflict-03', 'AOP-2022-0103', 'Доставка на офис оборудване', 'auth-mrrb', '30190000', 250000, 'BGN', 'открита процедура', 'closed', '2022-06-01', '2022-07-01'); + +-- eik_valid=1 so these behave like resolved winners; settlement feeds the seat-confirmation rung. +INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, legal_form, settlement) VALUES + ('eik:201122335', 'АЛФА СТРОЙ ООД', '201122335', '201122335', 1, 'ООД', 'Пловдив'), + ('eik:203445566', 'БЕТА КОНСУЛТ ЕООД', '203445566', '203445566', 1, 'ЕООД', 'София'), + ('eik:204556676', 'ГАМА ИНВЕСТ АД', '204556676', '204556676', 1, 'АД', 'Варна'); + +INSERT OR IGNORE INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES + ('c-conflict-01', 't-conflict-01', 'eik:201122335', 840000, 'BGN', '2022-03-14', 'Д-2022-101', 429000), + ('c-conflict-02', 't-conflict-02', 'eik:203445566', 372000, 'BGN', '2022-06-20', 'Д-2022-102', 190000), + ('c-conflict-03', 't-conflict-03', 'eik:204556676', 236000, 'BGN', '2022-08-02', 'Д-2022-103', 120000); + +-- person id = 'person:' || key(name) || '|' || key(institution) — (name, institution), never a bare +-- name (ADR-0026), so two namesakes at different bodies stay distinct. +INSERT OR IGNORE INTO persons (id, name) VALUES + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'Иван Петров Тестов'), + ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ', 'Мария Георгиева Образцова'); + +INSERT OR IGNORE INTO declarations + (id, person_id, xml_file, control_hash, folder_year, declared_year, template, category, institution, position, source_url) +VALUES + ('decl:2023:demo-ivan.xml', 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'demo-ivan.xml', 'demo-hash-ivan', '2023', '2023', 'assets', 'Местна власт', 'Община Пример', 'Кмет', 'https://register.cacbg.bg/2023/demo-ivan.xml'), + ('decl:2023:demo-maria.xml', 'person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ', 'demo-maria.xml', 'demo-hash-maria', '2023', '2023', 'assets', 'Местна власт', 'Община София', 'Директор дирекция', 'https://register.cacbg.bg/2023/demo-maria.xml'); + +INSERT OR IGNORE INTO declared_interests (id, declaration_id, entity_raw, entity_key, kind, detail, timing, seat) VALUES + ('di:decl:2023:demo-ivan.xml:1', 'decl:2023:demo-ivan.xml', 'АЛФА СТРОЙ ООД', 'АЛФА СТРОЙ ООД', 'shares', '50%', 'annual', 'Пловдив'), + ('di:decl:2023:demo-ivan.xml:2', 'decl:2023:demo-ivan.xml', 'ГАМА ИНВЕСТ АД', 'ГАМА ИНВЕСТ АД', 'shares', '0,5%', 'annual', 'Варна'), + ('di:decl:2023:demo-maria.xml:1', 'decl:2023:demo-maria.xml', 'БЕТА КОНСУЛТ ЕООД', 'БЕТА КОНСУЛТ ЕООД', 'shares', '100%', 'annual', 'София'); + +-- Three links, three outcomes: +-- published private_ownership — the official's own stake, own_institution='exact' (the strongest signal) +-- published family_ownership — a relative's declared stake; the relative is never named (ADR-0032) +-- held (АД, C_hold) — a joint-stock parcel: collected, never surfaced (ADR-0022 materiality) +INSERT OR IGNORE INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, + relation, interest_class, contemporaneous, own_institution, evidence_count, + first_declared_year, last_declared_year, contract_count, contract_value_eur, + first_contract_year, last_contract_year, status) +VALUES + ('il:person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', + 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'eik:201122335', '201122335', 'АЛФА СТРОЙ ООД', + 'exact_name_key', 'seed-demo', 'B_distinctive', 'owns', 'private_ownership', 1, 'exact', 1, + '2021', '2023', 1, 429000, '2022', '2022', 'published'), + ('il:person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', + 'person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ', 'eik:203445566', '203445566', 'БЕТА КОНСУЛТ ЕООД', + 'exact_name_key', 'seed-demo', 'A_seat', 'related', 'family_ownership', 1, 'exact', 1, + '2021', '2023', 1, 190000, '2022', '2022', 'published'), + ('il:person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|204556676', 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|204556676', + 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'eik:204556676', '204556676', 'ГАМА ИНВЕСТ АД', + 'exact_name_key', 'seed-demo', 'C_hold', 'owns', 'management_role', 0, 'none', 1, + '2021', '2023', 1, 120000, '2022', '2022', 'held'); + +INSERT OR IGNORE INTO interest_link_authorities (link_key, authority_id, authority_name, contract_count, value_eur, own) VALUES + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'auth-plovdiv', 'Община Пример', 1, 429000, 'exact'), + ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'auth-sofia', 'Община София', 1, 190000, 'exact'); + +-- Trade Register evidence seals (#279, ADR-0033, migration 0006). NOT optional decoration: since the +-- evidence ladder landed, SURFACED_OWNERSHIP requires a publishing seal, so a seeded link without one +-- renders NOTHING — which is the exact failure this fixture exists to prevent. A seal per link, one per +-- rung, so the dev surface shows the same three outcomes as production: +-- document — the register names this person in this company (the strongest rung) +-- confirmed — identity confirmed by declared data (seat), no person found in the act itself +-- bar_joint_stock — an АД: a declared parcel of shares is not a material ownership conflict +-- matched_fact stays inside the closed vocabulary (evidence.mjs isSealedFact) — never a name. +INSERT OR IGNORE INTO interest_link_evidence + (link_key, evidence_kind, registry_role, matched_fact, entry_number, entry_date, lookup_date, rules_version, live_status) +VALUES + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'document', 'owner', 'role:owner:CR_F_19_L', + '20220314150210', '2022-03-14', '2026-08-05', 'tr-rules-1', 'live'), + ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'confirmed', NULL, 'seat:СОФИЯ', + '20210902110455', '2021-09-02', '2026-08-05', 'tr-rules-1', 'live'), + -- The held link is sealed too — seals exist for held links so the review queue can explain itself, + -- and this one doubles as the fixture's negative case for the seal gate. + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|204556676', 'bar_joint_stock', NULL, NULL, + NULL, NULL, '2026-08-05', 'tr-rules-1', 'live'); diff --git a/scripts/ship-e2e.test.mjs b/scripts/ship-e2e.test.mjs index cf971a2e8..1d1509148 100644 --- a/scripts/ship-e2e.test.mjs +++ b/scripts/ship-e2e.test.mjs @@ -28,6 +28,10 @@ const HERE = dirname(fileURLToPath(import.meta.url)); const ROOT = resolve(HERE, '..'); const SCRIPT = resolve(HERE, 'ship-related-persons.mjs'); const MIG = resolve(ROOT, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// 0006 too: the evidence seal is one of the shipped tables (#279, ADR-0033), so it appears in the ship's +// TABLES and WIPE_ORDER. A target built from 0003 alone makes the generated wipe abort on „no such table" +// before a single row moves — the schema under test has to be the schema the ship writes. +const MIG_EVIDENCE = resolve(ROOT, 'packages/db/migrations/0009_interest_link_evidence.sql'); const D1_NAME = 'sigma-test-local'; // Derived from BOTH production constants, and the run below does NOT override either: one row past @@ -51,11 +55,12 @@ if (spawnSync('sqlite3', ['-version'], { stdio: 'ignore' }).error) { throw new Error('scripts/ship-e2e.test.mjs requires the sqlite3 binary on PATH'); } -/** The five served tables plus the two FK parents they reference. */ +/** The served свързани-лица tables plus the two FK parents they reference. */ const SCHEMA = `PRAGMA foreign_keys=ON; CREATE TABLE bidders(id TEXT PRIMARY KEY); CREATE TABLE authorities(id TEXT PRIMARY KEY); .read ${MIG} +.read ${MIG_EVIDENCE} INSERT INTO bidders(id) VALUES('eik:1'); INSERT INTO authorities(id) VALUES('auth:1');`; @@ -66,7 +71,11 @@ INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VAL ${Array.from( { length: links }, (_, i) => - `INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('il${i}','p1|${i}','p1','eik:1','1','e','v1','B_distinctive','owns','published');`, + `INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('il${i}','p1|${i}','p1','eik:1','1','e','v1','B_distinctive','owns','published');\n` + + // One seal per link. Since #279 a published link without one is exactly the state the audit fails + // the run on (C_no_evidence) and the read gate refuses to surface, so a corpus without seals is not + // a shape the ship should ever be asked to carry. + `INSERT INTO interest_link_evidence(link_key,evidence_kind,lookup_date,rules_version,live_status) VALUES('p1|${i}','document','2026-08-13','tr-rules-1','live');`, ).join('\n')} `; // interest_link_authorities is deliberately left EMPTY above: its expected count is 0, which is the @@ -77,13 +86,15 @@ INSERT INTO persons(id,name) VALUES('stale','Стар запис'); INSERT INTO declarations(id,person_id,xml_file,folder_year,template,source_url) VALUES('sd','stale','s.xml','2019','assets','u'); INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VALUES('sdi','sd','S','s','shares'); INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('sil','stale|0','stale','eik:1','1','s','v0','B_distinctive','owns','published'); -INSERT INTO interest_link_authorities(link_key,authority_id,authority_name) VALUES('stale|0','auth:1','A');`; +INSERT INTO interest_link_authorities(link_key,authority_id,authority_name) VALUES('stale|0','auth:1','A'); +INSERT INTO interest_link_evidence(link_key,evidence_kind,lookup_date,rules_version,live_status) VALUES('stale|0','confirmed','2019-01-01','tr-rules-0','live');`; const EXPECTED_ROWS = { persons: 1, declarations: 1, declared_interests: 1, interest_links: LINKS, + interest_link_evidence: LINKS, // one seal per link (#279) interest_link_authorities: 0, }; @@ -255,10 +266,15 @@ test('a request that never landed fails the run', (t) => { const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-short-')); t.after(() => rmSync(dir, { recursive: true, force: true })); - const { res } = runShip(dir, { env: { SHIP_FAKE_SKIP: 'interest_links.2.sql' } }); + // The skip targets a LEAF table on purpose. Since #279 the ship also carries interest_link_evidence, + // which has an FK to interest_links — so dropping an interest_links chunk now fails on the FOREIGN KEY + // as the orphaned seals land, before the read-back ever runs. That is a stronger guard, but it would + // leave the read-back gate itself unexercised, which is what this test is for. A seal chunk has no + // dependents, so its loss is invisible until the counts are compared. + const { res } = runShip(dir, { env: { SHIP_FAKE_SKIP: 'interest_link_evidence.2.sql' } }); assert.notEqual(res.status, 0, 'a short target must fail the run'); assert.match(res.stderr, /ship verification FAILED/); - assert.match(res.stderr, /interest_links: shipped \d+, target has \d+/); + assert.match(res.stderr, /interest_link_evidence: shipped \d+, target has \d+/); }); test('a read-back that answers with a non-number fails closed', (t) => { diff --git a/scripts/ship-related-persons.mjs b/scripts/ship-related-persons.mjs index 4188f6c81..543ce2b56 100644 --- a/scripts/ship-related-persons.mjs +++ b/scripts/ship-related-persons.mjs @@ -28,6 +28,8 @@ export const TABLES = [ 'declarations', 'declared_interests', 'interest_links', + // AFTER interest_links: a seal references its link, so inserting it first fails the FK (#279). + 'interest_link_evidence', 'interest_link_authorities', ]; // DELETE order for the pre-insert wipe — children before parents. related_persons_internal (PII, never @@ -35,6 +37,8 @@ export const TABLES = [ // carrying internal rows would block DELETE FROM declarations. export const WIPE_ORDER = [ 'interest_link_authorities', + // BEFORE interest_links, for the mirror reason: deleting a link whose seal survives fails the FK. + 'interest_link_evidence', 'related_persons_internal', 'interest_links', 'declared_interests', diff --git a/scripts/ship-related-persons.test.mjs b/scripts/ship-related-persons.test.mjs index b0d2d56a6..39e99d17c 100644 --- a/scripts/ship-related-persons.test.mjs +++ b/scripts/ship-related-persons.test.mjs @@ -13,6 +13,7 @@ import { sqlLiteral, sqlIdent, TABLES, + WIPE_ORDER, } from './ship-related-persons.mjs'; test('sqlLiteral escapes quotes, strips NUL, and NULLs non-finite/absent', () => { @@ -73,11 +74,31 @@ test('TABLES ships parents before children and covers the served related-persons 'declared_interests', 'interest_links', 'interest_link_authorities', + 'interest_link_evidence', ]) { assert.ok(TABLES.includes(t), `missing ${t}`); } }); +test('the evidence seal ships AFTER the links it references, and is wiped BEFORE them', () => { + // D1 enforces foreign keys, so ordering is not cosmetic: inserting a seal before its link fails, + // and deleting a link while a seal still references it fails the re-seed at interest_links. + assert.ok( + TABLES.indexOf('interest_link_evidence') > TABLES.indexOf('interest_links'), + 'a seal inserted before its link violates the FK', + ); + assert.ok( + WIPE_ORDER.indexOf('interest_link_evidence') < WIPE_ORDER.indexOf('interest_links'), + 'a link deleted while its seal survives violates the FK', + ); +}); + +test('every shipped table is wiped, and every wiped table is real', () => { + // A table added to TABLES but forgotten in WIPE_ORDER accumulates stale rows on every re-ship — + // the surface would then carry evidence for links that no longer exist. + for (const t of TABLES) assert.ok(WIPE_ORDER.includes(t), `${t} ships but is never wiped`); +}); + test('assertD1TargetAuthorized: declared env + allowlisted name + (name↔id) before a remote wipe (T48)', () => { const ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'; const ok = { diff --git a/scripts/ship-reseed.test.mjs b/scripts/ship-reseed.test.mjs index 4864029dd..44020c5c1 100644 --- a/scripts/ship-reseed.test.mjs +++ b/scripts/ship-reseed.test.mjs @@ -15,6 +15,9 @@ import { wipeSql, TABLES } from './ship-related-persons.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); const MIG = resolve(HERE, '..', 'packages/db/migrations/0003_related_persons_foundation.sql'); +// 0006 adds interest_link_evidence, which references interest_links — so it is part of the FK graph +// this test exists to police, and the wipe must clear it before its parent (#279, ADR-0033). +const MIG_EVIDENCE = resolve(HERE, '..', 'packages/db/migrations/0009_interest_link_evidence.sql'); // Stub only the two 0000 tables 0002's FKs reference (bidders, authorities) — a PK column is all a FK needs — // then apply the real 0002 and seed one row down every FK chain so a parent delete has live children. @@ -25,6 +28,7 @@ function newPopulatedDb() { CREATE TABLE bidders(id TEXT PRIMARY KEY); CREATE TABLE authorities(id TEXT PRIMARY KEY); .read ${MIG} +.read ${MIG_EVIDENCE} INSERT INTO bidders(id) VALUES('eik:1'); INSERT INTO authorities(id) VALUES('auth:1'); INSERT INTO persons(id,name) VALUES('p1','П Тест'); @@ -33,6 +37,7 @@ INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VAL INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('il1','p1|1','p1','eik:1','1','e','v1','B_distinctive','owns','published'); INSERT INTO interest_link_authorities(link_key,authority_id,authority_name) VALUES('p1|1','auth:1','A'); INSERT INTO related_persons_internal(id,declaration_id,related_name,related_kind) VALUES('rp1','d1','X','related_person'); +INSERT INTO interest_link_evidence(link_key,evidence_kind,matched_fact,lookup_date,rules_version,live_status) VALUES('p1|1','document','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'); `; execFileSync('sqlite3', ['-bail', db], { input: setup, stdio: 'pipe' }); return { dir, db }; diff --git a/scripts/tr/cache.mjs b/scripts/tr/cache.mjs new file mode 100644 index 000000000..764c16f2c --- /dev/null +++ b/scripts/tr/cache.mjs @@ -0,0 +1,278 @@ +// The deed cache — resumability, and the PII rail (issue #279, ADR-0033 decision 5). +// +// A registry deed contains third-party personal data: the names of owners and managers who hold no +// public office, and the company's street address. Two rails follow from that, and both live here: +// +// 1. The INDEX stores no name at all — ЕИК, dates, codes, verdicts, and a hash of the body. Names +// exist only in the raw JSON under git-ignored scratch/, are read only to produce a boolean, and +// never enter a public table, a response or a log. A hash rather than an excerpt, because an +// excerpt of a deed IS third-party personal data. +// 2. Nothing may carry a STANDALONE ten-digit run. That is the ЕГН shape, and the check is sound +// precisely because an ЕИК is 9 or 13 digits — never 10 — so it cannot reject a legitimate +// identifier. „Standalone" is load-bearing: a 13-digit ЕИК contains ten-digit substrings, so an +// unanchored match would refuse every клон. ЕГН was absent from every payload examined; this is +// the rail for the day one leaks. +// +// Resumability is the other job: the register throttles hard and a 429 ends the run (client.mjs), so a +// crawl must be able to pick up exactly where it stopped without re-requesting what it already has. + +import { DatabaseSync } from 'node:sqlite'; +import fs from 'node:fs'; +import path from 'node:path'; +import { safeEik } from './paths.mjs'; + +const SCHEMA = ` +CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, + status TEXT NOT NULL, -- fetched | outside_tr + http_status INTEGER, + fetched_at TEXT NOT NULL, + raw_path TEXT, -- relative to the raw dir; the ONLY place names live + body_sha256 TEXT, -- integrity + change detection, never an excerpt + legal_form_code INTEGER, + legal_form_verdict TEXT, -- closely_held | joint_stock | unknown (unknown WITHHOLDS) + seat_normalized TEXT, -- settlement only; never a street address (ADR-0010 item 3) + seat_entry_date TEXT, + latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, + outside_reason TEXT +); +CREATE INDEX IF NOT EXISTS idx_deeds_status ON deeds(status); +CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); +`; + +/** Open (creating if absent) the cache at `file`. Idempotent — never wipes an existing cache. */ +export function openCache(file) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const db = new DatabaseSync(file); + db.exec('PRAGMA journal_mode = WAL'); + db.exec(SCHEMA); + return db; +} + +// ── the ЕГН rail ────────────────────────────────────────────────────────────── +// ANCHORED, and that is the whole correctness of the rail. An ЕИК is 9 or 13 digits — never 10 — so +// a ten-digit run cannot be a legitimate identifier here. But that reasoning only holds when the run +// is matched as a WHOLE: an unanchored /\d{10}/ matches INSIDE the 13-digit ЕИК of a клон, and +// rawPath on the fetched path is `.json`, so every branch office would be refused and the crawl +// would abort on the first one. +const EGN_SHAPE = /(? `${r.eik}.json`), + ); + for (const name of names) { + if (!name.endsWith('.json') || known.has(name)) continue; + // The same ENOENT tolerance the expired loop above has, and for a sharper reason here: this loop + // runs AFTER the DB DELETE has committed, so an unguarded throw half-purges — rows gone, files + // still on disk — and reports the whole run as failed. The listing is a snapshot, so a name can + // legitimately be gone by the time we reach it (a concurrent purge, an operator clearing scratch). + // Already gone is the goal state. Anything else still surfaces: a purge that cannot delete has + // left third-party names on disk, and reporting success would be the failure it exists to prevent. + try { + unlink(path.join(rawDir, name)); + orphans++; + } catch (e) { + if (e.code !== 'ENOENT') throw e; + } + } + return { rows: expired.length, files, orphans }; +} diff --git a/scripts/tr/cache.test.mjs b/scripts/tr/cache.test.mjs new file mode 100644 index 000000000..a7357ec88 --- /dev/null +++ b/scripts/tr/cache.test.mjs @@ -0,0 +1,293 @@ +// node:test — the deed cache. Its job is to make the crawl resumable and to hold the PII rail. +// +// The rail (ADR-0033 decision 5): the INDEX stores no name at all — only ЕИК, dates, codes, verdicts +// and a body hash. Names exist solely in the raw JSON under git-ignored scratch/, are read only to +// produce a boolean, and never enter a public table, a response or a log. The ten-digit refusal below +// is the ЕГН guard, and it is sound precisely because an ЕИК is 9 or 13 digits — never 10. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + openCache, + upsertDeed, + markOutsideTr, + pendingEiks, + readDeed, + coverage, + purgeExpired, + RETENTION_DAYS, +} from './cache.mjs'; + +function tmpDb() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-cache-')); + return { dir, file: path.join(dir, 'tr-cache.sqlite') }; +} +const withCache = (fn) => { + const { dir, file } = tmpDb(); + const db = openCache(file); + try { + return fn(db); + } finally { + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } +}; + +const deed = (over = {}) => ({ + eik: '115536179', + httpStatus: 200, + fetchedAt: '2026-08-05T10:00:00Z', + rawPath: 'deeds/115536179.json', + bodySha256: 'a'.repeat(64), + legalFormCode: 4, + legalFormVerdict: 'closely_held', + seatNormalized: 'ПЛОВДИВ', + seatEntryDate: '2014-01-23', + latestOwnEntryDate: '2013-07-16', + ...over, +}); + +test('openCache is idempotent — re-opening an existing cache preserves rows', () => { + const { dir, file } = tmpDb(); + let db = openCache(file); + upsertDeed(db, deed()); + db.close(); + db = openCache(file); // must not wipe + assert.equal(readDeed(db, '115536179')?.eik, '115536179'); + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); +}); + +test('upsertDeed replaces on re-fetch rather than duplicating', () => + withCache((db) => { + upsertDeed(db, deed()); + upsertDeed(db, deed({ seatNormalized: 'СОФИЯ', fetchedAt: '2026-09-01T10:00:00Z' })); + assert.equal(coverage(db, ['115536179']).fetched, 1); + assert.equal(readDeed(db, '115536179').seatNormalized, 'СОФИЯ'); + })); + +test('pendingEiks returns only what is not yet cached — this is what makes a run resumable', () => + withCache((db) => { + upsertDeed(db, deed({ eik: '115536179' })); + markOutsideTr(db, '204556676', 'BULSTAT association'); + const want = ['115536179', '204556676', '201122335', '203445566']; + assert.deepEqual(pendingEiks(db, want).sort(), ['201122335', '203445566']); + })); + +test('a stale deed becomes pending again past the TTL, a fresh one does not', () => + withCache((db) => { + upsertDeed(db, deed({ fetchedAt: '2026-01-01T00:00:00Z' })); // long past + upsertDeed(db, deed({ eik: '201122335', fetchedAt: '2026-08-05T00:00:00Z' })); + const now = new Date('2026-08-05T12:00:00Z'); + assert.deepEqual(pendingEiks(db, ['115536179', '201122335'], { maxAgeDays: 35, now }), [ + '115536179', + ]); + })); + +test('coverage reports the fraction cached — the input to the fail-closed load gate', () => + withCache((db) => { + upsertDeed(db, deed({ eik: '115536179' })); + upsertDeed(db, deed({ eik: '201122335' })); + markOutsideTr(db, '204556676', 'ДЗЗД'); + const c = coverage(db, ['115536179', '201122335', '204556676', '203445566']); + assert.equal(c.wanted, 4); + assert.equal(c.fetched, 2); + assert.equal(c.outsideTr, 1); + assert.equal(c.missing, 1); + // „outside ТР" is a RESOLVED outcome, not a gap: it is known and permanent, so it counts as covered. + assert.equal(c.covered, 3); + })); + +// ── the PII rail ────────────────────────────────────────────────────────────── +test('the index REFUSES a value carrying a ten-digit run (the ЕГН guard)', () => + withCache((db) => { + // Sound because an ЕИК is 9 or 13 digits, never 10 — so this can never reject a legitimate code. + assert.throws( + () => upsertDeed(db, deed({ seatNormalized: 'СОФИЯ 8001014567' })), + /ten-digit|ЕГН/i, + ); + assert.throws(() => markOutsideTr(db, '204556676', 'подадено от 8001014567'), /ten-digit|ЕГН/i); + })); + +test('valid 9- and 13-digit codes are NOT caught by the ЕГН guard', () => + withCache((db) => { + assert.doesNotThrow(() => upsertDeed(db, deed({ outsideReason: null }))); + assert.doesNotThrow(() => markOutsideTr(db, '1155361790001', 'клон')); + })); + +// A 13-digit ЕИК (клон/подразделение) CONTAINS ten-digit substrings, so an unanchored /\d{10}/ +// rejects it — and rawPath on the fetched path is `.json`, derived from that very ЕИК. This is +// the exact shape fetch-deeds.mjs writes (path.relative(rawDir, deedPath(eik))), and upsertDeed sits +// past its JSON.parse/assertUicEcho try-catch, so a throw here aborts the whole crawl on the first +// branch office that returns a deed. The guard's own stated soundness ("an ЕИК is 9 or 13 digits, +// never 10") only holds if the run is matched as a WHOLE, which is why the pattern is anchored. +test('a 13-digit ЕИК does not trip the ЕГН guard through its own derived rawPath', () => + withCache((db) => { + assert.doesNotThrow(() => + upsertDeed(db, deed({ eik: '1155361790001', rawPath: '1155361790001.json' })), + ); + assert.equal(readDeed(db, '1155361790001').eik, '1155361790001'); + })); + +// The rail must not be a hand-maintained allowlist of four field names: upsertDeed binds thirteen +// values, and the next one added would bypass the check silently. Every bound value is screened. +test('the ЕГН guard screens EVERY bound value, not a hand-picked subset', () => + withCache((db) => { + for (const field of ['seatEntryDate', 'latestOwnEntryDate', 'fetchedAt']) + assert.throws(() => upsertDeed(db, deed({ [field]: '8001014567' })), /ten-digit|ЕГН/i, field); + })); + +// ...with one exemption, and it is measured rather than assumed: a sha256 hex digest is 64 chars of +// [0-9a-f], so a standalone ten-digit run occurs in ~7% of hashes (18% unanchored). Screening it +// would refuse roughly one deed in fourteen for no privacy gain — a hash is not an ЕГН, and it is a +// hash precisely so that no deed content reaches the index. +test('the body hash is exempt from the ЕГН guard — a digit run there is arithmetic, not an ЕГН', () => + withCache((db) => { + assert.doesNotThrow(() => upsertDeed(db, deed({ bodySha256: `8001014567${'a'.repeat(54)}` }))); + })); + +test('the schema exposes no column that could hold a person name', () => + withCache((db) => { + const cols = db + .prepare(`SELECT name FROM pragma_table_info('deeds')`) + .all() + .map((r) => r.name); + for (const forbidden of ['name', 'person', 'owner', 'manager', 'holder', 'full_name']) + assert.ok(!cols.includes(forbidden), `deeds.${forbidden} must not exist (PII rail)`); + // A hash, never an excerpt — an excerpt of a deed is third-party personal data. + assert.ok(cols.includes('body_sha256')); + })); + +test('markOutsideTr is permanent-by-intent and records WHY', () => + withCache((db) => { + markOutsideTr(db, '204556676', 'ДЗЗД — BULSTAT, not TR'); + const row = readDeed(db, '204556676'); + assert.equal(row.status, 'outside_tr'); + assert.match(row.outsideReason, /ДЗЗД/); + })); + +// ── retention (ADR-0033 decision 5) ─────────────────────────────────────────── +// The ADR promises „a 35-day retention and a purge step in the same job". Freshness (pendingEiks' +// maxAgeDays) only makes a row pending again — it re-REQUESTS, it never deletes. Only this purge +// removes the third-party names in the raw deed, so these tests are the difference between a stated +// TTL and an enforced one. +function withRaw(fn) { + const { dir, file } = tmpDb(); + const rawDir = path.join(dir, 'deeds'); + fs.mkdirSync(rawDir, { recursive: true }); + const db = openCache(file); + try { + return fn(db, rawDir); + } finally { + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } +} +const writeRaw = (rawDir, eik) => + fs.writeFileSync(path.join(rawDir, `${eik}.json`), '{"names":"third-party PII"}'); + +test('purgeExpired deletes the raw deed AND its row past the retention window', () => + withRaw((db, rawDir) => { + const now = new Date('2026-08-05T00:00:00Z'); + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-05-01T00:00:00Z' })); // ~96 days old + upsertDeed(db, deed({ eik: '201122335', fetchedAt: '2026-08-01T00:00:00Z' })); // 4 days old + writeRaw(rawDir, '115536179'); + writeRaw(rawDir, '201122335'); + + const res = purgeExpired(db, rawDir, { retentionDays: 35, now }); + assert.equal(res.rows, 1); + assert.equal(res.files, 1); + assert.equal(fs.existsSync(path.join(rawDir, '115536179.json')), false, 'PII must be gone'); + assert.equal(readDeed(db, '115536179'), null); + // The in-window deed is untouched — a purge that also evicted live cache would force a re-crawl, + // which is the one thing the pacing exists to avoid. + assert.equal(fs.existsSync(path.join(rawDir, '201122335.json')), true); + assert.ok(readDeed(db, '201122335')); + })); + +test('purgeExpired removes orphaned raw deeds — unreachable data is pure retained PII', () => + withRaw((db, rawDir) => { + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-08-01T00:00:00Z' })); + writeRaw(rawDir, '115536179'); + writeRaw(rawDir, '204556676'); // no index row: no read path can ever reach it + const res = purgeExpired(db, rawDir, { now: new Date('2026-08-05T00:00:00Z') }); + assert.equal(res.orphans, 1); + assert.equal(fs.existsSync(path.join(rawDir, '204556676.json')), false); + assert.equal(fs.existsSync(path.join(rawDir, '115536179.json')), true); + })); + +test('purgeExpired defaults to the 35-day window and tolerates an already-missing file', () => + withRaw((db, rawDir) => { + assert.equal(RETENTION_DAYS, 35); + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-06-25T00:00:00Z' })); // 41 days + upsertDeed(db, deed({ eik: '201122335', fetchedAt: '2026-07-15T00:00:00Z' })); // 21 days + // No raw file on disk for the expired row: already gone is the goal state, not an error. + const res = purgeExpired(db, rawDir, { now: new Date('2026-08-05T00:00:00Z') }); + assert.equal(res.rows, 1); + assert.equal(res.files, 0); + assert.ok(readDeed(db, '201122335'), 'the 21-day-old deed is inside the window'); + })); + +test('purgeExpired tolerates an orphan that vanished under it, and keeps sweeping', () => { + // The benign race: readdirSync lists a name, and it is gone by the time unlink runs (a concurrent + // purge, an operator clearing scratch/). The expired loop right above has always tolerated this; + // the orphan loop did not, and it throws AFTER the DB DELETE has committed — so the run half-purges, + // reports "purge failed", and leaves the operator unable to tell "already gone" from "an orphan + // still holding third-party names". Injected rather than staged, because the race cannot be timed + // from a test; the injection seam matches the one httpGet/sleep/now already use in this codebase. + return withRaw((db, rawDir) => { + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-08-01T00:00:00Z' })); + writeRaw(rawDir, '115536179'); + writeRaw(rawDir, '204556676'); // orphan 1 — disappears under us + writeRaw(rawDir, '831391124'); // orphan 2 — must still be swept + + const seen = []; + const res = purgeExpired(db, rawDir, { + now: new Date('2026-08-05T00:00:00Z'), + unlink: (p) => { + seen.push(path.basename(p)); + if (p.endsWith('204556676.json')) { + const err = new Error('ENOENT: no such file or directory'); + err.code = 'ENOENT'; + throw err; + } + fs.unlinkSync(p); + }, + }); + + assert.deepEqual(seen.sort(), ['204556676.json', '831391124.json'], 'both orphans attempted'); + assert.equal( + res.orphans, + 1, + 'a file that was already gone was not deleted BY US — do not count it', + ); + assert.equal(fs.existsSync(path.join(rawDir, '831391124.json')), false, 'the sweep continued'); + assert.equal( + fs.existsSync(path.join(rawDir, '115536179.json')), + true, + 'the live deed is untouched', + ); + }); +}); + +test('purgeExpired still refuses loudly on an orphan it could not delete for a REAL reason', () => { + // The other half, and the reason the guard is ENOENT-only. A permission error means retained PII is + // still on disk; reporting success would be the failure mode the purge exists to prevent. + return withRaw((db, rawDir) => { + writeRaw(rawDir, '204556676'); + assert.throws( + () => + purgeExpired(db, rawDir, { + now: new Date('2026-08-05T00:00:00Z'), + unlink: () => { + const err = new Error('EACCES: permission denied'); + err.code = 'EACCES'; + throw err; + }, + }), + /EACCES/, + ); + }); +}); diff --git a/scripts/tr/client.mjs b/scripts/tr/client.mjs new file mode 100644 index 000000000..473f5e309 --- /dev/null +++ b/scripts/tr/client.mjs @@ -0,0 +1,171 @@ +// HTTP client for the Търговски регистър public API (issue #279, ADR-0033). +// +// TLS: PLAIN HTTPS, verified against the system roots at Node's secure default. Do NOT reach for +// scripts/cacbg/tls.mjs here. That module exists because register.cacbg.bg serves an incomplete chain, +// so we pin ITS leaf key (ADR-0011) — `getPinned` hard-refuses every other host by design. This host +// chains correctly, so ordinary verification is available and is strictly stronger than a pin we would +// have to hand-maintain. Copying the pinning across "for consistency" would weaken this leg. +// +// Rate limiting: the register throttles, and when it does the block is SUSTAINED. An earlier spike saw +// HTTP 429 at roughly 50 cumulative requests ending in a burst, and thereafter 429 to every subsequent +// request — including simple /Deeds/{eik} calls that had worked seconds before. There is no +// `Retry-After` and no `X-RateLimit-*` header, so a client cannot pace against a published budget; it +// can only avoid tripping one. That makes a 429 an instruction to STOP, not a transient to retry +// through: `politeTrGet` retries 5xx and network faults with growing backoff, and never retries a 429. +// (#279 §3's "5 retries with growing backoff" and the decision that a 429 ends the run are consistent +// only if the retry set excludes 429.) The limiter is the operator's only way to express a rate +// preference, and tuning around it empirically is what spec §3.3's "NEVER bulk-scrape" forbids. + +import https from 'node:https'; +import { safeEik } from './paths.mjs'; + +export const TR_HOST = 'portal.registryagency.bg'; + +// Identify the crawler honestly. The operator's only lever on us is the rate limiter, so at minimum +// they should be able to see who is calling and where to complain. +export const TR_USER_AGENT = + 'sigma-bot/1.0 (+https://github.com/midt-bg/sigma) contact: via repo issues'; + +/** Thrown on HTTP 429. Distinguishable so the crawler can end the run instead of marking anything. */ +export class RateLimitError extends Error { + constructor(url) { + super(`REFUSE TO CONTINUE: ${TR_HOST} returned 429 for ${url} — the block is sustained; stop`); + this.name = 'RateLimitError'; + this.url = url; + } +} + +/** Refuse any URL that is not an https request to the register. Call before touching the network. */ +export function assertTrHost(url) { + let u; + try { + u = new URL(String(url)); + } catch { + throw new Error(`bad URL: ${url}`); + } + if (u.protocol !== 'https:' || u.hostname !== TR_HOST) { + throw new Error(`refusing non-${TR_HOST} host: ${url}`); + } + return url; +} + +/** The public deed endpoint for one ЕИК. Public JSON, no authentication. */ +export function deedUrl(eik) { + return `https://${TR_HOST}/CR/api/Deeds/${safeEik(eik)}`; +} + +const wait = (ms) => new Promise((r) => setTimeout(r, ms)); + +/** + * Default transport: `node:https`, system roots, secure defaults. + * + * NOT `fetch`. Measured against the live endpoint on 2026-08-05, the identical request returns + * **HTTP 500 with an empty body via undici's fetch and HTTP 200 with the full 34,398-byte deed via + * node:https** — same URL, same accept header, same user-agent. The server rejects something undici + * adds to the wire (encoding negotiation / connection handling); it is not an auth or rate-limit + * problem, and retrying only multiplies the failure. `scripts/cacbg/tls.mjs` is on node:https too, so + * both crawl legs now share one transport primitive. + * + * `rejectUnauthorized` is left at its secure default and named here on purpose: the sibling CACBG + * module deliberately does NOT verify against system roots (it pins a leaf instead, ADR-0011), so a + * reader comparing the two needs to see that this leg takes the ordinary, stronger path. + */ +/** + * Ceiling on a single response body. A measured deed is ~34 KB, so 8 MB is ~240× the real thing — + * this bounds abuse, not the register, and can never refuse a large-but-legitimate company. + */ +export const MAX_BODY_BYTES = 8 * 1024 * 1024; + +/** + * Read one response into a Buffer, refusing past `maxBytes`. + * + * The timeout on the request bounds how long a response may STALL; nothing bounded how much it may + * SEND. Without a counter the crawler buffers whatever arrives, so a wedged or hostile endpoint decides + * how much memory this process holds. It also composes badly with the parser: the erasure regex in + * deed.mjs backtracks quadratically on unclosed markup (measured 34K→3.7ms, 68K→13.5ms, 136K→53.2ms, + * 272K→215.4ms — ×4 per doubling), and that path is reachable only through a body large enough to make + * it matter. One byte counter bounds both, and destroying the request is the load-bearing half: merely + * rejecting the promise would leave the socket draining the rest of the response. + * + * Separated from `httpsGet` so it is testable without TLS or a live socket — `res` needs only to be an + * emitter of 'data'/'end'/'error', which is the same injection posture as politeTrGet's `httpGet`. + */ +export function collectBody(res, { req, maxBytes = MAX_BODY_BYTES } = {}) { + return new Promise((resolve, reject) => { + const chunks = []; + let size = 0; + let done = false; + const fail = (err) => { + if (done) return; + done = true; + req?.destroy(err); // tear the socket down; do not sit and drain what we already refused + reject(err); + }; + res.on('data', (c) => { + if (done) return; + size += c.length; + if (size > maxBytes) { + fail(new Error(`response too large: over ${maxBytes} bytes — refusing to buffer it`)); + return; + } + chunks.push(c); + }); + res.on('end', () => { + if (done) return; + done = true; + resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) }); + }); + res.on('error', fail); + }); +} + +export function httpsGet(url, { timeoutMs = 20_000, maxBytes = MAX_BODY_BYTES } = {}) { + return new Promise((resolve, reject) => { + const req = https.get( + url, + { headers: { accept: 'application/json', 'user-agent': TR_USER_AGENT } }, + (res) => collectBody(res, { req, maxBytes }).then(resolve, reject), + ); + req.setTimeout(timeoutMs, () => req.destroy(new Error(`timeout after ${timeoutMs}ms: ${url}`))); + req.on('error', reject); + }); +} + +/** + * GET with polite retries. 5xx and network faults retry with growing backoff; 2xx/4xx return as-is; + * **429 throws `RateLimitError` immediately and is never retried**. + * @param {string} url + * @param {{httpGet?:Function, sleep?:Function, tries?:number, backoffMs?:number}} [opts] + * `httpGet` and `sleep` are the injection seam — the whole retry policy is testable offline. + */ +export async function politeTrGet(url, opts = {}) { + const { httpGet = httpsGet, sleep = wait, tries = 5, backoffMs = 1000 } = opts; + assertTrHost(url); // before any request — a foreign host must cost zero packets + let backoff = backoffMs; + for (let attempt = 1; ; attempt++) { + let res; + try { + res = await httpGet(url); + } catch (err) { + if (attempt >= tries) throw err; + await sleep(backoff); + backoff *= 2; + continue; + } + // Checked before the retry branch: a 429 arriving mid-retry must abort the whole call, not be + // folded into the 5xx budget and not let a later 200 mask the block. + if (res.status === 429) throw new RateLimitError(url); + if (res.status >= 500) { + if (attempt >= tries) return res; // give the caller the last response to record + await sleep(backoff); + backoff *= 2; + continue; + } + return res; + } +} + +/** One deed by ЕИК. Thin wrapper so callers never build the URL themselves. */ +export function trGet(eik, opts) { + return politeTrGet(deedUrl(eik), opts); +} diff --git a/scripts/tr/client.test.mjs b/scripts/tr/client.test.mjs new file mode 100644 index 000000000..201426027 --- /dev/null +++ b/scripts/tr/client.test.mjs @@ -0,0 +1,197 @@ +// node:test — the Trade Register HTTP client. Offline: every test drives an injected getter. +// +// The load-bearing behaviour here is what the client REFUSES to do. The register rate-limits, and +// when it does the block is sustained (an earlier spike saw HTTP 429 at ~50 cumulative requests, then +// 429 to every subsequent call including ones that had just worked — no Retry-After, no quota header). +// So a 429 is an instruction to stop, not a transient to retry through. ADR-0033 decision 7. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { + TR_HOST, + TR_USER_AGENT, + deedUrl, + RateLimitError, + politeTrGet, + assertTrHost, + httpsGet, + collectBody, + MAX_BODY_BYTES, +} from './client.mjs'; + +const okRes = (body = '{}') => ({ status: 200, headers: {}, body: Buffer.from(body) }); + +test('deedUrl builds the documented endpoint and refuses a non-ЕИК', () => { + assert.equal(deedUrl('115536179'), `https://${TR_HOST}/CR/api/Deeds/115536179`); + assert.equal(deedUrl('000696327'), `https://${TR_HOST}/CR/api/Deeds/000696327`); + for (const bad of ['', null, '../../etc', '115536179?x=1', '11553617x', 'ЕИК 115536179']) + assert.throws(() => deedUrl(bad), /unsafe|ЕИК/i, String(bad)); +}); + +test('assertTrHost refuses every host but the register', () => { + assert.doesNotThrow(() => assertTrHost(`https://${TR_HOST}/CR/api/Deeds/115536179`)); + for (const bad of [ + 'https://evil.example/CR/api/Deeds/115536179', + `http://${TR_HOST}/CR/api/Deeds/115536179`, // plaintext — never + `https://${TR_HOST}.evil.example/x`, // suffix trick + `https://register.cacbg.bg/x`, // the OTHER register: different host, different TLS posture + ]) { + assert.throws(() => assertTrHost(bad), /host/i, bad); + } +}); + +test('a 429 throws RateLimitError and is NEVER retried', async () => { + let calls = 0; + const httpGet = async () => { + calls++; + return { status: 429, headers: {}, body: Buffer.from('') }; + }; + await assert.rejects( + () => politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {}, tries: 5 }), + RateLimitError, + ); + // The whole point: „5 retries with growing backoff" and „429 stops the run" are only consistent + // if the retry set EXCLUDES 429. One call, not five. + assert.equal(calls, 1, 'a 429 must not be retried'); +}); + +test('5xx IS retried with growing backoff, up to the try budget', async () => { + let calls = 0; + const waits = []; + const httpGet = async () => { + calls++; + return { status: 503, headers: {}, body: Buffer.from('') }; + }; + const res = await politeTrGet(deedUrl('115536179'), { + httpGet, + sleep: async (ms) => void waits.push(ms), + tries: 4, + }); + assert.equal(calls, 4); + assert.equal(res.status, 503, 'the last response is returned, not thrown'); + assert.deepEqual( + waits.map((w, i) => (i === 0 ? true : w > waits[i - 1])), + [true, true, true], + `backoff must grow: ${waits.join(',')}`, + ); +}); + +test('a network throw is retried, then rethrown when the budget is spent', async () => { + let calls = 0; + const httpGet = async () => { + calls++; + throw new Error('ECONNRESET'); + }; + await assert.rejects( + () => politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {}, tries: 3 }), + /ECONNRESET/, + ); + assert.equal(calls, 3); +}); + +test('a 429 arriving mid-retry aborts immediately instead of finishing the budget', async () => { + const seq = [503, 503, 429, 200]; + let calls = 0; + const httpGet = async () => { + const status = seq[calls++]; + return { status, headers: {}, body: Buffer.from('') }; + }; + await assert.rejects( + () => politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {}, tries: 5 }), + RateLimitError, + ); + assert.equal(calls, 3, 'must stop AT the 429, not carry on to the 200 behind it'); +}); + +test('200 and 404 return without any retry', async () => { + for (const status of [200, 404]) { + let calls = 0; + const httpGet = async () => { + calls++; + return status === 200 ? okRes() : { status, headers: {}, body: Buffer.from('') }; + }; + const res = await politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {} }); + assert.equal(res.status, status); + assert.equal(calls, 1); + } +}); + +test('politeTrGet refuses a foreign host before making any request', async () => { + let calls = 0; + const httpGet = async () => { + calls++; + return okRes(); + }; + await assert.rejects( + () => politeTrGet('https://evil.example/CR/api/Deeds/1', { httpGet }), + /host/i, + ); + assert.equal(calls, 0, 'the refusal must happen before the network call'); +}); + +test('the crawler identifies itself and points at somewhere to complain', () => { + // The rate limiter is the operator's only lever on us. Crawling anonymously takes away their + // ability to distinguish us from an abusive client, or to ask us to stop. + assert.match(TR_USER_AGENT, /sigma/i); + assert.match(TR_USER_AGENT, /https:\/\/github\.com\//); +}); + +test('the default transport is node:https, not fetch', () => { + // Measured 2026-08-05: the identical live request returns 500/empty via undici fetch and 200 with + // the full deed via node:https. Pinning the choice so a future "modernise to fetch" tidy-up has to + // confront it rather than silently break every lookup. + assert.equal(typeof httpsGet, 'function'); + assert.match(httpsGet.toString(), /https\.get/); +}); + +test('RateLimitError carries the url and is distinguishable from a generic failure', async () => { + const httpGet = async () => ({ status: 429, headers: {}, body: Buffer.from('') }); + const err = await politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {} }).catch( + (e) => e, + ); + assert.ok(err instanceof RateLimitError); + assert.ok(err instanceof Error); + assert.match(err.message, /115536179/); +}); + +// ── response size ───────────────────────────────────────────────────────────── +// The timeout bounds how long a response may STALL; nothing bounded how large it may GROW. A deed is +// ~34 KB measured; an unbounded reader buffers whatever arrives, so a wedged or hostile endpoint could +// have the crawler hold an arbitrary amount of memory. It also composes badly with the parser: the +// erasure regex in deed.mjs backtracks quadratically on unclosed markup (measured 34K→3.7ms, +// 68K→13.5ms, 136K→53.2ms, 272K→215.4ms — ×4 per doubling), and that path is only reachable through a +// body large enough to make it matter. One byte counter bounds both. +test('a response past the cap is refused and the request destroyed, not buffered', async () => { + const res = fakeRes(200); + const destroyed = []; + const p = collectBody(res, { req: { destroy: (e) => destroyed.push(e) }, maxBytes: 1024 }); + res.emit('data', Buffer.alloc(700)); + res.emit('data', Buffer.alloc(700)); // 1400 > 1024 + await assert.rejects(p, /too large|1024/i); + assert.equal(destroyed.length, 1, 'the socket must be torn down, not left draining'); +}); + +test('a response under the cap still resolves with the WHOLE body', async () => { + const res = fakeRes(200); + const p = collectBody(res, { req: { destroy: () => {} }, maxBytes: 1024 }); + res.emit('data', Buffer.from('{"uic":')); + res.emit('data', Buffer.from('"115536179"}')); + res.emit('end'); + const out = await p; + assert.equal(out.status, 200); + assert.equal(out.body.toString(), '{"uic":"115536179"}'); +}); + +test('the cap leaves real deeds far under it — it bounds abuse, not the register', () => { + // A measured deed is ~34 KB. The cap must sit well above that or it becomes a correctness bug that + // silently refuses large-but-legitimate companies. + assert.ok(MAX_BODY_BYTES >= 1_000_000, `cap ${MAX_BODY_BYTES} is too tight for a real deed`); +}); + +// A stand-in for an http.IncomingMessage: an emitter carrying a status code. +function fakeRes(status) { + const e = new EventEmitter(); + e.statusCode = status; + e.headers = {}; + return e; +} diff --git a/scripts/tr/deed.mjs b/scripts/tr/deed.mjs new file mode 100644 index 000000000..13837d7ae --- /dev/null +++ b/scripts/tr/deed.mjs @@ -0,0 +1,314 @@ +// Pure parser for a Търговски регистър deed (issue #279, ADR-0033). No I/O, no network, no state. +// +// The envelope is clean JSON, but every field's VALUE is an HTML fragment with semi-structured +// Bulgarian inside. That inner parser is where the libel risk lives, so the order of operations below +// is an invariant with tests on it, not a suggestion: +// +// decode HTML entities +// → split into entities on record-container / hr--report +// → drop entities marked erased +// → strip tags WITHIN one entity +// → separate the name from the address/stake WITHIN that entity +// → match tokens ONLY within a single entity +// +// Any other order silently merges blocks. Field CR_F_19_L routinely holds several съдружници in one +// string; matching a declarant against the whole field lets one person's given name combine with +// another's surname, and the output is a named public claim about the wrong human being. + +const OWNERSHIP_FIELDS = ['CR_F_18_L', 'CR_F_19_L', 'CR_F_23_L']; +const MANAGER_FIELD = 'CR_F_7_L'; +export const ROLE_FIELDS = [MANAGER_FIELD, ...OWNERSHIP_FIELDS]; +export { OWNERSHIP_FIELDS, MANAGER_FIELD }; + +// ── html ────────────────────────────────────────────────────────────────────── +const ENTITIES = { + '"': '"', + ''': "'", + '&': '&', + '<': '<', + '>': '>', + ' ': ' ', +}; +// `String.fromCodePoint` THROWS RangeError above U+10FFFF (and on a surrogate half), and the input is +// whatever the register put on the wire. An unguarded throw here does not stay local: it escapes +// entityBlocks and registrySeat, past the crawl loop's refuse-and-continue block (which covers only +// JSON.parse + assertUicEcho) and out of run() — one malformed escape in one deed ends a paced crawl +// that has already spent its request budget, and does the same to load.mjs at decision time. +// Out of range is not a character and cannot be part of a name, so it decodes to nothing: the rest of +// the entity still parses, which is the difference between losing a glyph and losing the run. +const MAX_CODE_POINT = 0x10ffff; +const codePoint = (n) => + Number.isInteger(n) && n >= 0 && n <= MAX_CODE_POINT ? String.fromCodePoint(n) : ''; + +/** Decode the entity set the register actually emits, plus numeric escapes. FIRST step, always. */ +function decodeEntities(s) { + return String(s) + .replace(/&(?:quot|apos|amp|lt|gt|nbsp);/g, (m) => ENTITIES[m]) + .replace(/&#(\d+);/g, (_, n) => codePoint(Number(n))) + .replace(/&#x([0-9a-f]+);/gi, (_, n) => codePoint(parseInt(n, 16))); +} + +/** Strip tags and collapse whitespace, INSIDE one already-isolated entity. */ +function stripTags(html) { + return String(html) + .replace(//gi, ' ') + .replace(/<[^>]*>/g, '') + .replace(/\s+/g, ' ') + .trim(); +} + +// An entity is erased when the register says so. MEASURED on a live deed: the marker is +// `erasure-text-inline` (with `ui-icon-erased`) and the container carries no `field-text` paragraph at +// all. `field-text--erased` is accepted as well — it is the other spelling reported for this register, +// and honouring both costs nothing while assuming one costs a wrong publish. +const ERASED_MARKER = /erasure-text-inline|ui-icon-erased|field-text--erased/; + +// The erasure notice, stripped so „Заличено обстоятелство." never reads as content. +// +// The `{0,2000}` bound is not cosmetic. With a plain `.*?` this backtracks QUADRATICALLY on markup where +// the opening div is never closed: every opening restarts a scan to end-of-input, measured at 34K→3.3ms, +// 68K→13.6ms, 136K→53.8ms, 272K→240ms, 1M→4.0s — ×4 per doubling. Bounding the lazy run makes each start +// position scan a fixed window instead, which is linear: the same inputs measure 5.3 / 10.3 / 24.8 / 41.6 +// / 187ms. A real notice is one short sentence, so 2000 characters is ~80× the live shape. +// +// Failing to match is safe by construction: `erased` is decided independently by ERASED_MARKER above, so +// an over-long notice still marks the block erased and liveFields still drops it. The only visible effect +// is that its text survives into the block — which in strict mode raises the drift alarm, loudly, rather +// than passing anything through silently. +const ERASURE_NOTICE = + /]*class=['"][^'"]*erasure-text-inline[^'"]*['"][^>]*>.{0,2000}?<\/div>/gis; + +/** + * Split one field's htmlData into the separate registered entities it holds. + * + * @param {string} html + * @param {{strict?:boolean}} [opts] `strict` turns the erased-with-content contradiction into a throw. + * @returns {{text:string, erased:boolean}[]} + */ +export function entityBlocks(html, { strict = false } = {}) { + if (html == null || String(html).trim() === '') return []; + const decoded = decodeEntities(html); // decode BEFORE splitting — see the order above + const chunks = decoded + .split(/]*>/i) + .flatMap((part) => part.split(/(?=]*class=['"][^'"]*record-container)/i)) + .map((c) => c.trim()) + .filter((c) => c !== ''); + + const out = []; + for (const chunk of chunks) { + const erased = ERASED_MARKER.test(chunk); + // Read the visible text WITHOUT the erasure notice, so „Заличено обстоятелство." never counts as + // content and an erased block reads as empty. + const withoutNotice = chunk.replace(ERASURE_NOTICE, ' '); + const text = stripTags(withoutNotice); + if (erased && text !== '' && strict) { + throw new Error( + `REFUSE: an erased entity carries content (${JSON.stringify(text.slice(0, 60))}) — the ` + + `"erased ⇒ empty" assumption has drifted; stop rather than guess which state is in force`, + ); + } + if (text === '' && !erased) continue; // structural noise, not an entity + out.push({ text, erased }); + } + return out; +} + +/** Every field in the deed, flattened out of sections → subDeeds → groups. */ +function allFields(deed) { + const out = []; + for (const s of deed?.sections ?? []) + for (const sd of s?.subDeeds ?? []) + for (const g of sd?.groups ?? []) for (const f of g?.fields ?? []) out.push(f); + return out; +} + +const isoDay = (v) => (v ? String(v).slice(0, 10) : null); + +/** + * The LIVE entities of the requested field codes — the single entry point to live state. + * Erased entities are dropped here and nowhere else, so the rule is auditable in one place. + * @returns {{nameCode:string, entryDate:string|null, entryNumber:string|null, entities:string[]}[]} + */ +export function liveFields(deed, nameCodes, opts = {}) { + const want = new Set(nameCodes); + const out = []; + for (const f of allFields(deed)) { + if (!want.has(f.nameCode)) continue; + const entities = entityBlocks(f.htmlData, opts) + .filter((b) => !b.erased) + .map((b) => b.text); + if (entities.length === 0) continue; + out.push({ + nameCode: f.nameCode, + // TEXT, never a number: a fieldEntryNumber like 20130716101007 exceeds 2^53 once combined. + entryNumber: f.fieldEntryNumber == null ? null : String(f.fieldEntryNumber), + entryDate: isoDay(f.fieldEntryDate), + entities, + }); + } + return out; +} + +// ── names ───────────────────────────────────────────────────────────────────── +/** + * Name tokens: NFC, upper case, split on non-letters, keep tokens of length ≥2. + * + * Dropping 1-character tokens is what makes „Г. И. Петров" a ONE-token name rather than a three-token + * one — an abbreviated name can then never reach the ≥3 tokens rung 2 requires, instead of passing on + * initials that match half the register. Latin letters are kept (never folded onto Cyrillic + * look-alikes) so a homoglyph is a non-match rather than a false match, matching companyNameKey's + * posture in packages/shared/src/company-name-key.ts. + * + * A near-twin of the module-private `holderTokens` in scripts/cacbg/parse.mjs — deliberately + * re-implemented rather than imported, because that module pulls in fast-xml-parser and would tie + * these pure tests to a workspace install. Keep the two in step. + */ +export function personTokens(name) { + return String(name ?? '') + .normalize('NFC') + .toUpperCase() + .split(/[^\p{L}]+/u) + .filter((t) => [...t].length >= 2); +} + +/** + * Does EVERY token of the declarant's name appear as a whole token of this ONE entity? + * + * Full subset, not a majority: of 301 measured matches, 46 were two-token only, which is precisely + * the homonym risk. Whole-token, not substring: „ПЕТРОВ" inside „ПЕТРОВА" is a different person. + * + * Callers MUST pass a single entity's text (see entityBlocks). Passing a whole field is the + * cross-entity bug, and no signature can prevent it — the test does. + */ +export function fullSubsetMatch(declarantName, entityText) { + const want = personTokens(declarantName); + if (want.length === 0) return false; + const have = new Set(personTokens(entityText)); + return want.every((t) => have.has(t)); +} + +// ── seat ────────────────────────────────────────────────────────────────────── +// Strip a settlement-type prefix only as a WHOLE token: „гр."/„с."/„общ."/„обл."/„ж.к." followed by a +// dot and optional space. R9 — a loose prefix strip turns СОФИЯ into ОФИЯ and ГРАДЕЦ into АДЕЦ. +const SETTLEMENT_PREFIX = /^(?:ГР|С|ОБЩ|ОБЛ|Ж\.К)\.\s*/u; + +/** Normalise a settlement name for comparison. Empty in ⇒ empty out, and empty NEVER confirms. */ +export function normalizeSettlement(raw) { + let s = String(raw ?? '') + .normalize('NFC') + .toUpperCase() + .replace(/\([^)]*\)/g, ' ') // „(столица)" + .trim(); + s = s.split(/[,/]/)[0].trim(); // cut at the first comma or slash — „с. Марково, п.к. 4108" + s = s.replace(SETTLEMENT_PREFIX, ''); + return s + .replace(/[^\p{L}\s-]/gu, '') + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * The company's registered settlement, from CR_F_5_L's „Населено място:" segment ONLY. + * + * ADR-0010 item 3 (addresses are never extracted) is honoured here, and the field makes that a live + * concern rather than a formality: CR_F_5_L also carries street, postcode, phone, fax, e-mail and + * website. Nothing but the settlement and the entry date leaves this function. + * @returns {{settlement:string, entryDate:string|null}} + */ +export function registrySeat(deed) { + for (const f of liveFields(deed, ['CR_F_5_L'])) { + for (const text of f.entities) { + const m = text.match(/Населено място:\s*([^,]+)/u); + if (m) return { settlement: normalizeSettlement(m[1]), entryDate: f.entryDate }; + } + } + return { settlement: '', entryDate: null }; +} + +// ── legal form ──────────────────────────────────────────────────────────────── +// Codes observed empirically (#279 §3 + the spike's catalogue read). DELIBERATELY incomplete: the +// nomenclature endpoint does not settle the enum (legalForm=4 and =10 return a byte-identical +// catalogue), so anything absent here is `unknown` and WITHHOLDS. In particular whether ЕАД carries +// its own code is unresolved — ООД and ЕООД turned out NOT to share one (4 vs 10), so assuming ЕАД +// shares 5 with АД would be exactly the fail-open this bar exists to prevent. +const FORM_CODES = new Map([ + [1, 'closely_held'], // ЕТ + [4, 'closely_held'], // ООД + [5, 'joint_stock'], // АД + [6, 'joint_stock'], // КДА + [10, 'closely_held'], // ЕООД +]); + +// The фирма's legal form is its SUFFIX under ЗТРРЮЛНЦ, and the deed envelope's `fullName` carries it +// („ПИМК" ООД) while CR_F_2_L is bare („ПИМК"). So the bar has a second, independent signal at zero +// extra cost. +// +// DELIBERATELY a twin of classify.mjs's JOINT_STOCK rather than an import, for the same reason +// personTokens twins parse.mjs's holderTokens: the dependency direction here is cacbg → tr (load.mjs +// imports this module), so importing back out of scripts/cacbg/ would close a cycle across the two +// directories. The two are pinned byte-identical by a test in deed.test.mjs — this comment used to +// claim КДА was missing from classify.mjs, which stopped being true in 5f64f5c, and an unenforced +// „keep these in step" note is exactly how that happens. +export const JOINT_SUFFIX = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; +const CLOSELY_SUFFIX = /(?:^|[\s"„“”«»])(ООД|ЕООД|ЕТ|ДЗЗД|КД|СД|КООПЕРАЦИЯ)[\s"„“”«»]*$/u; + +/** + * Legal-form verdict — a union of the numeric code and the mandated фирма suffix. + * Either signal saying joint-stock bars the link. Neither able to say ⇒ `unknown`, which withholds. + * @returns {{code:number|null, codeVerdict:string, suffixVerdict:string, verdict:string}} + */ +export function registryLegalForm(deed) { + const code = typeof deed?.legalForm === 'number' ? deed.legalForm : null; + const codeVerdict = (code != null && FORM_CODES.get(code)) || 'unknown'; + + const name = String(deed?.fullName ?? '') + .normalize('NFC') + .toUpperCase() + .trim(); + const suffixVerdict = JOINT_SUFFIX.test(name) + ? 'joint_stock' + : CLOSELY_SUFFIX.test(name) + ? 'closely_held' + : 'unknown'; + + const verdict = + codeVerdict === 'joint_stock' || suffixVerdict === 'joint_stock' + ? 'joint_stock' + : codeVerdict === 'closely_held' || suffixVerdict === 'closely_held' + ? 'closely_held' + : 'unknown'; + return { code, codeVerdict, suffixVerdict, verdict }; +} + +// ── refutation input ────────────────────────────────────────────────────────── +/** + * Latest entry date across the LIVE ownership fields, or null when none survives. + * + * The trap this avoids was present in the first company sampled: CR_F_23_L sits in the CURRENT deed + * dated 2013-07-16 carrying only „Заличено обстоятелство.". Counted naively it becomes the latest + * ownership entry and can refute a link it says nothing about. liveFields drops it. + */ +export function latestOwnershipEntryDate(deed) { + const dates = liveFields(deed, OWNERSHIP_FIELDS) + .map((f) => f.entryDate) + .filter(Boolean); + return dates.length ? dates.sort().at(-1) : null; +} + +/** + * The deed we got back must be the deed we asked for. + * + * R8: Bulgarian public bodies carry ЕИК of exactly the `000…` shape, so any numeric round-trip on the + * path silently rewrites the identifier. Without this rail, that failure publishes a claim about a + * different company under the right official's name. + */ +export function assertUicEcho(deed, requestedEik) { + const got = deed?.uic == null ? null : String(deed.uic); + if (got !== String(requestedEik)) { + throw new Error( + `REFUSE: deed uic echo mismatch — requested ${JSON.stringify(String(requestedEik))}, ` + + `deed reports ${JSON.stringify(got)}`, + ); + } + return deed; +} diff --git a/scripts/tr/deed.test.mjs b/scripts/tr/deed.test.mjs new file mode 100644 index 000000000..9c539c874 --- /dev/null +++ b/scripts/tr/deed.test.mjs @@ -0,0 +1,391 @@ +// node:test — the deed parser. Pure, offline, and the most dangerous code in this change. +// +// Every fixture below is REAL markup, copied verbatim from a live deed (ЕИК 115536179, fetched +// 2026-08-05) with person names replaced only where a test needs a specific shape. Writing this +// parser against imagined markup is how the entity-boundary bug ships. +// +// The failure this file exists to prevent: field CR_F_19_L holds THREE separate people in one string, +// separated by
. Matching a declarant's tokens against the whole field lets +// the given name of one person combine with the surname of another, and the result is a named public +// claim that a specific official owns a specific company — about the wrong person. ADR-0033 decision 2. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + entityBlocks, + liveFields, + personTokens, + fullSubsetMatch, + normalizeSettlement, + registrySeat, + registryLegalForm, + latestOwnershipEntryDate, + assertUicEcho, + JOINT_SUFFIX, +} from './deed.mjs'; + +// ── real markup ─────────────────────────────────────────────────────────────── +const F19_THREE = `

ПИМК ХОЛДИНГ ГРУП АД, ЕИК/ПИК 202294392, Държава: БЪЛГАРИЯ, Размер на дяловото участие: 59980.00 лв.


ПЕНКО НЕСТОРОВ НЕСТОРОВ, Държава: БЪЛГАРИЯ, Размер на дяловото участие: 10.00 лв.


ИЛИЯН КОСТАДИНОВ ФИЛИПОВ, Държава: БЪЛГАРИЯ, Размер на дяловото участие: 10.00 лв.

`; + +const F23_ERASED = `
Заличено обстоятелство.
`; + +const F5_SEAT = `

Държава: БЪЛГАРИЯ
Област: Пловдив, Община: Родопи
Населено място: с. Марково, п.к. 4108
бул./ул. местност ЗАХАРИДЕВО № 043А Телефон: 032/901102 и 032/945149
Адрес на електронна поща: office@pimk-bg.eu

`; + +const F7_MANAGER = `

АНТОН ИЦКОВ ЙОРДАНОВ, Държава: БЪЛГАРИЯ

`; + +const field = (nameCode, htmlData, over = {}) => ({ + nameCode, + htmlData, + fieldEntryNumber: '20130716101007', + fieldEntryDate: '2013-07-16T10:10:07', + fieldOperation: 3, + fieldIdent: '00190', + ...over, +}); +const deedOf = (fields, over = {}) => ({ + uic: '115536179', + fullName: '"ПИМК" ООД', + legalForm: 4, + sections: [{ subDeeds: [{ groups: [{ fields }] }] }], + ...over, +}); + +// ── T1 — the entity boundary ────────────────────────────────────────────────── +test('entityBlocks splits one field into its separate registered entities', () => { + const blocks = entityBlocks(F19_THREE); + assert.equal(blocks.length, 3, 'three съдружници, three blocks'); + assert.match(blocks[0].text, /ПИМК ХОЛДИНГ ГРУП АД/); + assert.match(blocks[1].text, /ПЕНКО НЕСТОРОВ НЕСТОРОВ/); + assert.match(blocks[2].text, /ИЛИЯН КОСТАДИНОВ ФИЛИПОВ/); + // No block may carry another block's name — that is the whole point. + assert.ok(!blocks[1].text.includes('ФИЛИПОВ')); + assert.ok(!blocks[2].text.includes('ПЕНКО')); +}); + +test('T1 — tokens from two DIFFERENT people never combine into a match', () => { + // „ПЕНКО … НЕСТОРОВ" and „ИЛИЯН КОСТАДИНОВ ФИЛИПОВ" are both in field 19. A declarant assembled + // from one person's given name and another's patronymic+surname must NOT match. + const frankenstein = 'ПЕНКО КОСТАДИНОВ ФИЛИПОВ'; + const blocks = entityBlocks(F19_THREE); + assert.equal( + blocks.some((b) => fullSubsetMatch(frankenstein, b.text)), + false, + 'cross-entity match — this is the libel bug', + ); + // Whole-field matching is exactly what must not be done; prove the naive approach WOULD have fired, + // so this test cannot silently pass because the matcher is broken in some other way. + assert.equal(fullSubsetMatch(frankenstein, F19_THREE), true, 'naive whole-field match fires'); +}); + +// The live register emits single-quoted attributes today, and every fixture above is verbatim from a +// real deed. But the entity split is the ONLY thing standing between three people and one merged +// token pool, and it must not be quiet about a markup change: if the register ever switches to +// class="…", a quote-specific pattern stops splitting, all three owners collapse into one block, and +// the frankenstein match above starts firing — a named public claim about a person who is not there. +// R7's doctrine for this module is „refuse loudly, never guess"; silently mis-splitting is neither. +const dq = (s) => s.replace(/class='([^']*)'/g, 'class="$1"'); + +test('T1 — the entity split survives double-quoted attributes (markup-drift hardening)', () => { + // Strip the
separators first. entityBlocks splits on BOTH
and record-container precisely + // because either may be absent; with the
present this test would pass on the
rule alone + // and prove nothing about the quote handling it is here to pin. + const blocks = entityBlocks(dq(F19_THREE).replace(/]*>/gi, '')); + assert.equal(blocks.length, 3, 'a quote style change must not merge three owners into one block'); + assert.equal( + blocks.some((b) => fullSubsetMatch('ПЕНКО КОСТАДИНОВ ФИЛИПОВ', b.text)), + false, + 'cross-entity match under double quotes — the libel bug via markup drift', + ); + assert.ok(blocks.some((b) => fullSubsetMatch('ПЕНКО НЕСТОРОВ НЕСТОРОВ', b.text))); +}); + +test('T1 — erasure is still detected and stripped under double-quoted attributes', () => { + const [block] = entityBlocks(dq(F23_ERASED)); + assert.equal(block.erased, true); + assert.equal(block.text, '', 'the erasure notice must not survive as content'); + // The strict contradiction check must not fire merely because the quote style changed. + assert.doesNotThrow(() => entityBlocks(dq(F23_ERASED), { strict: true })); +}); + +test('T1 positive control — the CORRECT declarant does match', () => { + // Without this, a matcher that always returns false passes every negative test above (ADR-0027). + const blocks = entityBlocks(F19_THREE); + assert.equal( + blocks.some((b) => fullSubsetMatch('ИЛИЯН КОСТАДИНОВ ФИЛИПОВ', b.text)), + true, + ); + assert.equal( + blocks.some((b) => fullSubsetMatch('Пенко Несторов Несторов', b.text)), + true, + 'case/spacing drift still matches within the right entity', + ); +}); + +test('T1 — HTML entities are decoded BEFORE splitting, not after', () => { + // Decode-after-split leaves "-bearing names mangled; decode-before-split is the tested order. + const encoded = F19_THREE.replace('ПИМК ХОЛДИНГ ГРУП АД', '"ПИМК ХОЛДИНГ ГРУП" АД'); + const blocks = entityBlocks(encoded); + assert.equal(blocks.length, 3); + assert.match(blocks[0].text, /"ПИМК ХОЛДИНГ ГРУП" АД/); + assert.ok(!blocks[0].text.includes('"'), 'entities must be decoded'); +}); + +test('T1 — a corporate съдружник contributes its own tokens, not a person match', () => { + // „ПИМК ХОЛДИНГ ГРУП АД, ЕИК/ПИК 202294392" is a legal entity. A person whose name happens to + // overlap its words must not match it into existence. + const blocks = entityBlocks(F19_THREE); + assert.equal( + fullSubsetMatch('ПИМК ХОЛДИНГ ГРУП', blocks[0].text), + true, + 'literal overlap exists', + ); + // …which is why rung 2 requires ≥3 tokens of a PERSON name; the guard lives in evidence.mjs (T2). +}); + +// ── erasure ─────────────────────────────────────────────────────────────────── +test('an erased entity is detected structurally and dropped from live state', () => { + // MEASURED: the live deed marks erasure with `erasure-text-inline` and the container carries NO + //

at all. An earlier note claimed the marker was `field-text--erased`; that + // class does not occur in the sampled deed. Both are treated as erasure so either shape is safe. + const blocks = entityBlocks(F23_ERASED); + assert.equal(blocks.length, 1); + assert.equal(blocks[0].erased, true); + const live = liveFields(deedOf([field('CR_F_23_L', F23_ERASED, { fieldOperation: 2 })]), [ + 'CR_F_23_L', + ]); + assert.deepEqual(live, [], 'an erased field contributes no live entity'); +}); + +test('the OTHER erasure spelling is honoured too', () => { + const alt = `

СТАР СОБСТВЕНИК

Заличено обстоятелство.
`; + assert.equal(entityBlocks(alt)[0].erased, true); +}); + +test('an erased-looking block carrying real content REFUSES the deed (drift alarm)', () => { + // R7: "erased ⇒ empty" is an empirical observation (93 of 93). If the register ever emits an erased + // marker beside live content, the assumption is broken and we must stop, not silently drop a real + // owner or silently keep a removed one. + const contradictory = `

ЖИВО ИМЕ ТУК

Заличено обстоятелство.
`; + assert.throws(() => entityBlocks(contradictory, { strict: true }), /erased.*content|drift/i); +}); + +test('an empty htmlData yields no entities and does not throw', () => { + assert.deepEqual(entityBlocks(''), []); + assert.deepEqual(entityBlocks(null), []); +}); + +// A numeric entity is attacker-shaped input in the only sense that matters here: it comes off the +// wire, and `String.fromCodePoint` throws RangeError above U+10FFFF. That throw escapes decodeEntities, +// escapes entityBlocks and registrySeat, and — because the crawl loop's try/catch covers only +// JSON.parse + assertUicEcho — escapes run() and kills the process. One malformed entity in one deed +// would end a paced crawl that has already spent its request budget. Out of range is not a person, so +// the only defensible reading is „no character": drop it and keep parsing the rest of the entity. +test('an out-of-range numeric entity is dropped, never thrown out of the parser', () => { + const overflow = F19_THREE.replace('ПЕНКО', '�ПЕНКО'); + const blocks = entityBlocks(overflow); + assert.equal(blocks.length, 3, 'the deed still parses into its three entities'); + assert.match(blocks[1].text, /ПЕНКО НЕСТОРОВ НЕСТОРОВ/, 'the surrounding name survives intact'); + assert.ok(!blocks[1].text.includes('&#'), 'the escape itself does not survive as literal text'); +}); + +test('the hex numeric form is guarded too — both decode lines, not just the decimal one', () => { + const overflow = F19_THREE.replace('ИЛИЯН', '�ИЛИЯН'); + const blocks = entityBlocks(overflow); + assert.equal(blocks.length, 3); + assert.match(blocks[2].text, /ИЛИЯН КОСТАДИНОВ ФИЛИПОВ/); +}); + +test('an in-range numeric entity still decodes — the guard bounds, it does not disable', () => { + // П is „П". A guard that dropped every numeric escape would silently mangle real names. + const blocks = entityBlocks(F19_THREE.replace('ПЕНКО', 'ПЕНКО')); + assert.match(blocks[1].text, /ПЕНКО НЕСТОРОВ НЕСТОРОВ/); +}); + +test('registrySeat survives the same malformed entity rather than aborting the load', () => { + // registrySeat sits OUTSIDE the crawl loop's refuse-and-continue block (fetch-deeds.mjs), and + // load.mjs calls it again at decision time — so an unguarded throw here takes down both legs. + const d = deedOf([field('CR_F_5_L', F5_SEAT.replace('с. Марково', '�с. Марково'))]); + const seat = registrySeat(d); + assert.equal(seat.settlement, 'МАРКОВО'); +}); + +test('liveFields keeps only the requested codes and reports entry date/number', () => { + const d = deedOf([ + field('CR_F_7_L', F7_MANAGER, { nameCode: 'CR_F_7_L', fieldEntryDate: '2017-09-15T00:00:00' }), + field('CR_F_19_L', F19_THREE), + field('CR_F_5_L', F5_SEAT), + ]); + const live = liveFields(d, ['CR_F_7_L', 'CR_F_19_L']); + assert.deepEqual(live.map((f) => f.nameCode).sort(), ['CR_F_19_L', 'CR_F_7_L']); + const f19 = live.find((f) => f.nameCode === 'CR_F_19_L'); + assert.equal(f19.entities.length, 3); + assert.equal(f19.entryDate, '2013-07-16'); + assert.equal(f19.entryNumber, '20130716101007'); + assert.equal(typeof f19.entryNumber, 'string', 'entry numbers exceed 2^53 — never a number'); +}); + +// ── T2 — the token rule ─────────────────────────────────────────────────────── +test('personTokens keeps tokens of length ≥2 and folds case/spacing', () => { + assert.deepEqual(personTokens('Иван Петров Георгиев'), ['ИВАН', 'ПЕТРОВ', 'ГЕОРГИЕВ']); + assert.deepEqual(personTokens(' иван петров '), ['ИВАН', 'ПЕТРОВ']); + // An initial is not a token — „Г. И. Петров" is ONE token, so it can never reach three. + assert.deepEqual(personTokens('Г. И. Петров'), ['ПЕТРОВ']); + // A hyphenated surname is one token, not two. + assert.deepEqual(personTokens('Мария Иванова-Петрова'), ['МАРИЯ', 'ИВАНОВА', 'ПЕТРОВА']); +}); + +test('fullSubsetMatch requires EVERY declarant token, not a majority', () => { + const entity = 'ИЛИЯН КОСТАДИНОВ ФИЛИПОВ, Държава: БЪЛГАРИЯ'; + assert.equal(fullSubsetMatch('ИЛИЯН КОСТАДИНОВ ФИЛИПОВ', entity), true); + // 2-of-3 must fail: of 301 measured matches, 46 were two-token only — the homonym risk itself. + assert.equal(fullSubsetMatch('ИЛИЯН КОСТАДИНОВ ПЕТРОВ', entity), false); + assert.equal(fullSubsetMatch('ИЛИЯН ПЕТРОВ ФИЛИПОВ', entity), false); +}); + +test('a token must match a WHOLE token, never a substring', () => { + // „ПЕТРОВ" must not be found inside „ПЕТРОВА"; that is a different person. + assert.equal(fullSubsetMatch('ИВАН ПЕТРОВ ГЕОРГИЕВ', 'ИВАН ПЕТРОВА ГЕОРГИЕВА'), false); +}); + +// ── T5 — settlement normalization ───────────────────────────────────────────── +test('T5 — the settlement prefix is stripped only as a whole token', () => { + // R9: a naive prefix strip turns СОФИЯ into ОФИЯ and ГРАДЕЦ into АДЕЦ. + assert.equal(normalizeSettlement('гр. Русе'), 'РУСЕ'); + assert.equal(normalizeSettlement('с. Марково'), 'МАРКОВО'); + assert.equal(normalizeSettlement('София'), 'СОФИЯ'); + assert.equal(normalizeSettlement('СОФИЯ'), 'СОФИЯ'); + assert.equal(normalizeSettlement('Градец'), 'ГРАДЕЦ'); + assert.equal(normalizeSettlement('гр.Пловдив'), 'ПЛОВДИВ'); + assert.equal(normalizeSettlement('София (столица)'), 'СОФИЯ'); +}); + +test('T5 — an empty settlement never equals another empty settlement', () => { + // „both blank ⇒ confirmed" would rubber-stamp every link with no seat data at all. + assert.equal(normalizeSettlement(''), ''); + assert.equal(normalizeSettlement(null), ''); + assert.equal(normalizeSettlement(' '), ''); +}); + +test('registrySeat reads the „Населено място" segment and NOTHING else', () => { + const seat = registrySeat(deedOf([field('CR_F_5_L', F5_SEAT, { nameCode: 'CR_F_5_L' })])); + assert.equal(seat.settlement, 'МАРКОВО'); + assert.equal(seat.entryDate, '2013-07-16'); + // ADR-0010 item 3: the parser never returns an address, phone, e-mail or website — and the deed + // demonstrably carries all four. + const blob = JSON.stringify(seat); + for (const leak of ['ЗАХАРИДЕВО', '032/901102', 'office@pimk-bg.eu', 'п.к.', '4108', 'Родопи']) + assert.ok(!blob.includes(leak), `seat must not carry ${leak}`); +}); + +// ── T3 — the joint-stock bar ────────────────────────────────────────────────── +test('T3 — the legal-form verdict is a UNION of the code and the ЗТРРЮЛНЦ suffix', () => { + const jointByCode = registryLegalForm(deedOf([], { legalForm: 5, fullName: 'НЕЩО СИ' })); + assert.equal(jointByCode.verdict, 'joint_stock'); + + // An UNKNOWN code must not fall through: the suffix decides, and if it cannot, we withhold. + const unknownButEad = registryLegalForm(deedOf([], { legalForm: 99, fullName: '"ГАМА" ЕАД' })); + assert.equal(unknownButEad.verdict, 'joint_stock', 'barred by the mandated suffix'); + + const unknownButEood = registryLegalForm(deedOf([], { legalForm: 99, fullName: '"БЕТА" ЕООД' })); + assert.equal(unknownButEood.verdict, 'closely_held', 'the bar is not blanket'); + + const unknownAndUnreadable = registryLegalForm(deedOf([], { legalForm: 99, fullName: 'НЕЩО' })); + assert.equal(unknownAndUnreadable.verdict, 'unknown', 'unknown withholds — it never publishes'); +}); + +test('T3 — КДА is barred (it is not in the existing closelyHeldForm token list)', () => { + assert.equal( + registryLegalForm(deedOf([], { legalForm: 99, fullName: '"X" КДА' })).verdict, + 'joint_stock', + ); + assert.equal( + registryLegalForm(deedOf([], { legalForm: 6, fullName: 'X' })).verdict, + 'joint_stock', + ); +}); + +test('T3 — a real ООД deed reads as closely held, by code AND by suffix', () => { + const v = registryLegalForm(deedOf([])); + assert.equal(v.code, 4); + assert.equal(v.verdict, 'closely_held'); + assert.equal(v.suffixVerdict, 'closely_held', 'fullName carries the form: "ПИМК" ООД'); +}); + +// ── T7 — the UIC echo ───────────────────────────────────────────────────────── +test('T7 — a deed whose UIC does not echo the request is REFUSED', () => { + // R8: ЕИК leading zeros are significant (public bodies are exactly 000…). If anything on the path + // rewrites the identifier, this is the rail that catches it before a claim is made about the + // wrong company. + assert.doesNotThrow(() => assertUicEcho(deedOf([]), '115536179')); + assert.throws(() => assertUicEcho(deedOf([]), '000696327'), /uic|echo/i); + assert.throws(() => assertUicEcho(deedOf([], { uic: '696327' }), '000696327'), /uic|echo/i); + assert.throws(() => assertUicEcho(deedOf([], { uic: null }), '115536179'), /uic|echo/i); +}); + +// ── the refutation input ────────────────────────────────────────────────────── +test('latestOwnershipEntryDate ignores ERASED ownership fields', () => { + // The trap, present in the first company sampled: CR_F_23_L is live in the current deed, dated + // 2013-07-16, and contains only „Заличено обстоятелство.". Read naively it becomes „latest + // ownership entry: 2013-07-16" and can refute a link it knows nothing about. + const d = deedOf([ + field('CR_F_19_L', F19_THREE, { fieldEntryDate: '2011-05-02T00:00:00' }), + field('CR_F_23_L', F23_ERASED, { fieldEntryDate: '2013-07-16T10:10:07', fieldOperation: 2 }), + ]); + assert.equal(latestOwnershipEntryDate(d), '2011-05-02', 'the erased 2013 entry must not count'); +}); + +test('latestOwnershipEntryDate is null when no live ownership field survives', () => { + const d = deedOf([field('CR_F_23_L', F23_ERASED, { fieldOperation: 2 })]); + assert.equal(latestOwnershipEntryDate(d), null); +}); + +// The erasure-notice strip used an unbounded lazy `.*?`, which backtracks quadratically when the opening +// div is never closed — each opening restarts a scan to end-of-input. Measured on that shape: 34K→3.3ms, +// 68K→13.6ms, 136K→53.8ms, 272K→240ms, 1M→4.0s (×4 per doubling). The parser runs against whatever the +// register returns, so that is remote-controlled CPU on a paced crawl with a per-request budget. +test('adversarial unclosed markup parses in linear time, not quadratically', () => { + // ~1 MB of unclosed erasure openings — the exact shape that triggers the backtracking. + const doc = '
z'.repeat(32_000); + const t = process.hrtime.bigint(); + entityBlocks(doc); + const ms = Number(process.hrtime.bigint() - t) / 1e6; + // Bounded measures ~190ms here and unbounded ~4000ms, so 1500ms separates them with ~8× headroom + // over the bounded path — wide enough not to flake on a loaded runner, tight enough to catch a + // reintroduced `.*?`. + assert.ok(ms < 1500, `entityBlocks took ${ms.toFixed(0)}ms on 1MB of unclosed markup`); +}); + +test('an over-long erasure notice still marks the block erased — the bound cannot leak a live owner', () => { + // If the notice exceeds the bound the regex simply does not strip it. `erased` is decided separately + // by ERASED_MARKER, so the block is still erased and liveFields still drops it: the failure mode of + // the bound is a noisier block, never a resurrected owner. + const long = `
${'Заличено. '.repeat(400)}
`; + const [block] = entityBlocks(long); + assert.equal(block.erased, true); + assert.deepEqual( + liveFields(deedOf([field('CR_F_19_L', long)]), ['CR_F_19_L']), + [], + 'an erased block contributes no live entity regardless of its notice length', + ); +}); + +// JOINT_SUFFIX here and JOINT_STOCK in scripts/cacbg/classify.mjs are the SAME rule — which legal-form +// suffixes mark a share-issuing company — held in two places because the TR parser cannot import out of +// scripts/cacbg/ without closing a cacbg↔tr cycle. The drift this risks has already happened once: 5f64f5c +// added КДА to classify.mjs while deed.mjs's comment still asserted it was absent there. A prose „keep +// these in step" note does not keep anything in step; this does. +test('the joint-stock suffix rule is identical in the TR parser and the classifier', async () => { + const { JOINT_STOCK } = await import('../cacbg/classify.mjs'); + assert.equal(JOINT_SUFFIX.source, JOINT_STOCK.source, 'the two patterns have diverged'); + assert.equal(JOINT_SUFFIX.flags, JOINT_STOCK.flags, 'the two patterns have diverged in flags'); + // Behavioural pin as well as textual: identical sources with different behaviour is impossible, but a + // future refactor could legitimately change BOTH sources while breaking one. These are the forms the + // bar exists for — every one must be caught by both, or a joint-stock parcel publishes as ownership. + for (const name of ['ТРЕЙС ГРУП ХОЛД АД', 'НЕЩО ЕАД', 'ФОНД АДСИЦ', 'НЕЩО КДА']) { + assert.equal(JOINT_SUFFIX.test(name), true, name); + assert.equal(JOINT_STOCK.test(name), true, name); + } + for (const name of ['АЛФА СТРОЙ ООД', 'БЕТА ЕООД', 'АД ГРУП ООД']) { + assert.equal(JOINT_SUFFIX.test(name), false, name); + assert.equal(JOINT_STOCK.test(name), false, name); + } +}); diff --git a/scripts/tr/eik.mjs b/scripts/tr/eik.mjs new file mode 100644 index 000000000..5a4d5b125 --- /dev/null +++ b/scripts/tr/eik.mjs @@ -0,0 +1,61 @@ +// ЕИК validity — the Node twin of the rule that already lives in SQL. +// +// `eik_valid` in scripts/normalize-raw.sql decides which bidders get an ЕИК-keyed identity at all, so +// it defines the ЕИК space the whole matcher works in. Until now that rule was callable ONLY from +// SQL, which is why the registry leg needs this: the crawler must decide, in Node, whether a code is +// worth a lookup. The two implementations are pinned against each other by a test that lifts the CASE +// expression straight out of the .sql file and runs both over the same values (eik.test.mjs) — a copy +// of the rule would drift from the thing it is copying. +// +// The rule (ЗТРРЮЛНЦ / БУЛСТАТ): +// 9-digit — weight digits 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by 3..10; +// a second 10 becomes 0. Digit 9 must equal the control. +// 13-digit — the leading 9 must themselves be a valid 9-digit ЕИК, then weight digits 9..12 by +// 2,7,3,5 (fallback 4,9,5,7; a second 10 becomes 0). Digit 13 must equal that control. +// +// Everything here is string-in, string-out. An ЕИК is an identifier, not a number: public bodies carry +// codes of exactly the `000…` shape, and a numeric round-trip drops the leading zeros and silently +// turns one company's identifier into another's. + +/** Weighted control digit over `digits`, with the standard second-pass fallback. @returns {number} */ +function control(digits, primary, fallback) { + const sum = (ws) => ws.reduce((acc, w, i) => acc + w * digits[i], 0) % 11; + const first = sum(primary); + if (first < 10) return first; + const second = sum(fallback); + return second < 10 ? second : 0; +} + +/** + * Is this a structurally valid ЕИК (9 or 13 digits, correct control digit)? + * Service codes (`000000000`, `0000000000000`) are rejected outright, matching the SQL: they pass the + * arithmetic but are placeholders, and letting them through collapsed unrelated foreign suppliers onto + * one node (#195). + * @param {unknown} eik @returns {boolean} + */ +export function eikChecksumValid(eik) { + const s = String(eik ?? ''); + if (s === '000000000' || s === '0000000000000') return false; + if (!/^\d+$/.test(s)) return false; + if (s.length !== 9 && s.length !== 13) return false; + + const d = [...s].map(Number); + const c9 = control(d.slice(0, 8), [1, 2, 3, 4, 5, 6, 7, 8], [3, 4, 5, 6, 7, 8, 9, 10]); + if (c9 !== d[8]) return false; + if (s.length === 9) return true; + + const c13 = control(d.slice(8, 12), [2, 7, 3, 5], [4, 9, 5, 7]); + return c13 === d[12]; +} + +/** + * Normalise a raw ЕИК string to digits only, or null when it is not one. + * Mirrors the SQL's `eik_clean`: strips a leading „ЕИК " label and surrounding whitespace, and does + * NOT otherwise repair the value. Validity is a separate question — `eikChecksumValid`. + * @param {unknown} raw @returns {string|null} + */ +export function normalizeEik(raw) { + const s = String(raw ?? '').trim(); + const stripped = (s.startsWith('ЕИК ') ? s.slice(4) : s).trim(); + return /^\d{9}$|^\d{13}$/.test(stripped) ? stripped : null; +} diff --git a/scripts/tr/eik.test.mjs b/scripts/tr/eik.test.mjs new file mode 100644 index 000000000..7a80a191e --- /dev/null +++ b/scripts/tr/eik.test.mjs @@ -0,0 +1,135 @@ +// node:test — the ЕИК checksum, and its PARITY with the SQL that already owns this rule. +// +// Why parity matters: `eik_valid` in scripts/normalize-raw.sql decides which bidders get an ЕИК-keyed +// identity at all, so it defines the ЕИК space the whole matcher works in. A JS twin that disagrees +// would silently accept a code the pipeline rejects (or vice versa) and the registry lookup would be +// made against an entity the rest of the system does not believe exists. The parity test below runs +// both implementations over the same values through node:sqlite, so they cannot drift apart. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { DatabaseSync } from 'node:sqlite'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { eikChecksumValid, normalizeEik } from './eik.mjs'; + +// Real ЕИК, verifiable against the public registers — the only external anchor this rule has. +const REAL = [ + '000696327', // Община София + '831661388', // Министерство на регионалното развитие + '115536179', // „ПИМК" ООД + '207695026', // „Профит Екстра" ЕООД +]; + +test('accepts real 9-digit ЕИК', () => { + for (const e of REAL) assert.equal(eikChecksumValid(e), true, e); +}); + +test('rejects a wrong-digit twin of every real ЕИК', () => { + // The failure this rule exists to stop (#195): a typo twin passing as a distinct real company and + // collapsing unrelated suppliers onto one node. Mutating the LAST digit must always break it. + for (const e of REAL) { + const bad = e.slice(0, 8) + ((Number(e[8]) + 1) % 10); + assert.equal(eikChecksumValid(bad), false, `${bad} (twin of ${e})`); + } +}); + +test('rejects service codes, wrong lengths and non-digits', () => { + for (const bad of [ + '000000000', + '0000000000000', + '', + null, + undefined, + '12345678', // 8 + '1234567890', // 10 — never a valid ЕИК length, which is what makes the ЕГН guard sound + '11553617x', + '115 536 179', + ]) { + assert.equal(eikChecksumValid(bad), false, String(bad)); + } +}); + +test('13-digit branch requires BOTH the 9-digit prefix and the 13th control digit', () => { + // A клон/поделение code: the leading 9 must themselves be a valid ЕИК, then weights 2,7,3,5. + const base = '115536179'; + const valid13 = thirteen(base, '001'); + assert.equal(valid13.length, 13, valid13); + assert.equal(eikChecksumValid(valid13), true, valid13); + // break the 13th digit + const broken = valid13.slice(0, 12) + ((Number(valid13[12]) + 1) % 10); + assert.equal(eikChecksumValid(broken), false, broken); + // a valid 13th control over an INVALID 9-prefix must still fail — both halves are load-bearing + assert.equal(eikChecksumValid(thirteen('115536170', '001')), false); +}); + +// Build a 13-digit ЕИК with a correct final control digit: 9-digit prefix + 3 free digits + control. +// The 2,7,3,5 weights run over positions 9..12 — that is the prefix's OWN control digit plus the three +// free ones — and position 13 is the result. +function thirteen(prefix9, three) { + const s = prefix9 + three; + const d = [...s].map(Number); + const w = (ws) => ws.reduce((a, x, i) => a + x * d[8 + i], 0) % 11; + let c = w([2, 7, 3, 5]); + if (c === 10) { + c = w([4, 9, 5, 7]); + if (c === 10) c = 0; + } + return s + c; +} + +test('normalizeEik strips the „ЕИК " prefix and surrounding whitespace, like the SQL does', () => { + assert.equal(normalizeEik('ЕИК 115536179'), '115536179'); + assert.equal(normalizeEik(' 115536179 '), '115536179'); + assert.equal(normalizeEik('115536179'), '115536179'); + assert.equal(normalizeEik('не е ЕИК'), null); + assert.equal(normalizeEik(null), null); +}); + +test('normalizeEik preserves leading zeros (public bodies are exactly 000…)', () => { + // Lose these and the crawler fetches a DIFFERENT company's deed. String in, string out — never a + // numeric round-trip. + assert.equal(normalizeEik('000696327'), '000696327'); + assert.equal(typeof normalizeEik('000696327'), 'string'); +}); + +// ── the anti-drift pin ──────────────────────────────────────────────────────── +test('JS twin agrees with normalize-raw.sql eik_valid on every value', () => { + const sql = readFileSync(fileURLToPath(new URL('../normalize-raw.sql', import.meta.url)), 'utf8'); + const expr = extractEikValidExpression(sql); + + const db = new DatabaseSync(':memory:'); + db.exec('CREATE TABLE probe (eik_clean TEXT)'); + const ins = db.prepare('INSERT INTO probe VALUES (?)'); + + const cases = [...REAL, ...REAL.map((e) => e.slice(0, 8) + ((Number(e[8]) + 1) % 10))]; + for (const e of REAL) cases.push(thirteen(e, '001'), thirteen(e, '002')); + // Every 9-digit code over a fixed prefix — exercises BOTH weight passes incl. the second-10 → 0 fallback. + for (let i = 0; i < 100; i++) cases.push('2011223' + String(i).padStart(2, '0')); + for (const bad of ['000000000', '0000000000000', '12345678', '1234567890', '11553617x']) + cases.push(bad); + + for (const c of cases) ins.run(c); + const rows = db.prepare(`SELECT eik_clean AS e, ${expr} AS v FROM probe`).all(); + db.close(); + + assert.ok(rows.length >= 120, `expected a broad probe set, got ${rows.length}`); + for (const { e, v } of rows) { + assert.equal( + eikChecksumValid(e), + v === 1, + `disagreement on ${JSON.stringify(e)} (SQL said ${v})`, + ); + } +}); + +/** + * Lift the `CASE … END AS eik_valid` expression straight out of normalize-raw.sql, so the test + * compares against the FILE and not a copy of it. A copy would drift with the thing it is pinning. + */ +function extractEikValidExpression(sql) { + const start = sql.indexOf(' CASE\n WHEN eik_clean IS NULL'); + assert.ok(start > 0, 'eik_valid CASE not found in normalize-raw.sql — the pin lost its anchor'); + const end = sql.indexOf('AS eik_valid', start); + assert.ok(end > start, 'eik_valid terminator not found'); + return sql.slice(start, end); +} diff --git a/scripts/tr/evidence.mjs b/scripts/tr/evidence.mjs new file mode 100644 index 000000000..56bc01efd --- /dev/null +++ b/scripts/tr/evidence.mjs @@ -0,0 +1,306 @@ +// The evidence ladder (issue #279 §5, ADR-0033 decision 1). Pure: deed in, verdict out. Zero network. +// +// Six outcomes, FIRST MATCH WINS: +// 1 bar_joint_stock АД / ЕАД / КДА — never published, whatever follows +// 2 document the declarant's full name is in a live CR_F_7/18/19/23_L entity +// 3 confirmed declared seat == registered seat, or the declarant wrote the ЕИК +// 4 refuted own stake only: absent from live state, and the live ownership record +// predates the declared period — the register covers it and does not name them +// 5 unknown everything else — held +// 6 outside_tr not in the register at all (ДЗЗД, БУЛСТАТ associations) — held +// +// WHAT THIS ESTABLISHES, precisely: the identity of the COMPANY — that the company behind the declared +// name is the same legal entity as the winner we matched. It does NOT establish that the official owns +// it; that claim comes from their own filed declaration and is not a heuristic at all. The failure mode +// of a wrong match is therefore not an invented ownership claim but a real official attached to the +// WRONG company's ЕИК, contracts and money. Still a false public statement about a named person, which +// is why rung 2 requires a full three-token subset match inside a single registry entity, and why the +// filters that can only withhold are kept (ADR-0033 decision 2). + +import { + liveFields, + fullSubsetMatch, + personTokens, + normalizeSettlement, + registrySeat, + registryLegalForm, + latestOwnershipEntryDate, + OWNERSHIP_FIELDS, + MANAGER_FIELD, + ROLE_FIELDS, +} from './deed.mjs'; + +/** + * Version of the RULES, not of the code. §8's monotonicity gate keys on this: a previously published + * link disappearing under an UNCHANGED rules version is a hard finding; under a changed one it is an + * expected diff. Bump it whenever a rung's meaning changes. + */ +export const RULES_VERSION = 'tr-rules-1'; + +/** Rung 2 needs a real three-part Bulgarian name (ЗГР чл. 9). Two tokens is the homonym risk itself. */ +const MIN_NAME_TOKENS = 3; + +/** + * The CLOSED vocabulary a sealed `matched_fact` may take: `seat:`, `role:owner:`, + * `role:manager:`, or `eik`. It must NEVER carry the matched NAME — the deed's names are read + * only to produce a boolean and never leave git-ignored scratch (#279 §9, ADR-0033 decision 5). + * + * The seat token bound is the whole rail. `seat:` is a legitimate prefix, so an unbounded settlement + * pattern admits `seat:ИВАН ПЕТРОВ ГЕОРГИЕВ` — a full three-part Bulgarian name (ЗГР чл. 9) wearing an + * allowed prefix, which is exactly the value a mis-split of the seat field would produce and exactly + * what the rail exists to reject. A settlement is one or two tokens („СОФИЯ", „ВЕЛИКО ТЪРНОВО", + * „ГЕНЕРАЛ ТОШЕВО"); a three-part name is exactly three. Bounding at two separates them cleanly, and a + * rarer 3-token seat stops the run for a human rather than publishing — the correct direction for a rail + * whose failure mode is putting somebody's name on a served column. + * + * Defined ONCE and consumed by both the writer (load.mjs) and the audit, so the two cannot drift into + * a state where the gate permits what the writer emits. + */ +export const MATCHED_FACT_RE = + /^(?:seat:\p{Lu}[\p{Lu}-]*(?: \p{Lu}[\p{Lu}-]*)?|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + +/** True when `fact` is a member of the closed vocabulary. `null` is legal — a rung may match no fact. */ +export function isSealedFact(fact) { + return fact == null || MATCHED_FACT_RE.test(String(fact)); +} + +// Court-registered companies were re-registered into the Търговски регистър in a single administrative +// push, which flattened their entry dates into this window. „Strictly before the declared period" +// certifies nothing when the date is an artefact of the migration rather than of the ownership, so the +// refutation rung is suppressed inside it (R13). A suppressed refutation falls through to `unknown` — +// held, not published, which is the safe direction. +const REREGISTRATION_START = '2011-01-01'; +const REREGISTRATION_END = '2012-12-31'; + +/** + * Find the declarant inside the live entities of the given field codes. + * Matching happens per ENTITY — never against a whole field — because one field routinely holds + * several people and combining tokens across them is the libel bug. + * @returns {{nameCode:string, entryNumber:string|null, entryDate:string|null}|null} + */ +function findPerson(deed, name, nameCodes) { + for (const f of liveFields(deed, nameCodes)) { + for (const entity of f.entities) { + if (fullSubsetMatch(name, entity)) { + return { nameCode: f.nameCode, entryNumber: f.entryNumber, entryDate: f.entryDate }; + } + } + } + return null; +} + +/** + * The registered seat, when it matches one THIS person declared for THIS company and was in force for the + * declared period. Shared by rung 3 (which publishes on it) and rung 2's company-identity corroborator. + * + * R10: seats move. A company that relocated INTO the declared settlement afterwards would confirm falsely, + * so the registered seat must predate the period. + * + * A null `firstDeclaredYear` FAILS the guard rather than skipping it. `load.mjs` passes null whenever no + * history row carried a parseable year, and an unknown year is not a satisfied temporal test — it is the + * absence of one. Reading it as „covers the period" made the weakest rung the only one with no temporal + * check, on exactly the links where we know least, and rung 4 already refuses to run without a year on the + * same ground. The undated-SEAT leg is different and stays: a seat with no entry date is the ordinary shape + * for a company that never moved, and it is checkable — a known year is still on the other side. + * + * @returns {{settlement:string, entryDate:string|null}|null} + */ +function matchDeclaredSeat(deed, declaredSeats, firstDeclaredYear) { + const seat = registrySeat(deed); + // Empty NEVER matches — otherwise every link with no seat data on either side rubber-stamps itself. + if (seat.settlement === '') return null; + if (firstDeclaredYear == null) return null; + if (seat.entryDate != null && seat.entryDate > `${firstDeclaredYear}-12-31`) return null; + const declared = declaredSeats.map(normalizeSettlement).filter((s) => s !== ''); + return declared.includes(seat.settlement) ? seat : null; +} + +/** + * Decide the evidence for one link. + * + * @param {object} input + * @param {object|null} input.deed parsed deed JSON; null only when `outsideTr` + * @param {boolean} [input.outsideTr] the ЕИК is not in the register at all + * @param {string} input.declarantName the office-holder's name as filed + * @param {string[]} [input.declaredSeats] seats declared BY THIS PERSON FOR THIS COMPANY only — + * 4.9% of company-name keys carry more than one distinct + * declared seat, so a company-only key would let one + * person's seat confirm another person's link + * @param {boolean} [input.declaredEik] the declarant wrote the ЕИК in the declaration + * @param {number|null} [input.firstDeclaredYear] + * @param {'self'|'family'} [input.scope] + * @param {boolean} [input.nameGloballyUnique] AND-gate on the WEAKEST rung only + * @param {boolean} [input.companyNameDistinctive] the declared фирма is unlikely to have a national + * twin. Gates an UNCORROBORATED rung 2 (ADR-0035). + * Defaults to FALSE: a caller that forgets it withholds. + * @returns {{kind:string, publishable:boolean, registryRole:string|null, matchedFact:string|null, + * entryNumber:string|null, entryDate:string|null, rulesVersion:string, + * shortName:boolean, latinInName:boolean}} + */ +export function evidenceVerdict(input) { + const { + deed, + outsideTr = false, + declarantName, + declaredSeats = [], + declaredEik = false, + firstDeclaredYear = null, + scope = 'self', + nameGloballyUnique = true, + // Fail-CLOSED, unlike `nameGloballyUnique` above. That one's permissive default is bounded — it gates + // only the weakest rung. This one gates the PRIMARY publishing rung, so a caller that forgets to pass + // it must withhold rather than publish a claim naming a real person against a company we did not + // establish. There is exactly one production caller (load.mjs) and it passes it explicitly. + companyNameDistinctive = false, + } = input; + + const tokens = personTokens(declarantName); + const telemetry = { + rulesVersion: RULES_VERSION, + // Counted, not silently dropped: a refusal we cannot see is a recall hole nobody can size. + shortName: tokens.length < MIN_NAME_TOKENS, + latinInName: /[A-Za-z]/.test(String(declarantName ?? '')), + }; + const verdict = (kind, publishable, extra = {}) => ({ + kind, + publishable, + registryRole: null, + matchedFact: null, + entryNumber: null, + entryDate: null, + ...telemetry, + ...extra, + }); + + if (outsideTr) return verdict('outside_tr', false); + if (deed == null) { + // Fail closed and loudly. A missing deed quietly downgraded to „unknown" is indistinguishable + // from a real hold, and hides a cache gap that should stop the run. + throw new Error('evidenceVerdict: deed is required unless outsideTr is set'); + } + + // ── rung 1 ────────────────────────────────────────────────────────────────── + // A union of the numeric code and the ЗТРРЮЛНЦ suffix; either saying joint-stock bars the link, and + // neither able to say means we withhold rather than guess. + const form = registryLegalForm(deed); + if (form.verdict === 'joint_stock') return verdict('bar_joint_stock', false); + if (form.verdict === 'unknown') return verdict('unknown', false); + + // The registered seat, matched against what THIS person declared for THIS company, with R10's temporal + // guard applied. Computed once and consumed by two rungs: rung 3 publishes „Потвърдено" on it, and rung 2 + // uses it as a COMPANY-IDENTITY corroborator. One implementation, because two copies of "what counts as a + // seat match" would eventually disagree about which links may be published. + const matchedSeat = matchDeclaredSeat(deed, declaredSeats, firstDeclaredYear); + + // ── rung 2 ────────────────────────────────────────────────────────────────── + // Only a full three-token name may assert. A Latin homoglyph makes the name a non-match rather than + // a false match — company-name-key.ts's posture, applied to people. + // + // The company gate (ADR-0035). A name match proves someone with these three tokens is registered in the + // company we LOOKED UP — never that this is the company the official declared. `resolveEntity` picks the + // sole WINNER holding the declared name and `nameGloballyUnique` ranges over bidders only, so an official + // whose real company never bid resolves to a same-named winner, and a homonym in that winner's deed + // completes a link false in both halves. Before rung 2 may assert, something other than the фирма must + // say the company is the declared one: + // • the declarant wrote the ЕИК — the national identifier resolves it outright (ADR-0028); or + // • the declared seat matches the registered one — a twin in another town is excluded; or + // • the фирма is distinctive enough that a national twin is improbable in the first place. + // The third is a bound, not a proof, and it is COUNTED (`documentUncorroborated`) so F8 can decide from + // the measured residual whether to tighten to the first two. Strict corroboration was the alternative; + // declared seats exist only on the ООД/ЕООД table of asset declarations, so its recall cost cannot be + // known before that measurement. + const companyCorroborated = declaredEik || matchedSeat != null; + const eligibleForDocument = !telemetry.shortName && !telemetry.latinInName; + if (eligibleForDocument) { + const owner = findPerson(deed, declarantName, OWNERSHIP_FIELDS); + const manager = owner ? null : findPerson(deed, declarantName, [MANAGER_FIELD]); + const hit = owner ?? manager; + if (hit && !companyCorroborated && !companyNameDistinctive) { + // A DISTINCT withholding kind, not a fall-through to `unknown`. „We matched a person but could not + // establish the company" and „we matched nothing" are different facts about a link, and the review + // queue (which is sealed for held links precisely to be reviewable) has to be able to tell them + // apart. It never publishes, and it carries no role or fact — asserting either would leak the very + // claim the rung just refused to make. + return verdict('document_uncorroborated', false); + } + if (owner) { + return verdict('document', true, { + registryRole: 'owner', + matchedFact: `role:owner:${owner.nameCode}`, + entryNumber: owner.entryNumber, + entryDate: owner.entryDate, + }); + } + if (manager) { + return verdict('document', true, { + registryRole: 'manager', + matchedFact: `role:manager:${manager.nameCode}`, + entryNumber: manager.entryNumber, + entryDate: manager.entryDate, + }); + } + } + + // ── rung 3 ────────────────────────────────────────────────────────────────── + // The weakest publishing rung, so it carries the extra AND-gate: a nationally shared company name + // cannot ride it (ADR-0017's holding, carried forward). The stronger „Документ" rung above is not + // gated — the register named this person in THIS company, which makes the name key moot. + // The declared-ЕИК leg is NOT name-gated. ADR-0028: the ЕИК is the identity, not the name, so it + // resolves the company deterministically even behind a nationally shared фирма — which is exactly the + // case ADR-0017 was written about. Gating it on name uniqueness would discard the strongest + // identifier we have precisely where it is most needed. + if (declaredEik) return verdict('confirmed', true, { matchedFact: 'eik' }); + + // The SEAT leg is name-gated, and only this one. ADR-0017's holding carried forward: a name shared by + // two ЕИК cannot support a name-derived identity claim. The seat still rescues a GENERIC name — that + // is the whole point of the rung — it just cannot rescue a NATIONALLY SHARED one. + if (nameGloballyUnique && matchedSeat != null) { + return verdict('confirmed', true, { + matchedFact: `seat:${matchedSeat.settlement}`, + entryDate: matchedSeat.entryDate, + }); + } + + // ── rung 4 ────────────────────────────────────────────────────────────────── + // OWN stakes only. For a family stake the registered owner is the relative, whose name we neither + // store nor check, so absence of the OFFICIAL from the deed is evidence of nothing. An early branch, + // not a caller convention. + if (scope === 'self' && firstDeclaredYear != null) { + const stillPresent = findPerson(deed, declarantName, ROLE_FIELDS); + const latest = latestOwnershipEntryDate(deed); + const inRereg = + latest != null && latest >= REREGISTRATION_START && latest <= REREGISTRATION_END; + if (!stillPresent && latest != null && !inRereg && latest < `${firstDeclaredYear}-01-01`) { + return verdict('refuted', false, { entryDate: latest }); + } + } + + // ── rung 5 ────────────────────────────────────────────────────────────────── + return verdict('unknown', false); +} + +/** + * Reconcile a DECLARED termination against the live deed (#279 §7). + * + * „Terminated" is an inference from silence — the commonest cause is a finished mandate, not a sale — + * so ADR-0021 E11's withdrawal is checked against the register before it takes effect. + * + * Phase 1 uses `terminated` ONLY. `label` is computed but deliberately not rendered: „и към днешна + * дата" asserts a present tense about a named person on evidence whose freshness is bounded by the + * cache refresh cycle, and it is deferred behind an LIA addendum (ADR-0033 decision 4). + * + * @returns {{terminated:boolean, label:'owner_today'|'manager_today'|null}} + */ +export function reconcileTermination({ deed, declarantName, scope = 'self' }) { + // Family first, structurally: there is nothing to look for, and looking would be an attempt to + // identify the relative. + if (scope !== 'self' || deed == null) return { terminated: true, label: null }; + + if (findPerson(deed, declarantName, OWNERSHIP_FIELDS)) { + return { terminated: false, label: 'owner_today' }; + } + if (findPerson(deed, declarantName, [MANAGER_FIELD])) { + return { terminated: true, label: 'manager_today' }; + } + return { terminated: true, label: null }; +} diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs new file mode 100644 index 000000000..d3f71085c --- /dev/null +++ b/scripts/tr/evidence.test.mjs @@ -0,0 +1,564 @@ +// node:test — the evidence ladder (ADR-0033 decision 1). Pure: deed in, verdict out. +// +// Six outcomes, first match wins. What each rung is allowed to CONCLUDE is the whole subject: +// the registry proves the identity of the COMPANY, never that the official owns it — the ownership +// claim comes from the official's own filed declaration. So a wrong match here does not invent an +// ownership claim, it attaches a real official to the wrong company's ЕИК, contracts and money. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + RULES_VERSION, + evidenceVerdict, + reconcileTermination, + MATCHED_FACT_RE, + isSealedFact, +} from './evidence.mjs'; + +const container = (t) => + `

${t}

`; +const joined = (...ts) => ts.map(container).join(`
`); +const ERASED = `
Заличено обстоятелство.
`; + +const fld = (nameCode, htmlData, entryDate = '2011-05-02T00:00:00') => ({ + nameCode, + htmlData, + fieldEntryNumber: '20110502101007', + fieldEntryDate: entryDate, + fieldOperation: 3, +}); +const deed = (fields, over = {}) => ({ + uic: '201122335', + fullName: '"АЛФА СТРОЙ" ООД', + legalForm: 4, + sections: [{ subDeeds: [{ groups: [{ fields }] }] }], + ...over, +}); + +const OWNER_DEED = deed([ + fld('CR_F_19_L', joined('ИВАН ПЕТРОВ ТЕСТОВ, Държава: БЪЛГАРИЯ', 'МАРИЯ СТОЯНОВА ИВАНОВА')), + fld('CR_F_5_L', container('Държава: БЪЛГАРИЯ
Населено място: гр. Пловдив, п.к. 4000')), +]); + +const base = { + deed: OWNER_DEED, + declarantName: 'Иван Петров Тестов', + declaredSeats: [], + declaredEik: false, + firstDeclaredYear: 2021, + scope: 'self', + nameGloballyUnique: true, + // The company was resolved by a name unlikely to have a national twin. The rung-2 tests below are about + // NAME matching inside a deed, so they hold this dimension fixed; the gate itself is tested separately. + companyNameDistinctive: true, +}; + +test('RULES_VERSION is a stable, non-empty identifier — §8 hangs off it', () => { + assert.equal(typeof RULES_VERSION, 'string'); + assert.ok(RULES_VERSION.length > 0); +}); + +// ── rung 1: the joint-stock bar wins over everything ────────────────────────── +test('rung 1 — a joint-stock company is barred even when the person IS in the deed', () => { + const ad = deed([fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))], { + legalForm: 5, + fullName: '"ГАМА ИНВЕСТ" АД', + }); + const v = evidenceVerdict({ ...base, deed: ad }); + assert.equal(v.kind, 'bar_joint_stock'); + assert.equal(v.publishable, false); +}); + +test('rung 1 — an UNKNOWN legal form withholds; it never falls through to a lower rung', () => { + const odd = deed([fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))], { + legalForm: 99, + fullName: 'НЕЩО БЕЗ ФОРМА', + }); + const v = evidenceVerdict({ ...base, deed: odd }); + assert.equal(v.kind, 'unknown'); + assert.equal(v.publishable, false); +}); + +// ── rung 2: „Документ" ──────────────────────────────────────────────────────── +test('rung 2 — a full-name match in a live ownership field publishes, with the role kept', () => { + const v = evidenceVerdict(base); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); + assert.equal(v.registryRole, 'owner'); + assert.equal(v.matchedFact, 'role:owner:CR_F_19_L'); + assert.equal(v.entryNumber, '20110502101007'); + assert.equal(v.entryDate, '2011-05-02'); +}); + +test('rung 2 — a manager-only match publishes but records the weaker role', () => { + const mgr = deed([ + fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ, Държава: БЪЛГАРИЯ')), + fld('CR_F_19_L', container('ДРУГО ЛИЦЕ ТУК')), + ]); + const v = evidenceVerdict({ ...base, deed: mgr }); + assert.equal(v.kind, 'document'); + assert.equal(v.registryRole, 'manager'); + assert.equal(v.matchedFact, 'role:manager:CR_F_7_L'); +}); + +test('rung 2 — a TWO-token declarant can never earn „Документ"', () => { + // 46 of 301 measured matches were two-token only, which is exactly the homonym risk. Falls to a + // lower rung rather than publishing on a name that half a register could satisfy. + const two = deed([fld('CR_F_19_L', container('ИВАН ТЕСТОВ, Държава: БЪЛГАРИЯ'))]); + const v = evidenceVerdict({ ...base, deed: two, declarantName: 'Иван Тестов' }); + assert.notEqual(v.kind, 'document'); + assert.equal(v.shortName, true, 'the refusal is counted, not silently dropped'); +}); + +test('rung 2 — the match must fall inside ONE entity (the libel guard, end to end)', () => { + const two = deed([ + fld('CR_F_19_L', joined('ПЕНКО НЕСТОРОВ НЕСТОРОВ', 'ИЛИЯН КОСТАДИНОВ ФИЛИПОВ')), + ]); + const v = evidenceVerdict({ ...base, deed: two, declarantName: 'ПЕНКО КОСТАДИНОВ ФИЛИПОВ' }); + assert.notEqual(v.kind, 'document'); +}); + +test('rung 2 — an ERASED ownership entry cannot produce a document match', () => { + const gone = deed([fld('CR_F_23_L', ERASED, '2013-07-16T10:10:07')]); + const v = evidenceVerdict({ ...base, deed: gone }); + assert.notEqual(v.kind, 'document'); +}); + +test('rung 2 — a Latin homoglyph in the name is a NON-match, and is counted', () => { + // company-name-key.ts deliberately does not fold Cyrillic↔Latin; person names take the same posture. + const v = evidenceVerdict({ ...base, declarantName: 'ИBAH ПЕТРОВ ТЕСТОВ' }); // Latin B, A, H + assert.notEqual(v.kind, 'document'); + assert.equal(v.latinInName, true); +}); + +// ── rung 3: „Потвърдено" ────────────────────────────────────────────────────── +test('rung 3 — a declared seat matching the registered seat confirms the company', () => { + const other = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив, п.к. 4000'), '2015-01-01T00:00:00'), + ]); + const v = evidenceVerdict({ ...base, deed: other, declaredSeats: ['Пловдив'] }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.publishable, true); + assert.equal(v.matchedFact, 'seat:ПЛОВДИВ'); +}); + +test('rung 3 — a declared ЕИК confirms the company on its own', () => { + const other = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК'))]); + const v = evidenceVerdict({ ...base, deed: other, declaredEik: true }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'eik'); +}); + +test('rung 3 — an EMPTY declared seat never confirms', () => { + const noSeat = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК'))]); + const v = evidenceVerdict({ ...base, deed: noSeat, declaredSeats: ['', ' '] }); + assert.notEqual(v.kind, 'confirmed'); +}); + +test('rung 3 — a seat registered AFTER the declared period does not confirm', () => { + // R10, and W0 measured that seats move: a company that relocated INTO the declared settlement after + // the fact would otherwise produce a false „Потвърдено". + const moved = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив'), '2024-06-01T00:00:00'), + ]); + const v = evidenceVerdict({ + ...base, + deed: moved, + declaredSeats: ['Пловдив'], + firstDeclaredYear: 2021, + }); + assert.notEqual(v.kind, 'confirmed'); +}); + +test('rung 3 — an UNKNOWN first declared year cannot confirm on a seat', () => { + // R10 again, from the other side. `load.mjs` passes `firstDeclaredYear: null` whenever no history row + // carried a parseable year, and a null year means the temporal check has NOTHING to compare against — + // not that the seat covers the period. The same relocated company as the test above, with the year + // unknown instead of 2021, must reach the same held outcome: an unknown guard is a failed guard. + const moved = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив'), '2024-06-01T00:00:00'), + ]); + const v = evidenceVerdict({ + ...base, + deed: moved, + declaredSeats: ['Пловдив'], + firstDeclaredYear: null, + }); + assert.notEqual(v.kind, 'confirmed'); +}); + +test('rung 3 — an unknown year holds a seat match even when the seat has NO entry date', () => { + // The nastier half: with no entry date on the register side AND no year on the declaration side there + // are two unknowns and zero evidence about the period, yet both legs of the old disjunction read TRUE. + // Rung 4's refutation leg already refuses to run without a year (`firstDeclaredYear != null`); the seat + // leg must refuse on the same ground, or the weakest rung is the one with no temporal check at all. + const undated = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: undated, + declaredSeats: ['Пловдив'], + firstDeclaredYear: null, + }); + assert.notEqual(v.kind, 'confirmed', 'two unknowns must not multiply into a public claim'); +}); + +test('rung 3 — a KNOWN year with an undated seat still confirms (the guard is not a blanket)', () => { + // Positive control. Bounding the null case must not quietly kill the rung: a seat with no entry date + // is the ordinary shape for a company that never moved, and it still confirms under a known year. + const undated = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: undated, + declaredSeats: ['Пловдив'], + firstDeclaredYear: 2021, + }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'seat:ПЛОВДИВ'); +}); + +test('rung 3 — the weakest rung ALSO requires global name uniqueness (ADR-0017 carried forward)', () => { + const other = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: other, + declaredSeats: ['Пловдив'], + nameGloballyUnique: false, + }); + assert.notEqual(v.kind, 'confirmed', 'a nationally shared name cannot ride the weakest rung'); +}); + +test('rung 3 — a declared ЕИК is NOT gated by name uniqueness (ADR-0028)', () => { + // The case ADR-0017 was written about — a фирма backing two ЕИК — is exactly where a declarant-supplied + // ЕИК is most valuable. Gating it on the name would discard the strongest identifier precisely when the + // name is useless. + const other = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК'))]); + const v = evidenceVerdict({ + ...base, + deed: other, + declaredEik: true, + nameGloballyUnique: false, + }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'eik'); +}); + +test('rung 3 — the seat rung DOES rescue a merely generic name; it is uniqueness that gates it', () => { + // The seat leg exists to rescue generic names (a bare one- or two-word фирма). Requiring the name to + // be distinctive would empty the rung of its entire purpose; only NATIONAL non-uniqueness blocks it. + const generic = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: generic, + declaredSeats: ['Пловдив'], + nameGloballyUnique: true, + }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'seat:ПЛОВДИВ'); +}); + +test('rung 3 — name uniqueness does NOT gate the stronger „Документ" rung', () => { + const v = evidenceVerdict({ ...base, nameGloballyUnique: false }); + assert.equal( + v.kind, + 'document', + 'the registry named the person in THIS company; the name key is moot', + ); +}); + +test('rung 2 — every OWNERSHIP field code can carry the match, not just CR_F_19_L', () => { + // OWNERSHIP_FIELDS is ['CR_F_18_L','CR_F_19_L','CR_F_23_L'] — едноличен собственик, съдружници, and + // ФЛ-търговец. Every owner test used CR_F_19_L, so a typo or a dropped entry in the other two would + // have silently withheld an entire ownership shape: a sole owner (the commonest ЕООД form) publishing + // as „Неизвестна" is a recall hole with no symptom. + for (const code of ['CR_F_18_L', 'CR_F_19_L', 'CR_F_23_L']) { + const d = deed([fld(code, container('ИВАН ПЕТРОВ ТЕСТОВ'))]); + const v = evidenceVerdict({ ...base, deed: d }); + assert.equal(v.kind, 'document', `${code} must carry an ownership match`); + assert.equal(v.registryRole, 'owner', `${code} is an OWNERSHIP field, not management`); + assert.equal(v.matchedFact, `role:owner:${code}`); + } + // POSITIVE CONTROL — a field that is NOT an ownership or manager field must not match at all, or the + // loop above would pass for a reason other than the one it claims. + const other = deed([fld('CR_F_99_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))]); + assert.notEqual(evidenceVerdict({ ...base, deed: other }).kind, 'document'); +}); + +// ── rung 2's company gate: the winner-vs-non-winner homonym (ADR-0035) ──────── +// +// The ladder proves a person with these three tokens is registered in the company we LOOKED UP. It cannot +// prove that company is the one the official declared. `resolveEntity` maps a declared name to the sole +// WINNER holding it, and `nameGloballyUnique` ranges over procurement bidders only — never the whole +// register. So when an official owns a same-named company that never bid, we resolve to the winner, and a +// three-token homonym in the winner's deed „proves" a link that is false in both halves. +// +// Two coincidences, and neither is rare in Bulgaria: a shared фирма and a shared three-part name. The gate +// asks for a reason to believe the COMPANY is the declared one before rung 2 may assert. +test('rung 2 — a GENERIC company name with no corroboration cannot publish on a name match alone', () => { + const v = evidenceVerdict({ ...base, companyNameDistinctive: false }); + assert.equal( + v.kind, + 'document_uncorroborated', + 'the deed names SOMEONE with this name in THIS company — not that this is the declared company', + ); + assert.equal(v.publishable, false); + // It must not fall through to `unknown`: the residual is the input to F8's decision on whether this gate + // tightens, and a rung-2 match withheld for want of company identity is a different fact from no match. + assert.equal(v.registryRole, null); + assert.equal(v.matchedFact, null); +}); + +test('rung 2 — a declared ЕИК corroborates the company, so the name match publishes', () => { + // POSITIVE CONTROL. The ЕИК IS the identity (ЗТРРЮЛНЦ, ADR-0028) — it resolves the company behind any + // shared фирма, which is exactly the collision the gate is about. + const v = evidenceVerdict({ ...base, companyNameDistinctive: false, declaredEik: true }); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); + assert.equal(v.registryRole, 'owner'); +}); + +test('rung 2 — a declared seat matching the registered seat corroborates the company', () => { + // POSITIVE CONTROL. The declarant put this company in гр. Пловдив and the register agrees; a national + // twin in another town is excluded by the same fact rung 3 publishes on. + const v = evidenceVerdict({ + ...base, + companyNameDistinctive: false, + declaredSeats: ['гр. Пловдив'], + }); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); +}); + +test('rung 2 — a DISTINCTIVE company name publishes uncorroborated (the gate is not a blanket)', () => { + // POSITIVE CONTROL, and the one that distinguishes this fix from disabling rung 2. A predicate that + // always withheld would satisfy the bar above; this is what it must NOT do. + const v = evidenceVerdict({ ...base, companyNameDistinctive: true }); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); +}); + +test('rung 2 — a seat that does NOT match cannot corroborate a generic name', () => { + // The corroborator has to actually corroborate. A declared seat in another town is evidence AGAINST the + // company being the declared one, so it certainly cannot rescue the rung. + const v = evidenceVerdict({ + ...base, + companyNameDistinctive: false, + declaredSeats: ['гр. Бургас'], // the deed says Пловдив + }); + assert.equal(v.kind, 'document_uncorroborated'); + assert.equal(v.publishable, false); +}); + +test('rung 2 — the seat corroborator carries the SAME temporal guard as rung 3 (R10)', () => { + // A seat registered after the declared period cannot corroborate anything: the company may have moved + // INTO that town afterwards. Rung 3 already refuses it; rungs 2 and 3 share one implementation so they + // cannot drift into disagreeing about what a seat match means. + const moved = deed([ + fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ')), + fld( + 'CR_F_5_L', + container('Държава: БЪЛГАРИЯ
Населено място: гр. Пловдив, п.к. 4000'), + '2023-07-01T00:00:00', + ), + ]); + const v = evidenceVerdict({ + ...base, + deed: moved, + companyNameDistinctive: false, + declaredSeats: ['гр. Пловдив'], + firstDeclaredYear: 2021, + }); + assert.equal(v.kind, 'document_uncorroborated'); +}); + +test('rung 2 — the company gate never rescues a link rung 1 has barred', () => { + // Ordering: a joint-stock bar outranks everything, corroborated or not. The gate adds a way to WITHHOLD, + // never a way to publish something a stronger rung refused. + const ad = deed([fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))], { + legalForm: 5, + fullName: '"ГАМА ИНВЕСТ" АД', + }); + const v = evidenceVerdict({ ...base, deed: ad, companyNameDistinctive: true, declaredEik: true }); + assert.equal(v.kind, 'bar_joint_stock'); +}); + +// ── rung 4: „Оборена" ───────────────────────────────────────────────────────── +test('rung 4 — absent from a deed whose ownership predates the declaration refutes the link', () => { + const older = deed([ + fld('CR_F_19_L', container('СЪВСЕМ ДРУГ СОБСТВЕНИК'), '2015-03-01T00:00:00'), + ]); + const v = evidenceVerdict({ ...base, deed: older, firstDeclaredYear: 2021 }); + assert.equal(v.kind, 'refuted'); + assert.equal(v.publishable, false); +}); + +test('rung 4 — the comparison is date-to-DATE, not date-to-year', () => { + // R17: „strictly before the first declared year" means before YYYY-01-01. An entry inside the first + // declared year does NOT cover the period and must not refute. + const inYear = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2021-06-15T00:00:00')]); + assert.notEqual( + evidenceVerdict({ ...base, deed: inYear, firstDeclaredYear: 2021 }).kind, + 'refuted', + ); + const justBefore = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2020-12-31T00:00:00')]); + assert.equal( + evidenceVerdict({ ...base, deed: justBefore, firstDeclaredYear: 2021 }).kind, + 'refuted', + ); +}); + +test('rung 4 — NEVER applies to a family stake', () => { + // The owner there is the relative, whose name we neither store nor check (ADR-0010 item 4, + // ADR-0032 decision 2), so „the official is not in the deed" says nothing at all. + const older = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2015-03-01T00:00:00')]); + const v = evidenceVerdict({ ...base, deed: older, scope: 'family', firstDeclaredYear: 2021 }); + assert.notEqual(v.kind, 'refuted'); + assert.equal(v.kind, 'unknown'); +}); + +test('rung 4 — suppressed inside the 2011–2012 re-registration window', () => { + // R13: court-registered companies had every entry date flattened into the re-registration window, + // so „strictly before" certifies nothing there. + const flattened = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2011-11-04T00:00:00')]); + const v = evidenceVerdict({ ...base, deed: flattened, firstDeclaredYear: 2021 }); + assert.notEqual(v.kind, 'refuted'); + assert.equal(v.kind, 'unknown'); +}); + +// ── rungs 5 and 6 ───────────────────────────────────────────────────────────── +test('rung 5 — everything else is „Неизвестна" and stays hidden', () => { + const recent = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2023-01-01T00:00:00')]); + const v = evidenceVerdict({ ...base, deed: recent, firstDeclaredYear: 2021 }); + assert.equal(v.kind, 'unknown'); + assert.equal(v.publishable, false); +}); + +test('rung 6 — outside the register is its own outcome, and is not publishable', () => { + const v = evidenceVerdict({ ...base, deed: null, outsideTr: true }); + assert.equal(v.kind, 'outside_tr'); + assert.equal(v.publishable, false); +}); + +test('a missing deed that is NOT marked outside-ТР is an error, not a silent hold', () => { + // Fail closed: a cache gap must be visible, never quietly downgraded to „unknown". + assert.throws(() => evidenceVerdict({ ...base, deed: null, outsideTr: false }), /deed/i); +}); + +// ── the seal ────────────────────────────────────────────────────────────────── +test('MATCHED_FACT_RE bounds a settlement to two tokens — a NAME cannot wear the seat: prefix', () => { + // The rail tested DIRECTLY, not just through whatever verdicts the ladder happens to produce. Both + // seal tests previously restated this regex locally and got it WRONG in the permissive direction — + // `seat:` followed by unlimited uppercase tokens — so a three-part Bulgarian name (ЗГР чл. 9) wearing + // an allowed prefix passed them. That value is exactly what a mis-split of the seat field produces, + // and it is the one shape this rail exists to keep off a served column. + for (const ok of [ + 'seat:СОФИЯ', + 'seat:ВЕЛИКО ТЪРНОВО', // a real two-token settlement must still pass + 'seat:ГЕНЕРАЛ ТОШЕВО', + 'seat:ЦАР-КАЛОЯН', // hyphenated is one token + 'role:owner:CR_F_19_L', + 'role:manager:CR_F_7_L', + 'role:owner:CR_F_23_L', + 'eik', + ]) + assert.equal(MATCHED_FACT_RE.test(ok), true, `wrongly rejected: ${ok}`); + + for (const bad of [ + 'seat:ИВАН ПЕТРОВ ГЕОРГИЕВ', // THE case: three tokens is a name, not a settlement + 'seat:ИВАН ПЕТРОВ ГЕОРГИЕВ ДРУГ', + 'ИВАН ПЕТРОВ ГЕОРГИЕВ', // a bare name with no prefix at all + 'role:owner:ИВАН ПЕТРОВ', // a name where a field code belongs + 'role:cashier:CR_F_19_L', // a role outside the vocabulary + 'seat:', // an empty settlement asserts nothing + 'eik:201122335', // the ЕИК itself is never stored, only the fact that one matched + ]) + assert.equal(MATCHED_FACT_RE.test(bad), false, `wrongly accepted: ${bad}`); + + // null is legal — a rung may match no fact — and that is isSealedFact's job, not the regex's. + assert.equal(isSealedFact(null), true); + assert.equal(isSealedFact('seat:ИВАН ПЕТРОВ ГЕОРГИЕВ'), false); +}); + +test('matched_fact stays inside the closed vocabulary — it can never carry a name', () => { + // The PRODUCTION predicate, imported — never a local copy of it. A re-stated regex here was looser + // than `MATCHED_FACT_RE` (it allowed `seat:` + unlimited tokens), so this loop certified values the + // real rail rejects and could not fail on the regression it exists to catch (cefothe, #309). + for (const v of [ + evidenceVerdict(base), + evidenceVerdict({ ...base, deed: deed([fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))]) }), + evidenceVerdict({ ...base, declaredEik: true }), + evidenceVerdict({ + ...base, + deed: deed([ + fld('CR_F_19_L', container('ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]), + declaredSeats: ['Пловдив'], + }), + ]) { + if (v.matchedFact == null) continue; + assert.ok(isSealedFact(v.matchedFact), `matched_fact escaped the vocabulary: ${v.matchedFact}`); + assert.ok(!/ИВАН|ПЕТРОВ|ТЕСТОВ/.test(v.matchedFact), 'a NAME reached matched_fact'); + } +}); + +test('every verdict carries the rules version that produced it', () => { + assert.equal(evidenceVerdict(base).rulesVersion, RULES_VERSION); +}); + +// ── §7 reconciliation ───────────────────────────────────────────────────────── +test('reconcileTermination — still a registered owner ⇒ NOT terminated', () => { + const r = reconcileTermination({ + deed: OWNER_DEED, + declarantName: 'Иван Петров Тестов', + scope: 'self', + }); + assert.equal(r.terminated, false); + assert.equal(r.label, 'owner_today'); +}); + +test('reconcileTermination — manager only ⇒ terminated as a stake, but the tie continues', () => { + const mgr = deed([ + fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ')), + fld('CR_F_19_L', container('ДРУГ')), + ]); + const r = reconcileTermination({ deed: mgr, declarantName: 'Иван Петров Тестов', scope: 'self' }); + assert.equal(r.terminated, true); + assert.equal(r.label, 'manager_today'); +}); + +test('reconcileTermination — absent from the live deed ⇒ the declared termination stands', () => { + const none = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ'))]); + const r = reconcileTermination({ + deed: none, + declarantName: 'Иван Петров Тестов', + scope: 'self', + }); + assert.equal(r.terminated, true); + assert.equal(r.label, null); +}); + +test('reconcileTermination — a FAMILY stake is never reconciled, by an early branch', () => { + // Structural, not a caller convention: the relative's name is not stored, so there is nothing to + // look for, and looking would be a de-anonymisation attempt. + const r = reconcileTermination({ + deed: OWNER_DEED, + declarantName: 'Иван Петров Тестов', + scope: 'family', + }); + assert.equal(r.terminated, true); + assert.equal(r.label, null); +}); diff --git a/scripts/tr/fetch-deeds.mjs b/scripts/tr/fetch-deeds.mjs new file mode 100644 index 000000000..cdea4f73f --- /dev/null +++ b/scripts/tr/fetch-deeds.mjs @@ -0,0 +1,296 @@ +// The deed crawler (issue #279 §3, ADR-0033). One request per candidate ЕИК, sequential and paced. +// +// This is the only component in the project that touches a public register at volume, so what it +// REFUSES to do is the substance: +// +// • It never goes faster than 1 request / 3 s, and the flag that sets the pace cannot be used to go +// faster — only slower. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the +// limiter is the operator's only way to state a preference, and tuning around it empirically is +// what that rule exists to prevent. +// • It never retries a 429. The block is sustained (see client.mjs), so a 429 ends the run with its +// own exit code and records NOTHING about the ЕИК that hit it — that ЕИК is unknown, not absent. +// • It never follows a link out of a deed. The candidate set is closed: whatever the caller passes +// in, nothing more. This is what keeps a bounded lookup from drifting into a crawl. +// • It only writes „outside the register" on a DOCUMENTED negative — measured to be an HTTP 200 +// with an empty body, not the 404 the issue predicts. A 5xx or a timeout is transient, and caching +// it as permanent would turn an outage into data that §8 never revisits. +// +// Resumable by construction: the cache is consulted first, so an interrupted run picks up exactly +// where it stopped and a complete cache costs zero requests. + +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { assertTrScratchIgnored, TR_DB, TR_RAW, safeEik, deedPath } from './paths.mjs'; +import { eikChecksumValid } from './eik.mjs'; +import { deedUrl, politeTrGet, RateLimitError, httpsGet } from './client.mjs'; +import { + openCache, + upsertDeed, + markOutsideTr, + pendingEiks, + purgeExpired, + RETENTION_DAYS, +} from './cache.mjs'; +import { + assertUicEcho, + registryLegalForm, + registrySeat, + latestOwnershipEntryDate, +} from './deed.mjs'; + +/** The documented polite pace: 1 request / 3 s (#279 §3). A floor, never a target to tune down. */ +export const MIN_INTERVAL_MS = 3000; + +/** + * Consecutive unresolved ЕИК before the run gives up rather than keep hammering. + * + * Deliberately small, because the unit is ЕИК and not requests: each unresolved candidate costs up to + * 5 attempts (#279 §3's documented retry budget), so the breaker's real cost is `BREAKER_TRIP × 5` + * requests against an endpoint that is already failing. At 10 that would be ~50 — the exact volume at + * which the register was observed to start returning a sustained 429. At 5 the worst case is ~25, + * which the same observation saw pass without a block. + */ +export const BREAKER_TRIP = 5; +/** Attempts per candidate, per #279 §3. Exported so the breaker's request budget is derivable. */ +export const TRIES_PER_EIK = 5; + +export function parseTrOptions(argv) { + const get = (name, def) => { + const i = argv.indexOf(`--${name}`); + return i >= 0 && argv[i + 1] ? argv[i + 1] : def; + }; + const posInt = (raw, name) => { + const n = Number(raw); + if (!Number.isInteger(n) || n < 1) + throw new Error(`--${name} must be a positive integer, got ${JSON.stringify(raw)}`); + return n; + }; + + const eiksFile = get('eiks-file', ''); + if (!eiksFile) throw new Error('--eiks-file is required (the closed candidate set)'); + + const limitRaw = get('limit', ''); + const intervalRaw = get('min-interval-ms', ''); + const minIntervalMs = intervalRaw ? posInt(intervalRaw, 'min-interval-ms') : MIN_INTERVAL_MS; + // Slower is always allowed; faster is not a knob. Making this un-passable is the point. + if (minIntervalMs < MIN_INTERVAL_MS) { + throw new Error( + `--min-interval-ms may not go below the documented pace of ${MIN_INTERVAL_MS}ms — ` + + `the register's rate limit is a stated preference, not an obstacle to tune around`, + ); + } + const maxAgeRaw = get('max-age-days', ''); + // --max-age-days is FRESHNESS (re-request past it); --retention-days is RETENTION (delete past it). + // They are separate flags because they are separate obligations: refreshing rewrites the personal + // data, only purging removes it. Retention defaults to the ADR's 35 days rather than to „off", so + // the rail holds for an operator who passes neither. + const retentionRaw = get('retention-days', ''); + return { + eiksFile, + limit: limitRaw ? posInt(limitRaw, 'limit') : Infinity, + minIntervalMs, + maxAgeDays: maxAgeRaw ? posInt(maxAgeRaw, 'max-age-days') : null, + retentionDays: retentionRaw ? posInt(retentionRaw, 'retention-days') : RETENTION_DAYS, + }; +} + +/** Read the closed candidate set: one ЕИК per line, blanks and `#` comments ignored. */ +export function readEiksFile(file) { + return fs + .readFileSync(file, 'utf8') + .split('\n') + .map((l) => l.trim()) + .filter((l) => l !== '' && !l.startsWith('#')); +} + +function atomicWrite(file, buf) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const tmp = `${file}.tmp-${process.pid}`; + fs.writeFileSync(tmp, buf); + fs.renameSync(tmp, file); +} + +/** + * Crawl the candidate ЕИК. Returns the intended process exit code, so the decision is testable + * without a global side effect: + * 0 — every candidate resolved (or the run was deliberately bounded by --limit) + * 1 — at least one candidate is unresolved (transient failure, refused deed, breaker tripped) + * 2 — the register rate-limited us; the run stopped and nothing was marked + * + * Every I/O edge is injectable so the whole policy is exercised offline. + */ +export async function run({ + httpGet = httpsGet, + sleep = (ms) => new Promise((r) => setTimeout(r, ms)), + now = () => new Date(), + guard = assertTrScratchIgnored, + dbFile = TR_DB, + rawDir = TR_RAW, + argv = process.argv, +} = {}) { + guard(); + const { eiksFile, limit, minIntervalMs, maxAgeDays, retentionDays } = parseTrOptions(argv); + + const requested = readEiksFile(eiksFile); + // A shape- or checksum-invalid code is dropped BEFORE any request: it cannot name a real company, + // so asking about it would spend the register's budget to learn nothing. + const candidates = []; + let invalid = 0; + for (const raw of requested) { + try { + const eik = safeEik(raw); + if (!eikChecksumValid(eik)) throw new Error('checksum'); + candidates.push(eik); + } catch { + invalid++; + } + } + + const db = openCache(dbFile); + try { + const pending = pendingEiks(db, candidates, { maxAgeDays, now: now() }); + const todo = Number.isFinite(limit) ? pending.slice(0, limit) : pending; + console.log( + `candidates ${candidates.length} · invalid ${invalid} · cached ${candidates.length - pending.length} · to fetch ${todo.length}`, + ); + + let unresolved = 0; + let consecutive = 0; + let first = true; + + for (const eik of todo) { + if (!first) await sleep(minIntervalMs); // pace BETWEEN requests, not before the first + first = false; + + let res; + try { + res = await politeTrGet(deedUrl(eik), { httpGet, sleep, tries: TRIES_PER_EIK }); + } catch (err) { + if (err instanceof RateLimitError) { + // Stop the whole run. Recording anything here would attribute the register's throttle to + // this ЕИК, which is a fact about us, not about the company. + console.error(`${err.message}\nSTOPPING — re-run later; progress so far is cached.`); + return 2; + } + console.error( + ` ${eik}: ${err instanceof Error ? err.message : err} (transient, not cached)`, + ); + unresolved++; + consecutive++; + if (consecutive >= BREAKER_TRIP) { + console.error(`breaker: ${consecutive} consecutive failures — aborting the run`); + return 1; + } + continue; + } + + // ── the documented negatives ──────────────────────────────────────────── + // MEASURED 2026-08-05: an ЕИК that is not a търговец answers **HTTP 200 with a ZERO-BYTE body**, + // not a 404 and not the HTML #279 §3 predicts. Verified on Община София (000696327): empty on + // two consecutive requests, while a real company returned its full 34,398-byte deed in the same + // window — so it is the register's answer, not an outage. + // + // The distinction that keeps R6 honest is the STATUS, not the empty body: an empty body under + // 200 is the register saying „no deed"; an empty body under 5xx is a failure and stays + // transient. Getting this backwards either caches a false negative forever or leaves ~4 ЕИК + // permanently unresolved so the run can never exit 0. + if (res.status === 200 && res.body.length === 0) { + markOutsideTr(db, eik, 'HTTP 200, empty body — no deed in the Търговски регистър', now()); + consecutive = 0; + continue; + } + if (res.status === 404) { + markOutsideTr(db, eik, 'HTTP 404 — not in the Търговски регистър (BULSTAT/ДЗЗД?)', now()); + consecutive = 0; + continue; + } + if (res.status !== 200) { + console.error(` ${eik}: HTTP ${res.status} after retries (transient, not cached)`); + unresolved++; + consecutive++; + if (consecutive >= BREAKER_TRIP) { + console.error(`breaker: ${consecutive} consecutive failures — aborting the run`); + return 1; + } + continue; + } + + // ONE refuse-and-continue block around EVERYTHING derived from the response — the JSON, the UIC + // echo, and the HTML parsing alike. The parsing used to sit outside it, which made the block's + // own promise false: a throw from deed.mjs escaped the loop, escaped run(), and killed the + // process, so one malformed deed ended a crawl that had already spent its paced request budget. + // The decode guard in deed.mjs removes the one throw we know of; this is the rail that holds when + // the next one appears, and the cost of being wrong here is measured in hours of pacing. + try { + const deed = JSON.parse(res.body.toString('utf8')); + // The deed we got back must be the deed we asked for, or every claim derived from it names + // the wrong company (R8). + assertUicEcho(deed, eik); + + // The raw response is the ONLY place names live; it stays under git-ignored scratch. Written + // only after the echo check, so a deed for the wrong company never lands on disk. + atomicWrite(deedPath(eik, rawDir), res.body); + const seat = registrySeat(deed); + const form = registryLegalForm(deed); + upsertDeed(db, { + eik, + status: 'fetched', + httpStatus: 200, + fetchedAt: now().toISOString(), + rawPath: path.relative(rawDir, deedPath(eik, rawDir)), + bodySha256: crypto.createHash('sha256').update(res.body).digest('hex'), + legalFormCode: form.code, + legalFormVerdict: form.verdict, + seatNormalized: seat.settlement || null, + seatEntryDate: seat.entryDate, + latestOwnEntryDate: latestOwnershipEntryDate(deed), + }); + } catch (err) { + console.error(` ${eik}: REFUSED — ${err instanceof Error ? err.message : err}`); + unresolved++; + consecutive++; + continue; + } + consecutive = 0; + } + + if (unresolved > 0) { + console.error(`${unresolved} candidate(s) unresolved — the cache is incomplete`); + return 1; + } + return 0; + } finally { + // The purge step ADR-0033 decision 5 puts „in the same job" — in `finally`, and that placement is + // the point. Retention is an obligation about other people's data, not a reward for a clean run, + // so it must also happen on the paths that leave early: a 429 (exit 2), a tripped breaker, an + // unresolved candidate. Under normal operation it removes nothing, because the monthly refresh + // rewrites each row well inside the window; anything it does delete is residue — a company that + // left the candidate set, or a refresh that never landed. + try { + const purged = purgeExpired(db, rawDir, { retentionDays, now: now() }); + if (purged.rows || purged.files || purged.orphans) { + console.log( + `purged ${purged.rows} row(s), ${purged.files} raw deed(s), ${purged.orphans} orphan(s) past ${retentionDays}d retention`, + ); + } + } catch (e) { + // A failed purge must be loud but must not mask the run's own outcome — especially not a 429, + // whose exit code is what tells the operator to back off. + console.error(`purge failed: ${e.message}`); + } + db.close(); + } +} + +// CLI entry. Kept off the import path so the module stays testable. +if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) { + run() + .then((code) => { + process.exitCode = code; + }) + .catch((err) => { + console.error(err); + process.exitCode = 1; + }); +} diff --git a/scripts/tr/fetch-deeds.test.mjs b/scripts/tr/fetch-deeds.test.mjs new file mode 100644 index 000000000..3a4b1dc01 --- /dev/null +++ b/scripts/tr/fetch-deeds.test.mjs @@ -0,0 +1,427 @@ +// node:test — the deed crawler, driven entirely through injected I/O. No network, no real scratch. +// +// This is the only component that touches a public register at volume, so what it must NOT do is the +// substance: never exceed the pace, never retry a 429, never turn a transient wall into permanent +// data, and never re-request what it already holds. Spec §3.3 permits a bounded per-ЕИК lookup and +// forbids bulk scraping; the difference between the two is enforced here. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { + parseTrOptions, + run, + MIN_INTERVAL_MS, + BREAKER_TRIP, + TRIES_PER_EIK, +} from './fetch-deeds.mjs'; + +const DEED = (uic) => ({ + uic, + fullName: '"АЛФА СТРОЙ" ООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: `

ИВАН ПЕТРОВ ТЕСТОВ

`, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + { + nameCode: 'CR_F_5_L', + htmlData: `

Населено място: гр. Пловдив

`, + fieldEntryNumber: '20110502101008', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], +}); + +const ok = (uic) => ({ + status: 200, + headers: {}, + body: Buffer.from(JSON.stringify(DEED(uic)), 'utf8'), +}); +const status = (s) => ({ status: s, headers: {}, body: Buffer.from('') }); + +// Two valid ЕИК (real checksums) plus one that is shape-valid but checksum-invalid. +const A = '201122335'; +const B = '203445566'; +const C = '204556676'; +const BAD_CHECKSUM = '201122336'; + +function ctx() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-crawl-')); + return { + dir, + dbFile: path.join(dir, 'tr-cache.sqlite'), + rawDir: path.join(dir, 'deeds'), + cleanup: () => fs.rmSync(dir, { recursive: true, force: true }), + }; +} +const eiksFile = (dir, eiks) => { + const f = path.join(dir, 'eiks.txt'); + fs.writeFileSync(f, eiks.join('\n') + '\n'); + return f; +}; + +/** Drive run() with a recording transport. `routes` maps ЕИК → response (or a function of attempt). */ +function harness(c, eiks, routes, extraArgv = []) { + const calls = []; + const waits = []; + const httpGet = async (url) => { + const eik = url.split('/').pop(); + calls.push(eik); + const r = routes[eik]; + return typeof r === 'function' ? r(calls.filter((e) => e === eik).length) : (r ?? status(404)); + }; + return { + calls, + waits, + promise: run({ + httpGet, + sleep: async (ms) => void waits.push(ms), + now: () => new Date('2026-08-05T12:00:00Z'), + guard: () => {}, + dbFile: c.dbFile, + rawDir: c.rawDir, + argv: ['node', 'fetch-deeds.mjs', '--eiks-file', eiksFile(c.dir, eiks), ...extraArgv], + }), + }; +} + +const openCacheRO = (dbFile) => new DatabaseSync(dbFile); +const rows = (dbFile) => { + const db = openCacheRO(dbFile); + const r = db.prepare('SELECT eik, status, outside_reason FROM deeds ORDER BY eik').all(); + db.close(); + return r; +}; + +// ── options ─────────────────────────────────────────────────────────────────── +test('parseTrOptions: defaults are the documented polite pace', () => { + const o = parseTrOptions(['node', 'x', '--eiks-file', '/tmp/e.txt']); + assert.equal(o.eiksFile, '/tmp/e.txt'); + assert.equal(o.limit, Infinity); + assert.equal(o.minIntervalMs, MIN_INTERVAL_MS); + assert.ok(MIN_INTERVAL_MS >= 3000, 'the documented pace is 1 request / 3 s'); +}); + +test('parseTrOptions: rejects a pace FASTER than the documented one', () => { + // Tuning around a limiter empirically is precisely what spec §3.3 forbids; make it un-passable. + assert.throws( + () => parseTrOptions(['node', 'x', '--eiks-file', '/e', '--min-interval-ms', '100']), + /min-interval/i, + ); +}); + +test('parseTrOptions: --eiks-file is required, and numeric flags are validated', () => { + assert.throws(() => parseTrOptions(['node', 'x']), /eiks-file/i); + for (const bad of ['0', '-1', 'abc', '1.5']) + assert.throws( + () => parseTrOptions(['node', 'x', '--eiks-file', '/e', '--limit', bad]), + /limit/i, + bad, + ); +}); + +// ── pacing ──────────────────────────────────────────────────────────────────── +test('requests are sequential and spaced by at least the documented interval', async () => { + const c = ctx(); + try { + const h = harness(c, [A, B, C], { [A]: ok(A), [B]: ok(B), [C]: ok(C) }); + assert.equal(await h.promise, 0); + assert.deepEqual(h.calls, [A, B, C], 'sequential, in order — never concurrent'); + const paces = h.waits.filter((w) => w >= MIN_INTERVAL_MS); + assert.ok(paces.length >= 2, `expected a pace wait between requests, got ${h.waits.join(',')}`); + } finally { + c.cleanup(); + } +}); + +// ── 429 ─────────────────────────────────────────────────────────────────────── +test('a 429 ends the run with exit 2 and marks NOTHING', async () => { + const c = ctx(); + try { + const h = harness(c, [A, B, C], { [A]: ok(A), [B]: status(429), [C]: ok(C) }); + assert.equal(await h.promise, 2, 'a rate-limit block is its own exit code'); + assert.deepEqual(h.calls, [A, B], 'stops AT the 429 — C is never requested'); + // B must not be recorded at all: it is unknown, not absent, and certainly not outside the register. + assert.deepEqual( + rows(c.dbFile).map((r) => r.eik), + [A], + ); + } finally { + c.cleanup(); + } +}); + +test('after a 429 the run resumes exactly where it stopped', async () => { + const c = ctx(); + try { + const first = harness(c, [A, B, C], { [A]: ok(A), [B]: status(429), [C]: ok(C) }); + assert.equal(await first.promise, 2); + const second = harness(c, [A, B, C], { [A]: ok(A), [B]: ok(B), [C]: ok(C) }); + assert.equal(await second.promise, 0); + assert.deepEqual(second.calls, [B, C], 'A is already cached — not re-requested'); + } finally { + c.cleanup(); + } +}); + +// ── resumability ────────────────────────────────────────────────────────────── +test('a re-run over a complete cache makes ZERO requests', async () => { + const c = ctx(); + try { + assert.equal(await harness(c, [A, B], { [A]: ok(A), [B]: ok(B) }).promise, 0); + const again = harness(c, [A, B], { [A]: ok(A), [B]: ok(B) }); + assert.equal(await again.promise, 0); + assert.deepEqual(again.calls, [], 'nothing to fetch'); + } finally { + c.cleanup(); + } +}); + +// ── permanence ──────────────────────────────────────────────────────────────── +test('an empty 200 is the register saying „no deed" and is cached as outside-ТР', async () => { + // MEASURED against the live API: an ЕИК that is not a търговец (Община София, 000696327) answers + // HTTP 200 with a ZERO-BYTE body — not the 404 or HTML #279 §3 predicts. Reproduced twice, with a + // real company returning its full deed in the same window, so it is an answer and not an outage. + const c = ctx(); + try { + const empty = { status: 200, headers: {}, body: Buffer.alloc(0) }; + assert.equal(await harness(c, [A], { [A]: empty }).promise, 0); + const [row] = rows(c.dbFile); + assert.equal(row.status, 'outside_tr'); + assert.match(row.outside_reason, /empty body/i); + } finally { + c.cleanup(); + } +}); + +test('an empty body under a 5xx stays TRANSIENT — the status decides, not the emptiness', async () => { + // The pair that keeps R6 honest. Both responses have a zero-byte body; only the 200 is an answer. + const c = ctx(); + try { + assert.equal(await harness(c, [A], { [A]: status(503) }).promise, 1); + assert.deepEqual(rows(c.dbFile), [], 'a 5xx must never become permanent „outside ТР"'); + } finally { + c.cleanup(); + } +}); + +test('a 404 is a DOCUMENTED negative and is cached as outside-ТР', async () => { + const c = ctx(); + try { + assert.equal(await harness(c, [A], { [A]: status(404) }).promise, 0); + const [row] = rows(c.dbFile); + assert.equal(row.status, 'outside_tr'); + assert.match(row.outside_reason, /404/); + } finally { + c.cleanup(); + } +}); + +test('a persistent 5xx is TRANSIENT and is never cached as outside-ТР', async () => { + // R6: „outside the register" is permanent by intent. Writing it after a transient wall turns a + // temporary outage into permanent data that §8 never re-examines. + const c = ctx(); + try { + const code = await harness(c, [A], { [A]: status(503) }).promise; + assert.equal(code, 1, 'an unresolved ЕИК makes the run incomplete'); + assert.deepEqual(rows(c.dbFile), [], 'nothing may be recorded from a 5xx'); + } finally { + c.cleanup(); + } +}); + +test('a deed whose UIC does not echo the request is REFUSED, not cached', async () => { + // R8, at the crawl boundary: if anything rewrote the identifier we would be caching one company's + // deed under another company's ЕИК, and every downstream claim about it would name the wrong firm. + const c = ctx(); + try { + const code = await harness(c, [A], { [A]: ok('999999999') }).promise; + assert.equal(code, 1); + assert.deepEqual(rows(c.dbFile), []); + assert.deepEqual(fs.existsSync(c.rawDir) ? fs.readdirSync(c.rawDir) : [], []); + } finally { + c.cleanup(); + } +}); + +test('a checksum-invalid ЕИК is skipped without ever being requested', async () => { + const c = ctx(); + try { + const h = harness(c, [BAD_CHECKSUM, A], { [A]: ok(A) }); + assert.equal(await h.promise, 0); + assert.deepEqual(h.calls, [A], 'the invalid code costs the register nothing'); + } finally { + c.cleanup(); + } +}); + +// ── output ──────────────────────────────────────────────────────────────────── +test('the raw deed is written under the raw dir and the index records only non-PII', async () => { + const c = ctx(); + try { + assert.equal(await harness(c, [A], { [A]: ok(A) }).promise, 0); + assert.deepEqual(fs.readdirSync(c.rawDir), [`${A}.json`]); + const db = openCacheRO(c.dbFile); + const row = db.prepare('SELECT * FROM deeds WHERE eik = ?').get(A); + db.close(); + assert.equal(row.status, 'fetched'); + assert.equal(row.legal_form_verdict, 'closely_held'); + assert.equal(row.seat_normalized, 'ПЛОВДИВ'); + assert.equal(row.body_sha256.length, 64); + // The raw file holds the names; the index must not. + assert.ok(!JSON.stringify(row).includes('ТЕСТОВ')); + assert.ok(fs.readFileSync(path.join(c.rawDir, `${A}.json`), 'utf8').includes('ТЕСТОВ')); + } finally { + c.cleanup(); + } +}); + +test('--limit bounds a run without marking the remainder as anything', async () => { + const c = ctx(); + try { + const h = harness(c, [A, B, C], { [A]: ok(A), [B]: ok(B), [C]: ok(C) }, ['--limit', '2']); + assert.equal(await h.promise, 0, 'a deliberately bounded run is not an incomplete one'); + assert.deepEqual(h.calls, [A, B]); + assert.deepEqual( + rows(c.dbFile).map((r) => r.eik), + [A, B], + ); + } finally { + c.cleanup(); + } +}); + +/** + * N genuinely checksum-valid 9-digit ЕИК. Generated, not hand-written: an earlier version of the + * breaker test below used made-up codes, of which only 1 in 10 was valid — so the crawler dropped + * them all before requesting anything and the assertion passed on zero calls. A test that exercises + * nothing is worse than no test (ADR-0027). + */ +function validEiks(n) { + const control = (p8) => { + const d = [...p8].map(Number); + let s = d.reduce((a, x, i) => a + x * (i + 1), 0) % 11; + if (s === 10) { + s = d.reduce((a, x, i) => a + x * (i + 3), 0) % 11; + if (s === 10) s = 0; + } + return s; + }; + const out = []; + for (let i = 0; out.length < n; i++) { + const p8 = String(20000000 + i); + out.push(p8 + control(p8)); + } + return out; +} + +test('the ЕИК generator used by the breaker test really produces valid codes', () => { + const eiks = validEiks(15); + assert.equal(eiks.length, 15); + assert.equal(new Set(eiks).size, 15); + // Proves the breaker test below actually reaches the network path rather than being filtered out. + assert.deepEqual( + eiks.filter((e) => e.length !== 9), + [], + ); +}); + +test('the circuit breaker aborts a sustained wall of failures', async () => { + // A long run against a broken endpoint must stop hammering, even though each individual 5xx is + // „transient". Distinct from the 429 path: that stops instantly and deliberately. + const c = ctx(); + try { + const many = validEiks(BREAKER_TRIP + 10); + const routes = Object.fromEntries(many.map((e) => [e, status(503)])); + const h = harness(c, many, routes); + assert.equal(await h.promise, 1); + + const attempted = [...new Set(h.calls)]; + assert.equal(attempted.length, BREAKER_TRIP, 'the breaker cuts the run at its threshold'); + assert.ok(attempted.length < many.length, 'and therefore short of the full candidate set'); + // The number that actually matters is REQUESTS, not candidates: each unresolved ЕИК costs the + // full retry budget, so the breaker's real cost is BREAKER_TRIP × TRIES_PER_EIK. Keep that under + // the ~50 at which the register was observed to start returning a sustained 429 — otherwise the + // safety mechanism is itself what trips the block. + assert.equal(h.calls.length, BREAKER_TRIP * TRIES_PER_EIK); + assert.ok( + h.calls.length <= 25, + `a failing run must not spend ${h.calls.length} requests before giving up`, + ); + assert.deepEqual(rows(c.dbFile), [], 'a wall of 5xx records nothing'); + } finally { + c.cleanup(); + } +}); + +// ── retention (ADR-0033 decision 5: „a purge step in the same job") ─────────── +test('the job purges past-retention deeds, and does so even when a 429 ends the run', async () => { + // Retention is an obligation about other people's data, not a reward for a clean run: the paths that + // leave early — a 429, a tripped breaker — are exactly the ones where a naive placement after the + // fetch loop would skip it and let third-party names sit on disk indefinitely. + for (const [label, route] of [ + ['clean run', ok(A)], + ['429 stops the run', status(429)], + ]) { + const c = ctx(); + try { + // An old deed with its raw file — well past the 35-day window at the harness's fixed clock. + fs.mkdirSync(c.rawDir, { recursive: true }); + fs.writeFileSync(path.join(c.rawDir, `${C}.json`), '{"owner":"ТРЕТО ЛИЦЕ"}'); + const db = new DatabaseSync(c.dbFile); + db.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT); + INSERT INTO deeds (eik,status,fetched_at,raw_path) VALUES ('${C}','fetched','2026-01-01T00:00:00Z','${C}.json');`); + db.close(); + + await harness(c, [A], { [A]: route }).promise; + + assert.equal( + fs.existsSync(path.join(c.rawDir, `${C}.json`)), + false, + `${label}: the past-retention raw deed must be deleted`, + ); + assert.equal( + rows(c.dbFile).some((r) => r.eik === C), + false, + `${label}: its index row must go with it`, + ); + } finally { + c.cleanup(); + } + } +}); + +test('the purge leaves in-window deeds alone — it is a privacy rail, not a cache eviction', async () => { + // If it evicted live cache, every run would re-request deeds it already holds, which is precisely + // the volume against the register the pacing exists to avoid. + const c = ctx(); + try { + await harness(c, [A], { [A]: ok(A) }).promise; + assert.equal(rows(c.dbFile).length, 1, 'the deed just fetched must survive its own job'); + assert.equal(fs.existsSync(path.join(c.rawDir, `${A}.json`)), true); + } finally { + c.cleanup(); + } +}); diff --git a/scripts/tr/paths.mjs b/scripts/tr/paths.mjs new file mode 100644 index 000000000..e325d2934 --- /dev/null +++ b/scripts/tr/paths.mjs @@ -0,0 +1,54 @@ +// Paths and path sanitizers for the Търговски регистър leg (issue #279, ADR-0033). +// +// Everything this leg writes lives under scratch/tr/, git-ignored, behind the same refuse-to-run rail +// the CACBG crawl uses — a deed carries third-party personal data (owner and manager names, the +// company's street address), so it is ADR-0010 decision 6 territory, extended by ADR-0033 to a second +// source with a stated retention. +// +// scratch/tr/deeds/.json raw response, atomic write +// scratch/tr/tr-cache.sqlite the index — ЕИК, dates, codes, verdicts. NO names. +// +// The constants below are only DEFAULTS for the CLI. Every function that touches the filesystem takes +// its path explicitly, so tests drive temp directories without mutating process state. + +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { assertScratchIgnored } from '../cacbg/guard.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); + +export const TR_SCRATCH = path.join(ROOT, 'scratch', 'tr'); +export const TR_RAW = path.join(TR_SCRATCH, 'deeds'); +export const TR_DB = path.join(TR_SCRATCH, 'tr-cache.sqlite'); + +/** Refuse to run unless scratch/tr is git-ignored. Call before the first fetch. */ +export function assertTrScratchIgnored() { + assertScratchIgnored('tr'); +} + +// A bare ЕИК: 9 or 13 digits, nothing else. Deliberately NOT `path.basename`-normalised — an ЕИК that +// needed normalising did not come from where we think it did, and silently repairing it is how you end +// up fetching a different company's deed. +const EIK_SHAPE = /^(?:\d{9}|\d{13})$/; + +/** + * Sanitize an ЕИК before it becomes a path segment or a URL segment. + * + * Returns the value VERBATIM — as a string, always. Bulgarian public bodies carry codes of exactly the + * `000…` shape, so a numeric round-trip anywhere on this path silently rewrites the identifier and the + * crawler fetches somebody else's deed (R8). + * + * Shape only. Whether the code's CHECKSUM is valid is a different question with a different remedy, + * answered by eik.mjs — conflating the two would report a real-but-invalid code as a path attack. + * @param {unknown} eik @returns {string} + */ +export function safeEik(eik) { + const s = String(eik ?? ''); + if (!EIK_SHAPE.test(s)) throw new Error(`unsafe ЕИК: ${JSON.stringify(eik)}`); + return s; +} + +/** Absolute path of the cached raw deed for an ЕИК, under `rawDir` (default TR_RAW). */ +export function deedPath(eik, rawDir = TR_RAW) { + return path.join(rawDir, `${safeEik(eik)}.json`); +} diff --git a/scripts/tr/paths.test.mjs b/scripts/tr/paths.test.mjs new file mode 100644 index 000000000..3401260d6 --- /dev/null +++ b/scripts/tr/paths.test.mjs @@ -0,0 +1,64 @@ +// node:test — path sanitizers and the refuse-to-run rail for the Trade Register leg. +// +// The deed cache holds third-party personal data (owner/manager names, company addresses), so the +// same rail the CACBG crawl runs behind applies here: everything is written under scratch/, and +// scratch/ must be git-ignored, asserted BEFORE any fetch. ADR-0010 decision 6 as extended by ADR-0033. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { TR_SCRATCH, TR_RAW, TR_DB, safeEik, assertTrScratchIgnored } from './paths.mjs'; +import { assertScratchIgnored } from '../cacbg/guard.mjs'; + +test('the TR scratch tree sits under scratch/ and is git-ignored in this repo', () => { + assert.ok(TR_SCRATCH.split(path.sep).includes('scratch'), TR_SCRATCH); + assert.ok(TR_RAW.startsWith(TR_SCRATCH)); + assert.ok(TR_DB.startsWith(TR_SCRATCH)); + assert.doesNotThrow(() => assertTrScratchIgnored()); +}); + +test('the guard is the CACBG one generalized, not a second copy', () => { + // A duplicated safety rail drifts from the original. assertScratchIgnored now takes the + // subdirectory, and its existing no-argument callers keep working unchanged. + assert.doesNotThrow(() => assertScratchIgnored()); + assert.doesNotThrow(() => assertScratchIgnored('tr')); + // .gitignore ignores `scratch/` WHOLESALE, so every subdirectory of it passes — an unknown name is + // not a way to reach the failure branch. Escape scratch/ instead (path.join normalises this to + // `docs/.probe`, which is tracked) to prove the guard still refuses when the target is not ignored. + assert.throws(() => assertScratchIgnored(path.join('..', 'docs')), /REFUSE TO RUN/); +}); + +test('safeEik accepts only a bare 9/13-digit code and returns it verbatim', () => { + assert.equal(safeEik('115536179'), '115536179'); + assert.equal(safeEik('1155361790001'), '1155361790001'); + // Leading zeros survive — public bodies are exactly this shape, and losing them fetches a + // DIFFERENT company's deed. + assert.equal(safeEik('000696327'), '000696327'); +}); + +test('safeEik refuses anything that could leave the intended path or URL', () => { + for (const bad of [ + '', + null, + undefined, + '..', + '../115536179', + '/115536179', + '115536179/../x', + '115536179?x=1', + '115536179#f', + '11553617x', + '11553617', + '1155361790', + 'ЕИК 115536179', + ' 115536179', + '115536179 ', + ]) { + assert.throws(() => safeEik(bad), /unsafe/i, JSON.stringify(bad)); + } +}); + +test('safeEik does not validate the CHECKSUM — that is a separate question', () => { + // Path safety and identity validity are different concerns: a shape-valid but checksum-invalid code + // must still be rejectable by the caller with a specific reason, not conflated into „unsafe path". + assert.equal(safeEik('115536170'), '115536170'); +});