diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index dbee8724..f01f1b8a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -24,7 +24,7 @@ ## Чеклист -- [ ] Комитите следват [conventional commits](https://www.conventionalcommits.org) и **нямат** `Co-Authored-By:` trailer +- [ ] Комитите следват [conventional commits](https://www.conventionalcommits.org) и **нямат** `Co-Authored-By:` trailer, който сочи към агент (Claude Code, Codex, Cursor, Copilot). Трейлъри с **хора** са наред и не се махат — те са начинът заслугата на сътрудника да оцелее при squash - [ ] PR-ът е с **един логически обхват** и е от форк към `midt-bg/sigma:main` - [ ] `pnpm typecheck` минава - [ ] `pnpm test` (поне за засегнатите пакети) минава diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cebb18a..b4a7f739 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: container: image: semgrep/semgrep:1.170.0@sha256:c98f8829eea377274ee4b10656458b078b88232469b2ff913f091c2317347c9d steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Static analysis (semgrep) run: semgrep scan --config p/security-audit --config p/secrets --config p/typescript --metrics=off --error @@ -55,7 +55,7 @@ jobs: tar -xzf "$RUNNER_TEMP/gitleaks.tar.gz" -C "$RUNNER_TEMP" gitleaks "$RUNNER_TEMP/gitleaks" dir . --redact --no-banner - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2c3f42ae..2ed7a02a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -83,7 +83,7 @@ jobs: SIGMA_VECTORIZE_NAME: ${{ vars.SIGMA_VECTORIZE_NAME }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 @@ -145,6 +145,26 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0003_related_persons_foundation.sql + # 0009 attaches the Trade Register evidence seal (#279, ADR-0033). Both migrations are + # applied by name here because wrangler's migration ledger is empty on this D1 (the base + # schema was created out-of-band), so `d1 migrations apply` would collide on 0000. + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0009_interest_link_evidence.sql + # 0010 owns the publishing-gate enforcement for EVERY database (#279 §2) — fresh and already + # deployed alike. It is deliberately NOT declared in 0003: that migration is already applied + # everywhere, `CREATE TABLE IF NOT EXISTS` never revisits an existing table, and the ship step + # wipes ROWS, not definitions — so an in-place CHECK would exist only on databases built after + # the edit, and be absent on exactly the database that serves the site. Enforced with BEFORE + # INSERT/UPDATE triggers, NOT a table rebuild: the create-copy-drop-rename route would expose + # the foreign keys and strip every evidence seal, emptying the public surface until the next + # monthly run (see the migration header). `control_hash` stays NULLABLE by design — the register + # omits it on some declarations; the natural-key index folds those with COALESCE instead. + # Idempotent: every statement is `IF NOT EXISTS` over a converging definition, so re-applying it + # on each deploy is a no-op. Verified by three consecutive applications, gate still enforcing. + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0010_publishing_gate_constraints.sql # The base schema was created out-of-band with `d1 execute --file`, so wrangler's migration # ledger is empty and `d1 migrations apply` would collide on 0000. Probe the actual table # instead. SQLite has no `ADD COLUMN IF NOT EXISTS`; a completion-marker table is created only @@ -214,6 +234,78 @@ jobs: ;; esac + # #305 additive columns (migrations 0006/0007) must exist BEFORE the Worker serves: the contract-page + # query reads amendments.value_restated/value_suspect, and promote-amendments/refresh-slice also INSERT + # value_treatment — so all three must be present or the read AND the ETL write fail. Same rationale as + # the currency step above: the migration ledger is empty, so `d1 migrations apply` would collide on + # 0000. Probe the actual table and ALTER only the missing columns (SQLite has no ADD COLUMN IF NOT + # EXISTS). These are pure additions with safe defaults (INTEGER NOT NULL DEFAULT 0 / nullable TEXT), so + # no backfill or completion marker is needed — an ALTER populates every existing row. Malformed + # responses are fatal. This also makes 0006/0007 replay-safe when they were applied out-of-ledger. + # #306 folds in here rather than adding a second step of its own (as PR #308's own note asked): + # refresh-slice.sql and promote-amendments.sql write contract_number_raw + link_method into served + # `amendments`, so the first cron after release would crash on the missing columns. One probe, one + # mechanism — a second hand-written step is another chance for the alias bug #310 had to fix. + - name: Apply amendment restated/suspect + provenance columns + if: steps.guard.outputs.ok == 'true' + run: | + node scripts/wrangler-render.mjs apps/web/wrangler.jsonc + # Each alias below MUST be the column name itself: read_flag looks the row up by the very string + # ensure_column is called with. An alias that merely describes the column (has_restated) makes + # `Object.hasOwn(row, key)` false for every column, which the probe treats as an unreadable answer + # and turns into a hard failure — so the step could never succeed, on any database. + schema_json="$(pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes --json \ + --command "SELECT + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_restated') AS value_restated, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_treatment') AS value_treatment, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'value_suspect') AS value_suspect, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'contract_number_raw') AS contract_number_raw, + (SELECT COUNT(*) FROM pragma_table_info('amendments') WHERE name = 'link_method') AS link_method")" + + read_flag() { + printf '%s' "$schema_json" | node -e ' + const fs = require("fs"); + let payload; + try { + payload = JSON.parse(fs.readFileSync(0, "utf8")); + } catch { + process.exit(2); + } + const result = Array.isArray(payload) ? payload[0] : payload; + const row = result && Array.isArray(result.results) ? result.results[0] : null; + const key = process.argv[1]; + if (!row || !Object.hasOwn(row, key)) process.exit(2); + process.exit(Number(row[key]) === 1 ? 0 : 1); + ' "$1" + } + + add_column() { + echo "amendments.$1 missing; adding it." + pnpm --filter @sigma/web exec wrangler d1 execute "${SIGMA_D1_NAME:-sigma}" \ + --config wrangler.deploy.jsonc --remote --yes \ + --command "ALTER TABLE amendments ADD COLUMN $2" + } + + ensure_column() { + set +e + read_flag "$1" + status="$?" + set -e + case "$status" in + 0) echo "amendments.$1 already exists." ;; + 1) add_column "$1" "$2" ;; + *) echo "::error::Could not determine whether amendments.$1 exists."; exit 1 ;; + esac + } + + ensure_column value_restated "value_restated INTEGER NOT NULL DEFAULT 0" + ensure_column value_treatment "value_treatment TEXT" + ensure_column value_suspect "value_suspect INTEGER NOT NULL DEFAULT 0" + # #306 provenance: NULL on both = the row linked by contract_number directly (or is unlinked). + ensure_column contract_number_raw "contract_number_raw TEXT" + ensure_column link_method "link_method TEXT" + # `run deploy`, not `deploy` — bare `pnpm deploy` is a pnpm built-in, not our package script. - name: Deploy explorer (sigma) if: steps.guard.outputs.ok == 'true' diff --git a/.github/workflows/related-persons-data.yml b/.github/workflows/related-persons-data.yml index deb24884..2beb2706 100644 --- a/.github/workflows/related-persons-data.yml +++ b/.github/workflows/related-persons-data.yml @@ -16,17 +16,63 @@ name: Related-persons data foundation (build + ship) # XML (fetch.mjs skips files already on disk) — it fills gaps, not a from-scratch rebuild unless the cache is cleared. on: + # MONTHLY, and deliberately on the EXISTING workflow rather than a second one. #279 §9 asks for the + # publishing decisions to run on a cadence; duplicating the job would create a second ship path with + # its own copy of the credential guards, the D1-target guard and the ship floor — and the copy is the + # one nobody exercises. + # + # WHY MONTHLY AND NOT DAILY, which is what §9 and ADR-0033 originally describe. The decision and its + # rejected alternatives are recorded in ADR-0034, which supersedes ADR-0033's cadence line — this + # comment is the operational summary, not the record, so a reader who disagrees has somewhere to argue. + # A decision cannot be recomputed + # without the raw deeds — evidenceVerdict's strongest rung matches the declarant's name against the + # register's own text — and the raw deeds must not be persisted between runs: they carry the names and + # addresses of co-owners and managers who hold no public office, and an Actions cache entry lives on + # GitHub's storage under a restore-keys chain where ADR-0033 decision 5's 35-day retention cannot reach + # it. So the deeds live and die with one runner, and every run that decides must also crawl. A daily + # decision would therefore mean ~400 daily requests against somebody else's register, which is what + # spec §3.3 exists to prevent — so the cadence follows the deeds, not the other way round. + # + # Restoring a daily decision needs the crawl to emit the per-(link, ЕИК) match verdict, so only + # booleans cross a run boundary and nothing has to hold a name. That is a design change, not a + # schedule change, and it is deliberately not folded in here. + # + # A scheduled run takes the same path as a manual one, with two differences: + # • it targets STAGING (the environment default below). Production stays manual, so the + # assertD1TargetAuthorized guard keeps a prod write a deliberate act. + # • `full_crawl` is empty and therefore false, so it never crawls the CACBG corpus — it re-runs the + # decisions over the cached corpus, which is exactly what #279 §9 means by a pure, zero-network + # decision. Registry lookups are the one exception and run from inside this job on the 1st of the + # month (see „Decide whether to hit the register"): they cannot be a separate workflow, because the + # raw deeds they fetch carry third-party names and must not be persisted between runners. + # If the corpus cache has been evicted, extract yields nothing, the ship floor refuses, and the run + # fails loudly rather than wiping the surface. + schedule: + # 03:00 UTC on the 1st. See the note above: the decision and the crawl are one pass, so the decision + # cadence is the crawl cadence — #279 §9's monthly registry rhythm. + - cron: '0 3 1 * *' workflow_dispatch: inputs: environment: description: Target D1 environment type: choice - options: [dev, staging, production] - default: dev + # No `dev` environment exists — every fallback in this file is `|| 'staging'` and the note above + # says the job targets staging. A `dev` default sent a manual dispatch to an environment with no + # secrets, so it failed at the credential guard rather than running where it was meant to. + options: [staging, production] + default: staging full_crawl: description: Crawl the CACBG corpus (slow). RESUMES from the cached raw XML — fetch.mjs skips files already on disk, so this fills gaps rather than re-fetching from scratch (clear the cache for a true rebuild). Off = extract from the existing raw cache without crawling. type: boolean default: false + tr_max_age_days: + description: Re-fetch deeds older than this many days (default 30). Use a large value to force a full refresh. + type: string + default: '30' + tr_limit: + description: Stop after this many registry lookups (blank = all pending). Bound a first run while watching for a 429. + type: string + default: '' # Least-privilege: this workflow only reads the repo; it authenticates to Cloudflare with a scoped token. permissions: @@ -34,19 +80,20 @@ permissions: concurrency: # Never let two data-foundation writes to the same environment overlap; do not cancel one in flight. - group: related-persons-data-${{ inputs.environment }} + group: related-persons-data-${{ inputs.environment || 'staging' }} cancel-in-progress: false jobs: build-and-ship: runs-on: ubuntu-latest # A from-scratch corpus crawl of the whole register is the long pole (all year-folders, politely - # throttled) and must fit in ONE job with extract/resolve/audit/ship after it — 120 was too tight and - # timed out mid-crawl. 300 leaves headroom under GitHub's 6-hour hard cap. A run past this is wedged + # throttled) and must fit in ONE job with the registry lookups, extract/resolve/audit/ship after it — + # 120 was too tight and timed out mid-crawl. The Търговски регистър pass adds ~20 minutes on top + # (~400 candidates at 1 request / 3 s). 300 leaves headroom under GitHub's 6-hour hard cap. A run past this is wedged # (stalled gov-server I/O) — fail rather than burn the slot. The raw corpus is cached (below) so a # re-run resumes instead of re-crawling from scratch. timeout-minutes: 300 - environment: ${{ inputs.environment }} + environment: ${{ inputs.environment || 'staging' }} env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} @@ -54,16 +101,16 @@ jobs: SIGMA_D1_NAME: ${{ vars.SIGMA_D1_NAME }} # The DECLARED intended environment — the independent anchor the ship guard checks the (name, id) pair # against (T48), so a consistent-but-wrong pair can't wipe the wrong DB. See assertD1TargetAuthorized. - SIGMA_SHIP_ENV: ${{ inputs.environment }} + SIGMA_SHIP_ENV: ${{ inputs.environment || 'staging' }} # Suppression (takedown) list keying — load.mjs fingerprints link-suppressions.jsonl under this salt and # refuses to build if the list is non-empty but the salt is unset (fail-closed, ADR-0031). Without this # the first suppression entry would break every ship; the key version guards a coordinated salt rotation. SUPPRESSION_SALT: ${{ secrets.SUPPRESSION_SALT }} SUPPRESSION_KEY_VERSION: ${{ vars.SUPPRESSION_KEY_VERSION || '1' }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24 # node:sqlite (DatabaseSync) + native TS type-stripping, no flags cache: pnpm @@ -87,20 +134,25 @@ jobs: # live in each Environment's SIGMA_D1_NAME var; the ship step additionally requires SIGMA_SHIP_ENV to match # and PROVES the name↔SIGMA_D1_ID pair against Cloudflare before deleting — assertD1TargetAuthorized.) # The slot set is kept in lockstep with PRODUCTION_SLOTS in scripts/ship-related-persons.mjs (#226). + # $SIGMA_SHIP_ENV, never `${{ inputs.environment }}` inline. `type: choice` is enforced by the UI + # only — a POST to .../dispatches can pass an arbitrary string, which an inline expression pastes + # into the shell before `[` ever evaluates it. This block is also the one that decides whether the + # production guard fires, so a crafted value could both inject and walk past the guard it was + # meant to trip. The env binding above passes the same value as data, where it cannot be code. - name: Guard — SIGMA_D1_NAME matches the environment run: | case "${SIGMA_D1_NAME}" in sigma-blue|sigma-green) IS_PROD_SLOT=1 ;; *) IS_PROD_SLOT=0 ;; esac - if [ "${{ inputs.environment }}" = "production" ]; then + if [ "${SIGMA_SHIP_ENV}" = "production" ]; then if [ "$IS_PROD_SLOT" != "1" ]; then echo "::error::production requires SIGMA_D1_NAME to be a prod slot (sigma-blue|sigma-green), got '${SIGMA_D1_NAME}' — refusing to ship." exit 1 fi else if [ "$IS_PROD_SLOT" = "1" ]; then - echo "::error::SIGMA_D1_NAME='${SIGMA_D1_NAME}' (a PRODUCTION slot) on a '${{ inputs.environment }}' run — a rebuild would WIPE prod. Set a non-production SIGMA_D1_NAME variable for this Environment." + echo "::error::SIGMA_D1_NAME='${SIGMA_D1_NAME}' (a PRODUCTION slot) on a '${SIGMA_SHIP_ENV}' run — a rebuild would WIPE prod. Set a non-production SIGMA_D1_NAME variable for this Environment." exit 1 fi fi @@ -112,7 +164,7 @@ jobs: # (Cross-run only — a hard mid-crawl job timeout can't save; durable incremental persistence is the # sigma-etl Worker's R2 job per ADR-0006. GitHub's 10 GB repo cache is the ceiling here.) - name: Restore raw CACBG corpus cache - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: scratch/cacbg/raw key: cacbg-raw-${{ github.run_id }} @@ -129,7 +181,7 @@ jobs: # run resumes from here. Skipped on full_crawl=false runs (nothing new was fetched). - name: Save raw CACBG corpus cache if: ${{ always() && inputs.full_crawl }} - uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: scratch/cacbg/raw key: cacbg-raw-${{ github.run_id }} @@ -151,11 +203,29 @@ jobs: work="$GITHUB_WORKSPACE/data/work" mkdir -p "$work" rm -f "$work/eop.sql" "$work/backfill.sqlite" + # interest_links + interest_link_evidence come along too, and NOT for the resolver's benefit — + # load.mjs drops and rebuilds them. They are the input to the monotonicity gate: load.mjs + # exports the currently-published set BEFORE the wipe, and audit.mjs fails the run on a link + # that was published last time, under an unchanged rules_version, and is not published now. + # Without them the work DB starts with no interest_links at all, the pre-wipe export reads + # „no such table", the snapshot is [] on every run, and the gate protects nothing. + # They are absent on a first run (0003/0006 not yet applied), which is why they are exported + # separately and tolerated as missing rather than folded into the parity loop below. pnpm --filter @sigma/web exec wrangler d1 export "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote -y \ --table bidders --table contracts --table tenders --table authorities \ --output "$work/eop.sql" sqlite3 "$work/backfill.sqlite" < "$work/eop.sql" + if pnpm --filter @sigma/web exec wrangler d1 export "$SIGMA_D1_NAME" \ + --config wrangler.deploy.jsonc --remote -y \ + --table interest_links --table interest_link_evidence \ + --output "$work/prior-links.sql"; then + sqlite3 "$work/backfill.sqlite" < "$work/prior-links.sql" + echo " prior surface: $(sqlite3 "$work/backfill.sqlite" \ + "SELECT COUNT(*) FROM interest_links WHERE status='published';") published link(s)" + else + echo "::notice::No prior interest_links on this D1 (first run) — the monotonicity gate has no baseline yet." + fi for t in bidders contracts tenders authorities; do local_n=$(sqlite3 "$work/backfill.sqlite" "SELECT COUNT(*) FROM $t;") remote_json=$(pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ @@ -169,9 +239,60 @@ jobs: fi done + # ── Trade Register lookups ──────────────────────────────────────────────────────────────────── + # In THIS job, deliberately, and not in a workflow of their own. The decision run needs the raw + # deeds (evidenceVerdict matches declarant names against the register's own text, which is exactly + # the data the index refuses to store), and those deeds carry the names and addresses of co-owners + # and managers who hold no public office. Handing them between runners means persisting them — + # an Actions cache entry survives on GitHub's storage under a restore-keys chain, where the 35-day + # retention ADR-0033 decision 5 promises cannot reach it. Same runner, same job, deleted with it. + # + # The previous split also could not work on its own terms: the crawl job gated on a candidate list + # only the decision job produced, the decision job refused without a cache only the crawl job + # produced, and neither persisted its half. Both were permanently green no-ops. + - name: Emit the candidate ЕИК list for the crawler + run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/load.mjs --emit-candidates + + # ALWAYS, on every run that reaches here — not conditionally. The deeds do not survive the runner, + # so „decide without crawling" is not a state this job can be in: load.mjs's coverage gate below + # would refuse the empty cache and the run would fail. Skipping the crawl is only meaningful once + # the crawl emits verdicts rather than deeds (see the note on the schedule above). + # + # Pace and refusals live in scripts/tr/fetch-deeds.mjs and are not configurable here on purpose: + # 1 request / 3 s, sequential, a closed candidate set, and a 429 ends the run without marking + # anything. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the limiter is + # the operator's only way to state a rate preference, so we do not tune around it. + - name: Refresh the Trade Register deed cache + env: + MAX_AGE: ${{ inputs.tr_max_age_days || '30' }} + LIMIT: ${{ inputs.tr_limit }} + run: | + set -euo pipefail + ARGS=(--eiks-file scratch/cacbg/staging/candidate-eiks.txt --max-age-days "$MAX_AGE") + [ -n "$LIMIT" ] && ARGS+=(--limit "$LIMIT") + set +e + node scripts/tr/fetch-deeds.mjs "${ARGS[@]}" + code=$? + set -e + # Exit 2 is the rate limiter, and it is NOT a build failure: the run stopped politely, marked + # nothing, and the partial cache is valid. It is not a licence to publish either — load.mjs's + # coverage gate below refuses a partial cache on its own, which is where that decision belongs. + if [ "$code" -eq 2 ]; then + echo "::warning::The register rate-limited us; the crawl stopped and the cache is resumable." + exit 0 + fi + exit "$code" + - name: Resolve → build domain in a work sqlite (libel gate must pass) run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/load.mjs + # The raw deeds are third-party personal data and exist only to produce booleans. They are deleted + # with the runner regardless; deleting them explicitly means a failure AFTER this point cannot + # leave them sitting in a job artifact or a debug upload someone adds later. + - name: Delete the raw deeds + if: always() + run: rm -rf scratch/tr/deeds + - name: Audit published links (fails the run on any libel-invariant violation) run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/audit.mjs @@ -185,6 +306,20 @@ jobs: pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ --config wrangler.deploy.jsonc --remote --yes \ --file ../../packages/db/migrations/0003_related_persons_foundation.sql + # 0006 attaches the Trade Register evidence seal (#279, ADR-0033). Both migrations are + # applied by name here because wrangler's migration ledger is empty on this D1 (the base + # schema was created out-of-band), so `d1 migrations apply` would collide on 0000. + pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0009_interest_link_evidence.sql + # 0007 retrofits the publishing-gate CHECK constraints and control_hash NOT NULL onto an + # ALREADY-PROVISIONED D1 (#279 §2). 0003 declares them now, but `CREATE TABLE IF NOT EXISTS` + # never revisits an existing table and the ship step wipes ROWS, not definitions — so without + # this a live database keeps the unconstrained shape no matter how often the data reloads. + # Rebuild-based and therefore idempotent: re-applying copies an already-constrained table. + pnpm --filter @sigma/web exec wrangler d1 execute "$SIGMA_D1_NAME" \ + --config wrangler.deploy.jsonc --remote --yes \ + --file ../../packages/db/migrations/0010_publishing_gate_constraints.sql - name: Ship свързани-лица tables → D1 # ship-related-persons.mjs shells out to `wrangler d1 execute` with cwd=apps/web. Two things it @@ -198,7 +333,14 @@ jobs: node scripts/wrangler-render.mjs apps/web/wrangler.jsonc cp apps/web/wrangler.deploy.jsonc apps/web/wrangler.jsonc export PATH="$GITHUB_WORKSPACE/apps/web/node_modules/.bin:$GITHUB_WORKSPACE/node_modules/.bin:$PATH" - node scripts/ship-related-persons.mjs --work-db data/work/backfill.sqlite --remote --yes + # --min-links is passed EXPLICITLY, not left at its default of 50. Under #279 the surface is + # measured at ~329 links, so 50 no longer protects anything: a partially-restored Trade + # Register cache yields roughly 80 published links, which clears a floor of 50, ships a + # decimated surface and wipes the rest. The loader's coverage gate is the first defence and + # this is the second (ADR-0033 decision 7). Lower it deliberately, in a PR, if a genuinely + # smaller surface is ever expected. + node scripts/ship-related-persons.mjs --work-db data/work/backfill.sqlite --remote --yes \ + --min-links 250 - name: Reindex search projection → officials searchable (same batch the cron runs) # ship writes ONLY the свързани-лица domain tables (interest_links, persons, …), NOT search_index. diff --git a/.github/workflows/scripts-test.yml b/.github/workflows/scripts-test.yml index 7ee3ef4c..61deb9f7 100644 --- a/.github/workflows/scripts-test.yml +++ b/.github/workflows/scripts-test.yml @@ -20,31 +20,36 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 # node:test only — the scripts under test import nothing outside node built-ins and each other, - # so no `pnpm install` is required. + # so no `pnpm install` is required. The one external dependency is the `sqlite3` binary, which + # ship-e2e.test.mjs drives as the real target its fake wrangler writes into; it ships with the + # ubuntu-latest image, and the test fails loudly (never skips) if it is ever absent. - run: node --test scripts/*.test.mjs - # свързани-лица CACBG pipeline tests (parse/classify/load/audit/tr-census/extract). These use - # node:sqlite (DatabaseSync) and import the shared companyNameKey .ts via the register-ts resolve - # hook, so they need Node 24 (node:sqlite + native TS type-stripping, no experimental flag) rather - # than the Node 22 above. The libel-critical resolution logic gates merges here. + # свързани-лица pipeline tests — the CACBG leg (parse/classify/load/audit/extract) and the + # Търговски регистър leg (scripts/tr: ЕИК checksum, HTTP client, deed cache). These use node:sqlite + # (DatabaseSync) and import the shared companyNameKey .ts via the register-ts resolve hook, so they + # need Node 24 (node:sqlite + native TS type-stripping, no experimental flag) rather than the Node 22 + # above. The libel-critical resolution logic gates merges here. cacbg: runs-on: ubuntu-latest timeout-minutes: 10 permissions: contents: read steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24 cache: pnpm # parse.mjs imports fast-xml-parser (a workspace dep), so this job needs node_modules — unlike # the plain-node scripts above. Install before the scraper tests run. - run: pnpm install --frozen-lockfile - - run: node --import ./scripts/cacbg/register-ts.mjs --test scripts/cacbg/*.test.mjs + # Both legs, or the glob silently excludes a whole module: scripts/tr/*.test.mjs matched NO job + # until this line existed, so every test in it would have been decorative. + - run: node --import ./scripts/cacbg/register-ts.mjs --test scripts/cacbg/*.test.mjs scripts/tr/*.test.mjs diff --git a/AGENTS.md b/AGENTS.md index 8ed755cf..545ae557 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,7 @@ No `develop`, no `staging`. Maintainers with write access work on short-lived fe - Use [conventional commits](https://www.conventionalcommits.org): `(): `. Types: `feat`, `fix`, `docs`, `refactor`, `test`, `build`, `ci`, `chore`, `perf`, `style`. Subject is lowercase imperative, no trailing period. - Use the `/smart-commit` and `/suggest-commit` skills when drafting messages. They produce the canonical format for this repo. -- **Never include `Co-Authored-By:` trailers.** Keep the history clean; CI may grep for this. +- **Never credit a coding agent in a `Co-Authored-By:` trailer** (Claude Code, Codex, Cursor, Copilot). Trailers naming **people** are fine and must not be stripped — GitHub generates them from the PR's commit authors on squash, and they are what keeps a contributor's credit on `main`, since the squash commit's own author is always the PR opener. - Small, focused commits are encouraged. Commit as you go — not all at the end. Easier to review and revert. Don't mix unrelated changes in one commit. ## Pull requests diff --git a/apps/etl/src/eop.test.ts b/apps/etl/src/eop.test.ts index a96a50df..851ff51f 100644 --- a/apps/etl/src/eop.test.ts +++ b/apps/etl/src/eop.test.ts @@ -1,5 +1,28 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { computeWorkerCatchupPlan, listBucketForDay } from './eop'; +import { computeWorkerCatchupPlan, listBucketForDay, stageBaseFromBucket } from './eop'; + +/** + * A response whose stream is deliberately left open, so `cancel()` on the underlying source really + * fires. A stream that is enqueued *and closed* would report success without proving anything: the + * spec short-circuits `cancel()` once a stream is already closed. + */ +function openBodyResponse(init: ResponseInit & { url?: string }): { + response: Response; + cancelled: () => boolean; +} { + let cancelled = false; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('')); + }, + cancel() { + cancelled = true; + }, + }); + const response = new Response(body, init); + if (init.url) Object.defineProperty(response, 'url', { value: init.url }); + return { response, cancelled: () => cancelled }; +} function fakeDbFromFreshness(maxLoadedDate: string): D1Database { const db = { @@ -53,3 +76,96 @@ describe('EOP fetch host allowlist', () => { ); }); }); + +describe('EOP responses the ingest walks away from', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('releases the body of a missing bucket instead of leaving the stream open', async () => { + const { response, cancelled } = openBodyResponse({ status: 403 }); + vi.stubGlobal('fetch', vi.fn(async () => response) as unknown as typeof fetch); + + await expect(listBucketForDay('2026-06-01')).resolves.toBeNull(); + expect(cancelled()).toBe(true); + }); + + it('releases the body of a failed bucket listing before throwing', async () => { + const { response, cancelled } = openBodyResponse({ status: 500 }); + vi.stubGlobal('fetch', vi.fn(async () => response) as unknown as typeof fetch); + + await expect(listBucketForDay('2026-06-01')).rejects.toThrow(/bucket 2026-06-01: HTTP 500/); + expect(cancelled()).toBe(true); + }); + + it('releases the body of a blocked redirect before throwing', async () => { + const { response, cancelled } = openBodyResponse({ + status: 200, + url: 'https://evil.example/open-data-2026-06-01/', + }); + vi.stubGlobal('fetch', vi.fn(async () => response) as unknown as typeof fetch); + + await expect(listBucketForDay('2026-06-01')).rejects.toThrow(/blocked redirected EOP fetch/); + expect(cancelled()).toBe(true); + }); + + // The one drain site the first cut of this change left uncovered: bypassing it kept the whole suite + // green. A blocked redirect on an OBJECT fetch is a different call path from the bucket listing. + it('releases the body of a redirected object fetch before throwing', async () => { + const { response, cancelled } = openBodyResponse({ + status: 200, + url: 'https://evil.example/open-data-2026-06-01/contracts.json', + }); + vi.stubGlobal('fetch', vi.fn(async () => response) as unknown as typeof fetch); + + await expect( + stageBaseFromBucket( + {} as D1Database, + { + day: '2026-06-01', + bucketUrl: 'https://storage.eop.bg/open-data-2026-06-01/', + keys: { contracts: 'contracts.json' }, + }, + '2026-06-01T00:00:00.000Z', + ), + ).rejects.toThrow(/blocked redirected EOP fetch from storage\.eop\.bg to evil\.example/); + expect(cancelled()).toBe(true); + }); + + // Cancellation must be initiated, never awaited: a stream whose cancel() never settles must not be + // able to wedge the ingest. Before this, `await res.body.cancel()` made listBucketForDay hang forever. + it('does not wait for a cancel that never settles', async () => { + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('x')); + }, + cancel() { + return new Promise(() => {}); // never settles + }, + }); + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(body, { status: 403 })) as unknown as typeof fetch, + ); + + await expect(listBucketForDay('2026-06-01')).resolves.toBeNull(); + }); + + it('releases the body of a failed object fetch before throwing', async () => { + const { response, cancelled } = openBodyResponse({ status: 500 }); + vi.stubGlobal('fetch', vi.fn(async () => response) as unknown as typeof fetch); + + await expect( + stageBaseFromBucket( + {} as D1Database, + { + day: '2026-06-01', + bucketUrl: 'https://storage.eop.bg/open-data-2026-06-01/', + keys: { contracts: 'contracts.json' }, + }, + '2026-06-01T00:00:00.000Z', + ), + ).rejects.toThrow(/HTTP 500/); + expect(cancelled()).toBe(true); + }); +}); diff --git a/apps/etl/src/eop.ts b/apps/etl/src/eop.ts index 96f63195..7a0fffab 100644 --- a/apps/etl/src/eop.ts +++ b/apps/etl/src/eop.ts @@ -70,14 +70,32 @@ const dayUrl = (baseUrl: string, day: string): string => const objectUrl = (bucketUrl: string, key: string): string => `${bucketUrl}${encodeURIComponent(key)}`; -function assertAllowedFinalHost(requestUrl: string, responseUrl: string): void { +function disallowedFinalHost(requestUrl: string, responseUrl: string): string | null { const requested = new URL(requestUrl); const final = new URL(responseUrl || requestUrl); - if (final.host !== requested.host) { - throw new Error(`blocked redirected EOP fetch from ${requested.host} to ${final.host}`); + if (final.host === requested.host) return null; + return `blocked redirected EOP fetch from ${requested.host} to ${final.host}`; +} + +// A response body that is never read keeps its stream open for the rest of the invocation; the +// collector is not a substitute for releasing it. Every path below that walks away from a response +// without reading it - a blocked redirect, a missing bucket, any non-OK status - releases it here. +// Deliberately NOT awaited: cancelling only needs to be INITIATED for the runtime to release the +// stream, and awaiting it would make every caller hostage to a cancel() that never settles, which +// is precisely the failure mode this file exists to reduce. +function discardBody(res: Response): void { + try { + void res.body?.cancel().catch(() => {}); + } catch { + // Already consumed, locked, or errored - there is nothing left to release either way. } } +function releaseAndFail(res: Response, message: string): never { + discardBody(res); + throw new Error(message); +} + function decodeXml(s: string): string { return s .replace(/</g, '<') @@ -175,9 +193,13 @@ export async function listBucketForDay( ): Promise { const bucketUrl = dayUrl(opts.baseUrl ?? DEFAULT_BASE_URL, day); const res = await fetch(bucketUrl); - assertAllowedFinalHost(bucketUrl, res.url); - if (res.status === 403 || res.status === 404) return null; - if (!res.ok) throw new Error(`bucket ${day}: HTTP ${res.status}`); + const blocked = disallowedFinalHost(bucketUrl, res.url); + if (blocked) return releaseAndFail(res, blocked); + if (res.status === 403 || res.status === 404) { + discardBody(res); + return null; + } + if (!res.ok) return releaseAndFail(res, `bucket ${day}: HTTP ${res.status}`); const keys: BucketKeys = {}; for (const key of parseBucketKeys(await res.text())) { @@ -189,8 +211,9 @@ export async function listBucketForDay( async function fetchJson(url: string): Promise { const res = await fetch(url); - assertAllowedFinalHost(url, res.url); - if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`); + const blocked = disallowedFinalHost(url, res.url); + if (blocked) return releaseAndFail(res, blocked); + if (!res.ok) return releaseAndFail(res, `${url}: HTTP ${res.status}`); return res.json(); } diff --git a/apps/etl/src/index.test.ts b/apps/etl/src/index.test.ts index 536ef5ba..c1c62649 100644 --- a/apps/etl/src/index.test.ts +++ b/apps/etl/src/index.test.ts @@ -214,3 +214,57 @@ describe('RefreshWorkflow FX loading (#158)', () => { expect(usd.amount_eur).toBeCloseTo(EXPECTED_USD_EUR, 2); }); }); + +// The Workers runtime logs an error on every *successful* instance of this Workflow, so the absence +// of errors proves nothing about whether the cron actually ran. A finished refresh has to announce +// itself — and, just as importantly, a failed one must not. +describe('RefreshWorkflow completion signal', () => { + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + function captureLogs(): () => Record[] { + const lines: string[] = []; + vi.spyOn(console, 'log').mockImplementation((line: unknown) => { + lines.push(String(line)); + }); + return () => + lines.flatMap((line) => { + try { + return [JSON.parse(line) as Record]; + } catch { + return []; + } + }); + } + + it('announces a finished refresh with the run summary', async () => { + const db = freshServedDb(); + stubFetchRoutes(); + const logs = captureLogs(); + + const result = await runRefresh(makeWorkflow(db)); + + const done = logs().find((e) => e.event === 'etl_refresh_complete'); + expect(done).toBeDefined(); + expect(done?.staged).toBe(result.staged); + expect(done?.derived).toBe(result.derived); + expect(done?.to).toBe(TODAY); + }); + + it('stays silent when the refresh throws', async () => { + const db = freshServedDb(); + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new Error('storage.eop.bg unreachable'); + }) as unknown as typeof fetch, + ); + const logs = captureLogs(); + + await expect(runRefresh(makeWorkflow(db))).rejects.toThrow(/storage\.eop\.bg unreachable/); + + expect(logs().some((e) => e.event === 'etl_refresh_complete')).toBe(false); + }); +}); diff --git a/apps/etl/src/index.ts b/apps/etl/src/index.ts index a3ad9afc..cb352af5 100644 --- a/apps/etl/src/index.ts +++ b/apps/etl/src/index.ts @@ -94,6 +94,12 @@ export class RefreshWorkflow extends WorkflowEntrypoint { let results: Awaited> = []; let staged = 0; let derived = 0; + // The runtime logs an error ("...your Worker's code had hung...") on every *successful* instance + // of this Workflow, at the instant run() returns - measured across runs of 5 and 30 steps, see + // docs/etl.md. "No errors in the dashboard" is therefore not a health signal here, so a refresh + // that actually finished has to say so itself. Logged from the finally, after the staging drop, + // so it only ever claims success for a run that survived its own cleanup. + let outcome: RefreshResult | null = null; try { await step.do('create-transient-staging', async () => @@ -109,7 +115,8 @@ export class RefreshWorkflow extends WorkflowEntrypoint { if (staged === 0) { console.warn(JSON.stringify({ level: 'warn', event: 'etl_zero_ingest', fetchedAt, plan })); - return { ...plan, days: results.length, staged: 0, derived: 0 }; + outcome = { ...plan, days: results.length, staged: 0, derived: 0 }; + return outcome; } // FX rates BEFORE the derive (#158): the CLI paths run scripts/load-fx.mjs first, but this @@ -180,9 +187,13 @@ export class RefreshWorkflow extends WorkflowEntrypoint { } }); - return { ...plan, days: results.length, staged, derived }; + outcome = { ...plan, days: results.length, staged, derived }; + return outcome; } finally { await step.do('drop-transient-staging', async () => dropTransientStaging(this.env.DB)); + if (outcome) { + console.log(JSON.stringify({ level: 'info', event: 'etl_refresh_complete', ...outcome })); + } } } } diff --git a/apps/web/app/components/ConflictCards.tsx b/apps/web/app/components/ConflictCards.tsx index 4bbfeba4..a716c425 100644 --- a/apps/web/app/components/ConflictCards.tsx +++ b/apps/web/app/components/ConflictCards.tsx @@ -20,6 +20,7 @@ import { officialHref, partitionContracts, relationLabel, + registryEvidenceLabel, temporalLabel, } from '../lib/conflicts'; @@ -181,6 +182,26 @@ function ConflictCard({ )} + {/* The Trade Register fact the link's identity rests on (#279, ADR-0033). This is what makes + „every shown link explains itself" true rather than a promise: a reader can open the same act + we read and check it. The wording is careful — the register records a ROLE, it does not + certify the ownership claim, which comes from the official's own declaration. */} +
+
Регистър
+
+ + + {registryEvidenceLabel(l)} + {l.registryEntryDate ? ` · вписване ${l.registryEntryDate}` : ''} + {/* The entry NUMBER, not just its date: a date does not identify a record, and this is + what a reader types to find the same act we read. Rendered only when present — + a seat/ЕИК confirmation cites no act entry, and an empty „№" would read as missing + data rather than as an inapplicable field. */} + {l.registryEntryNumber ? ` · № ${l.registryEntryNumber}` : ''} + {l.registryLookupDate ? ` · справка ${l.registryLookupDate}` : ''} + +
+
{l.contractCount > 0 && ( diff --git a/apps/web/app/lib/assistant/describe-schema.ts b/apps/web/app/lib/assistant/describe-schema.ts index 6157b382..8e887417 100644 --- a/apps/web/app/lib/assistant/describe-schema.ts +++ b/apps/web/app/lib/assistant/describe-schema.ts @@ -10,11 +10,13 @@ export const DATA_TRAPS: string[] = [ 'Парични агрегати: СУМИРАЙ САМО `contracts.amount_eur` (каноничен EUR, безопасен за сумиране). ' + 'НИКОГА не сумирай `contracts.amount` — то е „както е записано" в смесена валута (`currency`), само за показване.', 'Канонична база за всяка парична сума: `contracts.amount_eur IS NOT NULL`. НЕ филтрирай по ' + - '`value_flag`: включи `ok`, `review`, `annex_suspect`, `value_low` и поправените `value_suspect` редове.', + '`value_flag`: включи `ok`, `review`, `annex_suspect`, `annex_total_suspect`, `value_low` и ' + + 'поправените `value_suspect` редове.', '`amount_eur IS NULL` означава, че няма използваема EUR стойност (например `value_suspect` без ' + 'прогноза за поправка или чужда валута без FX курс); само тези редове се изключват от парични суми.', - '`value_flag` ∈ {ok, review, annex_suspect, value_suspect, value_low} мени значението на стойността ' + - 'на реда, но не и каноничната база; `date_flag` ∈ {ok, signed_after_publication} е вердикт за датата.', + '`value_flag` ∈ {ok, review, annex_suspect, annex_total_suspect, value_suspect, value_low} мени ' + + 'значението на стойността на реда, но не и каноничната база; `date_flag` ∈ {ok, ' + + 'signed_after_publication} е вердикт за датата.', "`tenders.procedure_type = 'неизвестна'` маркира СИНТЕТИЧНИ (само-договорни) преписки — " + 'изключи ги при анализ на разпределението по процедура, освен ако нарочно ги искаш.', '`lots` са на grain по обособена позиция — не ги брой едно към едно срещу `contracts`.', diff --git a/apps/web/app/lib/conflicts.test.ts b/apps/web/app/lib/conflicts.test.ts index a4bfd5f8..6cec490a 100644 --- a/apps/web/app/lib/conflicts.test.ts +++ b/apps/web/app/lib/conflicts.test.ts @@ -21,6 +21,7 @@ import { partitionContracts, relationLabel, temporalLabel, + registryEvidenceLabel, } from './conflicts'; function link(over: Partial = {}): ConflictLink { @@ -43,7 +44,13 @@ function link(over: Partial = {}): ConflictLink { contemporaneousValueEur: 40_000_000, firstContractYear: '2021', lastContractYear: '2024', - sourceUrl: 'https://register.cacbg.bg/2024/i.xml', + sourceUrl: 'https://register.cacbg.bg/2024/x.xml', + // #279: a link only reaches the DTO when its identity rests on a Trade Register fact. + evidenceKind: 'document', + registryRole: 'owner', + registryEntryNumber: '20110502101007', + registryEntryDate: '2011-05-02', + registryLookupDate: '2026-08-05', ...over, }; } @@ -523,3 +530,31 @@ describe('authorityShareDisplay', () => { }); }); }); + +describe('registryEvidenceLabel', () => { + // The wording is load-bearing. The register records a ROLE; it does not certify that the official owns + // anything — that claim comes from their own declaration and is rendered separately. A label that said + // „собственик според ТР" would assert something the evidence does not support (ADR-0033 decision 2). + it('reports what the act records, never an ownership conclusion', () => { + expect(registryEvidenceLabel({ evidenceKind: 'document', registryRole: 'owner' })).toBe( + 'лицето е вписано като съдружник/собственик', + ); + expect(registryEvidenceLabel({ evidenceKind: 'document', registryRole: 'manager' })).toBe( + 'лицето е вписано като управител', + ); + }); + + it('a seat/ЕИК confirmation claims identity, not a registry role', () => { + // „Потвърдено" means the COMPANY was identified from something the official declared — nobody was + // found in the act, so the label must not imply anyone was. + const label = registryEvidenceLabel({ evidenceKind: 'confirmed', registryRole: null }); + expect(label).toBe('самоличност, потвърдена по декларирани данни'); + expect(label).not.toMatch(/вписан/); + }); + + it('never renders the word „собственик" for a mere confirmation', () => { + expect(registryEvidenceLabel({ evidenceKind: 'confirmed', registryRole: 'owner' })).not.toMatch( + /собственик/, + ); + }); +}); diff --git a/apps/web/app/lib/conflicts.ts b/apps/web/app/lib/conflicts.ts index 50a6f16b..7e0908c1 100644 --- a/apps/web/app/lib/conflicts.ts +++ b/apps/web/app/lib/conflicts.ts @@ -24,10 +24,45 @@ const RELATION_LABEL: Record = { }; /** Bulgarian label for a declared relation. Unknown values pass through — never invent a stronger claim. */ +/** + * How the company's identity was established, in the register's own terms (#279, ADR-0033). + * + * Deliberately does NOT say the official owns anything: „вписан съдружник/собственик" reports what the + * act RECORDS, while the ownership claim itself comes from the official's own declaration and is + * rendered separately as „дялово участие". „Потвърдено" means the company was identified by a fact the + * official declared — the seat or the ЕИК — not that anybody was found in the act. + */ +export function registryEvidenceLabel(l: { + evidenceKind: 'document' | 'confirmed'; + registryRole: 'owner' | 'manager' | null; +}): string { + if (l.evidenceKind === 'confirmed') return 'самоличност, потвърдена по декларирани данни'; + return l.registryRole === 'manager' + ? 'лицето е вписано като управител' + : 'лицето е вписано като съдружник/собственик'; +} + export function relationLabel(relation: string): string { return RELATION_LABEL[relation] ?? relation; } +/** + * How to describe a PAGE's set of links in prose (#279 §2.6). The card labels above are already + * family-aware; the surrounding page copy was not, and asserted „собствен дял" — an OWN stake — above + * cards that correctly read „свързано лице". On a family-only page that is a false claim about the named + * official, and it is the second source of truth the card-label fix set out to remove. + * + * Derived from the links themselves rather than passed in, so a page cannot describe a set it isn't + * rendering. Mixed sets get the neutral wording: it is the only phrasing true of every card. + */ +export function declaredStakeNoun(links: { relation: string }[]): string { + const anyFamily = links.some((l) => l.relation === 'related'); + const anySelf = links.some((l) => l.relation !== 'related'); + if (anyFamily && !anySelf) return 'дял на свързано лице'; + if (anyFamily && anySelf) return 'деклариран дял — собствен или на свързано лице'; + return 'собствен дял'; +} + // Defense in depth: the slug is base64url and the ЕИК numeric today (so encoding is a no-op), but if either // assumption ever drifts, an un-escaped `/`, `?` or `#` would break routing and the cache key. Escape the // dynamic segments unconditionally (ydimitrof #226, conflicts.ts). diff --git a/apps/web/app/routes/conflict.company.tsx b/apps/web/app/routes/conflict.company.tsx index 9dd28595..5de8e12e 100644 --- a/apps/web/app/routes/conflict.company.tsx +++ b/apps/web/app/routes/conflict.company.tsx @@ -9,7 +9,7 @@ import { ConflictCards } from '../components/ConflictCards'; import { publicCache } from '../lib/cache'; import { withDbRetry } from '../lib/retry'; import { seoMeta } from '../lib/meta'; -import { companyProfileHref } from '../lib/conflicts'; +import { companyProfileHref, declaredStakeNoun } from '../lib/conflicts'; // Officials with a published declared interest in one winner (by ЕИК). Reads interest_links only. 404 when // no official has a published link to this company — never an empty page under a company's name. @@ -59,7 +59,7 @@ export default function ConflictCompany({ loaderData }: Route.ComponentProps) { } title={company} - lede={`Длъжностни лица, декларирали собствен дял в това дружество пред КПКОНПИ. ${count(links.length)} ${plural(links.length, 'връзка', 'връзки')} — всяка е точно съвпадение по фирмено име.`} + lede={`Длъжностни лица, декларирали ${declaredStakeNoun(links)} в това дружество пред КПКОНПИ. ${count(links.length)} ${plural(links.length, 'връзка', 'връзки')} — всяка почива на проверим факт от Търговския регистър.`} /> diff --git a/apps/web/app/routes/conflict.methodology.tsx b/apps/web/app/routes/conflict.methodology.tsx index 32936a30..d7b1f9d4 100644 --- a/apps/web/app/routes/conflict.methodology.tsx +++ b/apps/web/app/routes/conflict.methodology.tsx @@ -182,6 +182,76 @@ export default function ConflictMethodology() { връзката е двусмислена и не се публикува — никога не отгатваме кой от субектите е имал предвид деклараторът.

+ +

Проверка в Търговския регистър

+

+ Съвпадението по име вече не е достатъчно, за да се публикува + връзка. За всяко дружество правим отделна справка в{' '} + Търговския регистър (публична, без регистрация) и връзката се + показва само ако самоличността на дружеството е потвърдена от{' '} + проверим факт в акта. Редът е следният — прилага се първото, което + е изпълнено: +

+
    +
  1. + Акционерна форма (АД, ЕАД, КДА) — не се показва никога. Книгата + на акционерите не е публична, тоест твърдението е непроверимо; освен това дялът + може да е миноритарен или борсов, тоест несъществен. +
    2. +
  2. + Документ. И трите имена на лицето се срещат в живо поле на акта — + управители, съдружници, едноличен собственик или физическо лице-търговец. Пазим + ролята (съдружник/собственик или само управител), защото тя не е едно и също.{' '} + + Съвпадението по име само по себе си не стига: изисква се и нещо извън фирменото + наименование да установи, че дружеството е декларираното + {' '} + — деклариран ЕИК, декларирано седалище, съвпадащо с вписаното, или достатъчно + отличително фирмено наименование. Иначе връзката остава скрита. Причината е, че + дружеството се разпознава сред изпълнителите по обществени поръчки: ако + лицето притежава едноименно дружество, което никога не е кандидатствало, търсенето + би посочило чуждото — и съименник в неговия акт би „потвърдил" връзка, невярна и в + двете си части. +
    3. +
  3. + Потвърдено. Лицето не е намерено в акта, но декларирано от него + седалище съвпада с регистрираното за този ЕИК, или самият + декларатор е изписал ЕИК. Тук се потвърждава{' '} + кое е дружеството, а не кой го притежава. +
    4. +
  4. + Оборена. Само за собствен дял: лицето не фигурира никъде в акта, + а най-късното вписване за собственост е преди декларирания период + — тоест записът покрива целия период и не назовава лицето. Връзката отпада. +
    5. +
  5. + Неизвестна — всичко останало. Остава скрита. +
    6. +
+

+ Какво доказва това и какво не. Регистърът потвърждава{' '} + самоличността на дружеството; той не удостоверява, че лицето притежава дял + — това твърдение идва от неговата собствена декларация. Затова на картата пише + „лицето е вписано като съдружник/собственик", а не „лицето е собственик според ТР". +

+

+ Ограниченията, честно. Търговският регистър{' '} + не съдържа ЕГН, затова съвпадението по трите имена не е абсолютно + доказателство за самоличност — възможен е съименник. Затова се + изисква пълно съвпадение и на трите имена в{' '} + един и същ запис на акта (две от три не се приема; имена с инициали + или с латински букви не се приемат). Седалищата се променят, затова регистрираното + седалище се приема само ако е вписано преди декларирания период —{' '} + и само когато този период е известен. Ако от декларациите не може + да се извлече година, проверката няма с какво да се направи и връзката остава + скрита. Заличените записи се пропускат навсякъде. +

+

+ Справките се опресняват месечно, а датата на справката и номерът на + вписването се показват на всяка връзка — така всяко твърдение носи своя източник и + своята давност. Актовете се пазят само служебно, за срок до 35 дни; имена на трети + лица от тях не се съхраняват и не влизат в никакъв публичен запис. +

@@ -267,7 +337,11 @@ export default function ConflictMethodology() { Прекратен дял. Ако лице подаде по-късна декларация за собственост, която вече не изброява дадено дружество, връзката се смята за приключила и се оттегля от публичната част — така остарял запис - не внушава конфликт, който е приключил. + не внушава конфликт, който е приключил. Това обаче е извод от мълчание: + най-честата причина да спреш да декларираш е изтекъл мандат, а не продажба. Затова + преди оттеглянето сверяваме с живия акт в Търговския регистър — + ако лицето и днес е вписано като съдружник или собственик, дялът не е прекратен и + връзката остава.

@@ -363,11 +437,13 @@ export default function ConflictMethodology() {

Разглеждаме сигнала в срок до 7 работни дни. При основателен сигнал - връзката се премахва незабавно и се записва в списък за премахване, - който оцелява при всяко следващо обновяване — веднъж премахната - връзка не се възстановява при повторно зареждане на данните. Премахваме връзка при - доказана неточност; не заличаваме коректни записи по искане на изпълнител или - възложител — данните идват от публични източници и остават публични. + връзката се премахва веднага след решението и се записва в списък + за премахване, който оцелява при всяко следващо обновяване — веднъж + премахната връзка не се възстановява при повторно зареждане на данните. Премахването + се прилага незабавно в базата; страницата може да се вижда в кеш{' '} + до един час след това. Премахваме връзка при доказана неточност; не + заличаваме коректни записи по искане на изпълнител или възложител — данните идват от + публични източници и остават публични.

diff --git a/apps/web/app/routes/conflict.official.tsx b/apps/web/app/routes/conflict.official.tsx index 8ef91879..01710bf5 100644 --- a/apps/web/app/routes/conflict.official.tsx +++ b/apps/web/app/routes/conflict.official.tsx @@ -8,6 +8,7 @@ import { ConflictCards } from '../components/ConflictCards'; import { publicCache } from '../lib/cache'; import { withDbRetry } from '../lib/retry'; import { seoMeta } from '../lib/meta'; +import { declaredStakeNoun } from '../lib/conflicts'; // One office-holder's declared ownership links. Reads private-ownership interest_links only. 404 (not an // empty page) when the person has no published link — a bare page under someone's name reads as an @@ -39,6 +40,9 @@ export async function loader({ params, context }: Route.LoaderArgs) { export default function ConflictOfficial({ loaderData }: Route.ComponentProps) { const { official, links } = loaderData; + // Family-AWARE, not family-blind: the page must not assert an own stake above cards that say „свързано + // лице", and must not go vague where the stake really is the official's own (§2.6). + const stake = declaredStakeNoun(links); return ( <>

Данните са от собствените декларации на лицето (публичен регистър на КПКОНПИ), - съпоставени точно с регистъра на изпълнителите. Показваме само 100% съвпадения и само - собствен деклариран дял. Сигнал за неточност:{' '} - Методология → Поправки. + съпоставени точно с регистъра на изпълнителите. Показваме само 100% съвпадения, и само + когато самоличността на дружеството е потвърдена от Търговския регистър. Сигнал за + неточност: Методология → Поправки.

= {}): ConflictLink { contemporaneousValueEur: 30_000_000, firstContractYear: '2020', lastContractYear: '2024', - sourceUrl: 'https://register.cacbg.bg/2024/i.xml', + sourceUrl: 'https://register.cacbg.bg/2024/x.xml', + // #279: a link only reaches the DTO when its identity rests on a Trade Register fact. + evidenceKind: 'document', + registryRole: 'owner', + registryEntryNumber: '20110502101007', + registryEntryDate: '2011-05-02', + registryLookupDate: '2026-08-05', ...over, }; } @@ -94,6 +100,29 @@ describe('/conflicts/official/:id — render', () => { expect(card.textContent).not.toContain('Кмет Тестов'); }); + it('a family-only page never asserts the official owns the stake (§2.6)', async () => { + // The CARD labels were made tense-neutral and family-aware, but the page lede and the section hint + // still read „декларирало собствен дял" — a second source of truth on the very page that renders a + // relative's stake. On a family-only page that is a false claim about the named official, printed + // above a card that correctly says „свързано лице". + await mount(ConflictOfficial as never, { + official: 'Кмет Тестов', + links: [link({ relation: 'related', company: 'ЕВРОСТРОЙ 21 ЕООД', eik: '333' })], + }); + expect(text()).not.toContain('собствен дял'); + expect(text()).toContain('деклариран дял на свързано лице'); + }); + + it('an own-stake page still says so — the wording is family-AWARE, not family-blind', async () => { + // POSITIVE CONTROL. Removing the claim everywhere would satisfy the assertion above while making the + // page vaguer than the data warrants: a self stake IS the official's own and should read that way. + await mount(ConflictOfficial as never, { + official: 'Кмет Тестов', + links: [link({ relation: 'owns', company: 'ЕВРОСТРОЙ 21 ЕООД', eik: '333' })], + }); + expect(text()).toContain('собствен дял'); + }); + it('meta() names the person in the title and marks the page noindex', () => { const tags = officialMeta({ data: { official: 'Иван Петров', links: [] }, @@ -133,6 +162,35 @@ describe('/conflicts/company/:eik — render', () => { }); describe('/conflicts/methodology — render', () => { + it('discloses the matching rule verbatim — every rung, and what each may conclude', async () => { + // ADR-0021 E10 makes this page the disclosure of the rule, and ADR-0033 decision 7 makes it a LAUNCH + // CONDITION rather than a follow-up: a heuristic that asserts something about a named person is only + // defensible if the reader can see exactly what was asserted and why. Nothing but a test keeps the + // page in step with the ladder — the rule can change in evidence.mjs and leave the page describing a + // system that no longer exists, which is worse than not disclosing it at all. + await mount(ConflictMethodology as never, {}); + const t = text(); + // every rung of the ladder, by the name the seal and the card use + for (const rung of ['Документ', 'Потвърдено', 'Оборена', 'Неизвестна']) + expect(t).toContain(rung); + // rung 1 — the joint-stock bar and its reason (the „11 акции" trap) + expect(t).toContain('Акционерна форма'); + expect(t).toContain('не е публична'); + // rung 2 — all three names, one record, and the two refusals + expect(t).toContain('пълно съвпадение и на трите имена'); + expect(t).toContain('един и същ запис'); + // ADR-0035 — the company gate, the part a reader most needs to judge the claim + expect(t).toContain('Съвпадението по име само по себе си не стига'); + // R10 — the seat's temporal guard, both halves + expect(t).toContain('вписано преди декларирания период'); + expect(t).toContain('когато този период е известен'); + // the honest limit: no ЕГН, so a homonym is possible + expect(t).toContain('не съдържа ЕГН'); + expect(t).toContain('съименник'); + // what the register proves and what it does not — the distinction the whole surface rests on + expect(t).toContain('самоличността на дружеството'); + }); + it('states the three libel rails in plain language', async () => { await mount(ConflictMethodology as never, {}); const t = text(); diff --git a/apps/web/app/routes/conflicts.render.test.tsx b/apps/web/app/routes/conflicts.render.test.tsx index 68dad5fb..814c3670 100644 --- a/apps/web/app/routes/conflicts.render.test.tsx +++ b/apps/web/app/routes/conflicts.render.test.tsx @@ -36,6 +36,12 @@ function link(over: Partial = {}): ConflictLink { firstContractYear: '2020', lastContractYear: '2024', sourceUrl: 'https://register.cacbg.bg/2024/i.xml', + // #279: a link only reaches the DTO when its identity rests on a Trade Register fact. + evidenceKind: 'document', + registryRole: 'owner', + registryEntryNumber: '20110502101007', + registryEntryDate: '2011-05-02', + registryLookupDate: '2026-08-05', ...over, }; } @@ -47,6 +53,13 @@ const familyLink = link({ company: 'ЕВРОСТРОЙ 21 ЕООД', eik: '333', relation: 'related', // a close relative's stake — anonymized + // The shape the QUERY actually yields for a family link, not the base factory's self-link default. + // `findPerson` searches the act for the OFFICIAL; a relative's stake is registered to the relative, + // so the official is never found and no `document` rung is reachable — only a seat/ЕИК confirmation. + // A fixture carrying `document`/`owner` here tested a row that cannot exist and hid the assertion + // below, which is the one that matters: no registry-role claim on an anonymized card. + evidenceKind: 'confirmed' as const, + registryRole: null, ownInstitution: false, contractCount: 1, contractValueEur: 250_000, @@ -188,8 +201,16 @@ describe('/conflicts route — render', () => { (a) => a.textContent === 'декларация', ); expect(sourceAnchor?.getAttribute('href')).toBe('https://register.cacbg.bg/2024/i.xml'); - // the zero-contract link has no toggle (contractCount === 0) and a muted „—" source - expect(text()).toContain('—'); + // Scoped to the card that actually has sourceUrl: null. A page-wide toContain('—') passes on any + // em-dash anywhere — including the ones the value and date cells render — so it would survive the + // source branch being deleted outright. + const cards = container.querySelectorAll('.conflict-card'); + const noSourceCard = [...cards].find((c) => c.textContent?.includes('ПРАЗЕН ООД'))!; + expect(noSourceCard).toBeTruthy(); + expect( + [...noSourceCard.querySelectorAll('a')].some((a) => a.textContent === 'декларация'), + ).toBe(false); + expect(noSourceCard.textContent).toContain('—'); }); it('a zero-contract link renders no „Виж договорите" toggle', async () => { @@ -238,3 +259,57 @@ describe('/conflicts route — render', () => { ).not.toBeNull(); }); }); + +describe('Trade Register evidence on the card (#279, ADR-0033)', () => { + it('renders the registry fact the link rests on, so the card explains itself', async () => { + await renderConflicts([link({ evidenceKind: 'document', registryRole: 'owner' })]); + const text = container.textContent ?? ''; + expect(text).toContain('Регистър'); + expect(text).toContain('лицето е вписано като съдружник/собственик'); + expect(text).toContain('вписване 2011-05-02'); // WHICH entry + expect(text).toContain('справка 2026-08-05'); // and HOW FRESH it is + // The entry NUMBER is what makes the claim findable in the register — a date alone does not + // identify a record. It was carried to every client in the DTO and never rendered, which is the + // one payload that costs bytes and answers nothing (cefothe, #309). + expect(text).toContain('20110502101007'); + }); + + it('omits the entry number rather than printing an empty label when there is none', async () => { + // POSITIVE CONTROL for the row's shape: a confirmed link (seat/ЕИК) has no act entry to cite, so + // the label must be absent entirely — not „· №" with nothing after it, which reads as missing data + // rather than as an inapplicable field. Scoped to the evidence label: „№" alone is the card RANK. + await renderConflicts([ + link({ evidenceKind: 'confirmed', registryRole: null, registryEntryNumber: null }), + ]); + const text = container.textContent ?? ''; + expect(text).toContain('Регистър'); + expect(text).not.toContain('· №'); + }); + + it('a seat/ЕИК confirmation never implies somebody was found in the act', async () => { + await renderConflicts([link({ evidenceKind: 'confirmed', registryRole: null })]); + const text = container.textContent ?? ''; + expect(text).toContain('самоличност, потвърдена по декларирани данни'); + expect(text).not.toContain('вписано като'); + }); + + it('a FAMILY card never carries a registry-role claim — the relative is not in the act we read', async () => { + // The production shape for a family link, which the fixture used to contradict: `findPerson` looks + // for the OFFICIAL, and a relative's stake is registered to the relative, so the official is never + // found and the rung can only ever be `confirmed`/`registryRole: null`. A card that said „лицето е + // вписано като съдружник/собственик" here would assert that the named official is recorded in the + // register as an owner of this company — a false, named, libel-shaped claim, on the one card whose + // whole design is that the stakeholder stays anonymous (ADR-0030/0032). + await renderConflicts([familyLink]); + const familyCard = container.querySelector('.conflict-card')!; + expect(familyCard.textContent).toContain('деклариран дял на свързано лице'); + expect(familyCard.textContent).toContain('самоличност, потвърдена по декларирани данни'); + expect(familyCard.textContent).not.toContain('вписано като'); + }); + + it('links out to the register so a reader can check the same act we read', async () => { + await renderConflicts([link({ eik: '201122335' })]); + const hrefs = [...container.querySelectorAll('a')].map((a) => a.getAttribute('href') ?? ''); + expect(hrefs.some((h) => h.includes('201122335'))).toBe(true); + }); +}); diff --git a/apps/web/app/routes/contract.tsx b/apps/web/app/routes/contract.tsx index 8eb1175b..e3347461 100644 --- a/apps/web/app/routes/contract.tsx +++ b/apps/web/app/routes/contract.tsx @@ -222,7 +222,13 @@ export default function Contract({ loaderData }: Route.ComponentProps) {
Текуща стойност
{v.currentEur != null ? money(v.currentEur) : '—'} - {v.suspect &&
{UNVERIFIED_VALUE_LABEL}
} + {v.currentValueDoubled ? ( +
+ стойността изглежда двойно отчетена и не се показва +
+ ) : ( + v.suspect &&
{UNVERIFIED_VALUE_LABEL}
+ )} {v.deltaPct != null && (
{signedPct(v.deltaPct)} спрямо сключване
)} @@ -230,8 +236,10 @@ export default function Contract({ loaderData }: Route.ComponentProps) {
{v.suspect && (

- Показана е публикуваната стойност от източника, без СИГМА да я коригира. Виж{' '} - методология. + {v.currentValueDoubled + ? 'Текущата стойност изглежда двойно отчетена в източника и затова не се показва. ' + : 'Показана е публикуваната стойност от източника, без СИГМА да я коригира. '} + Виж методология.

)} {c.frameworkAwards != null && ( @@ -267,11 +275,32 @@ export default function Contract({ loaderData }: Route.ComponentProps) { {c.amendments.map((a, i) => ( {a.date ? longDate(a.date) : '—'} + {/* #305 residual: an uncorrectable double-count — the source's value_after is the + untrusted doubled figure, so show „—" and mark the row rather than a number we + can't stand behind. A `restated` row is the opposite: СИГМА corrected the doubled + total from the основание text, so we show the corrected number and flag that we + rewrote it. */} - {a.valueAfterEur != null ? moneyBare(a.valueAfterEur) : '—'} + {a.suspect ? ( + <> + — непотвърден тотал + + ) : a.valueAfterEur != null ? ( + <> + {moneyBare(a.valueAfterEur)} + {a.restated && ( + <> + {' '} + коригиран тотал + + )} + + ) : ( + '—' + )} - {a.deltaEur != null ? signedMoney(a.deltaEur) : '—'} + {!a.suspect && a.deltaEur != null ? signedMoney(a.deltaEur) : '—'} diff --git a/coverage-baseline.json b/coverage-baseline.json index 5abe85c9..064f448f 100644 --- a/coverage-baseline.json +++ b/coverage-baseline.json @@ -6,24 +6,24 @@ "branches": 58.2 }, "apps/web": { - "lines": 89.7, - "branches": 81.8 + "lines": 91, + "branches": 82.4 }, "packages/config": { "lines": 92.8, "branches": 72.2 }, "packages/db": { - "lines": 94.2, - "branches": 79 + "lines": 94.5, + "branches": 79.3 }, "packages/ingest": { - "lines": 85.8, - "branches": 80 + "lines": 86.3, + "branches": 80.4 }, "packages/shared": { - "lines": 95.4, - "branches": 80 + "lines": 95.5, + "branches": 80.8 } } } diff --git a/docs/README.md b/docs/README.md index 0a3bf6e0..16f03f97 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,12 @@ - [`etl-pipeline-state.md`](etl-pipeline-state.md) — анализ на текущото състояние на ETL pipeline-а. - [`etl-architecture.md`](etl-architecture.md) — целевата ETL архитектура (RFC): предложение за състоянието и реда на изпълнение. - [`v1-implementation-plan.md`](v1-implementation-plan.md) — precompute слоят и пагинацията (защо rollup-и и keyset вместо per-request GROUP BY / OFFSET). +- [`implementation-plans/286-ocds-amendment-unp.md`](implementation-plans/286-ocds-amendment-unp.md) — защо OCDS анексите не се свързват с договор (OCID вместо УНП) и планът за поправка през bridge-а `tender.id → УНП` + prefer-EOP dedup (#286). +<<<<<<< HEAD +- [`implementation-plans/306-amendment-contract-namespace-link.md`](implementation-plans/306-amendment-contract-namespace-link.md) — защо 1 937 EOP анекса не се свързват с договор (номерът на анекса е в друго именно пространство от деловодния номер) и планът за поправка чрез value-anchor (`value_before → signing_value`, 99.99% точност) (#306). +======= +- [`implementation-plans/305-amendment-value-double-count.md`](implementation-plans/305-amendment-value-double-count.md) — защо стойността на анекс се удвоява (ЦАИС ЕОП слага новия **тотал** в полето за промяна) и планът за откриване/поправка: `annex_total_suspect` флаг + текстова хеуристика за възстановяване на истинския тотал (#305). +>>>>>>> origin/main - [`integrity-gate.md`](integrity-gate.md) — reconciliation gate-ът: hard asserts върху тоталите при import/CI. - [`anomaly-report.md`](anomaly-report.md) — cross-row аномалии при опресняване: какво `value_flag` не хваща на ниво отделен договор. - [`deploy.md`](deploy.md) — деплой към Cloudflare: двата Worker-а (`sigma`, `sigma-etl`) и споделеният D1 per environment. diff --git a/docs/adr/0007-scope-and-certainty-bar.md b/docs/adr/0007-scope-and-certainty-bar.md index 18fcb446..c41f4255 100644 --- a/docs/adr/0007-scope-and-certainty-bar.md +++ b/docs/adr/0007-scope-and-certainty-bar.md @@ -1,6 +1,7 @@ # ADR-0007: Scope and certainty bar - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — решения 2 и 3 (изричното изключение за „евристика, която твърди") - Date: 2026-07-05 - Deciders: lb (Head of AI, ИО), Claude - Related: spec §1–2, §8 diff --git a/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md b/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md index 7803e901..8783b3ac 100644 --- a/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md +++ b/docs/adr/0009-name-uniqueness-guard-and-publish-tiers.md @@ -1,8 +1,9 @@ # ADR-0009: Name-uniqueness is not absolute → single-ЕИК guard + publish tiers -- Status: Accepted +- Status: Superseded by [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Date: 2026-07-05 - Deciders: lb, Claude +- Superseded by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Related: spec §5; refines [ADR-0008](0008-deterministic-name-to-eik-resolution.md) ## Context diff --git a/docs/adr/0010-pii-posture.md b/docs/adr/0010-pii-posture.md index 8f09ead8..cf9898a0 100644 --- a/docs/adr/0010-pii-posture.md +++ b/docs/adr/0010-pii-posture.md @@ -1,6 +1,7 @@ # ADR-0010: PII posture - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — решения 3 и 6 (кешът от актове и срокът на съхранение) - Date: 2026-07-05 - Deciders: lb, Claude - Related: spec §8; КЗЛД guidance; GDPR Art. 6(1)(c)/(f), Art. 85 diff --git a/docs/adr/0015-tr-name-uniqueness-census.md b/docs/adr/0015-tr-name-uniqueness-census.md index b3fb2789..c86d8eec 100644 --- a/docs/adr/0015-tr-name-uniqueness-census.md +++ b/docs/adr/0015-tr-name-uniqueness-census.md @@ -1,8 +1,9 @@ # ADR-0015: TR name-uniqueness census (promoting tier-C generic-name matches) -- Status: Accepted (design; implemented as a Phase-1 pipeline step) +- Status: Superseded by [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Date: 2026-07-05 - Deciders: lb, Claude +- Superseded by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Related: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md); spec §5 ## Context diff --git a/docs/adr/0017-name-collision-tier-gate.md b/docs/adr/0017-name-collision-tier-gate.md index 8d4a234b..38ed31b5 100644 --- a/docs/adr/0017-name-collision-tier-gate.md +++ b/docs/adr/0017-name-collision-tier-gate.md @@ -1,8 +1,9 @@ # ADR-0017: A globally non-unique name cannot ride the name-distinctive tier — even with a certain ЕИК -- Status: Accepted +- Status: Superseded by [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) - Date: 2026-07-05 - Deciders: lb, Claude +- Superseded by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — the name-distinctive tier it gates leaves the publishing path; the `nameGloballyUnique` precondition survives, narrowed to the weakest evidence rung - Related: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md), [ADR-0016](0016-free-text-entity-resolution.md); `scripts/cacbg/load.mjs`, `scripts/cacbg/audit.mjs` ## Context diff --git a/docs/adr/0021-methodology-page-and-temporal-freshness.md b/docs/adr/0021-methodology-page-and-temporal-freshness.md index 810a74b2..d2f0d9cd 100644 --- a/docs/adr/0021-methodology-page-and-temporal-freshness.md +++ b/docs/adr/0021-methodology-page-and-temporal-freshness.md @@ -1,6 +1,7 @@ # ADR-0021: Public methodology/corrections page (E10) + temporal freshness & divestment expiry (E11) - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — E11 (етикетите за настояще време се отлагат за фаза 2) - Date: 2026-07-06 - Deciders: lb, Claude - Related: [ADR-0007](0007-scope-and-certainty-bar.md), [ADR-0019](0019-private-interest-vs-ex-officio-classification.md), [ADR-0020](0020-conflict-explorer-surface-posture.md); spec §5/§8/§9; `scripts/cacbg/load.mjs`, `apps/web/app/routes/conflict.methodology.tsx`, `packages/db/src/queries/related-persons.ts` diff --git a/docs/adr/0023-anonymized-family-ownership-surface.md b/docs/adr/0023-anonymized-family-ownership-surface.md index 3ead3824..d5dcea50 100644 --- a/docs/adr/0023-anonymized-family-ownership-surface.md +++ b/docs/adr/0023-anonymized-family-ownership-surface.md @@ -1,8 +1,9 @@ # ADR-0023: Anonymized close-relative (family) ownership surface -- Status: Accepted +- Status: Superseded by [ADR-0030](0030-family-ownership-withheld-nameless-aggregate.md) - Date: 2026-07-07 - Deciders: lb, Claude +- Superseded by: [ADR-0030](0030-family-ownership-withheld-nameless-aggregate.md) - Related: [ADR-0007](0007-scope-and-certainty-bar.md), [ADR-0010](0010-pii-posture.md), [ADR-0019](0019-private-interest-vs-ex-officio-classification.md), [ADR-0022](0022-public-surface-private-ownership-only.md); `scripts/cacbg/parse.mjs`, `scripts/cacbg/load.mjs`, `packages/db/src/queries/related-persons.ts`, `apps/web/app/routes/conflict*.tsx` ## Context diff --git a/docs/adr/0028-declared-eik-is-a-determining-identifier.md b/docs/adr/0028-declared-eik-is-a-determining-identifier.md index 9f123482..9ad76597 100644 --- a/docs/adr/0028-declared-eik-is-a-determining-identifier.md +++ b/docs/adr/0028-declared-eik-is-a-determining-identifier.md @@ -1,6 +1,7 @@ # ADR-0028: Declared ЕИК is a determining identifier — tier `A_eik`, exempt from the TR census - Status: Accepted +- Amended by: [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) — декларираният ЕИК остава определящ, но вече иска и регистърно доказателство - Date: 2026-07-11 - Deciders: lb, Claude - Related: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md), [ADR-0015](0015-tr-name-uniqueness-census.md), [ADR-0016](0016-free-text-entity-resolution.md); `scripts/cacbg/load.mjs`, `scripts/cacbg/classify.mjs` diff --git a/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md new file mode 100644 index 00000000..82633fb7 --- /dev/null +++ b/docs/adr/0033-registry-evidence-replaces-name-distinctiveness.md @@ -0,0 +1,364 @@ +# ADR-0033: A link publishes only against a Trade Register fact — the evidence ladder replaces the name-distinctiveness tier + +- Status: Accepted (design; implemented as the #279 pipeline change) +- Date: 2026-08-05 +- Deciders: Todor (maintainer), Claude +- Supersedes: [ADR-0009](0009-name-uniqueness-guard-and-publish-tiers.md), [ADR-0015](0015-tr-name-uniqueness-census.md), [ADR-0017](0017-name-collision-tier-gate.md) +- Amends: [ADR-0007](0007-scope-and-certainty-bar.md) (decisions 2 and 3), [ADR-0010](0010-pii-posture.md) (decisions 3 and 6), [ADR-0021](0021-methodology-page-and-temporal-freshness.md) (E11 only — E10 stands and is strengthened), [ADR-0028](0028-declared-eik-is-a-determining-identifier.md) +- Related: [ADR-0008](0008-deterministic-name-to-eik-resolution.md), [ADR-0011](0011-host-scoped-tls-pinning.md), [ADR-0026](0026-person-grain-name-institution.md), [ADR-0027](0027-overmerge-gate-is-telemetry-not-a-gate.md), [ADR-0031](0031-suppressions-version-controlled-fingerprinted.md), [ADR-0032](0032-family-ownership-published-under-public-interest.md); spec §3.3/§5/§8/§9, [related-persons-lia.md](../spec/related-persons-lia.md); `scripts/tr/`, `scripts/cacbg/{load,classify}.mjs`, `packages/db/src/queries/related-persons.ts`, `apps/web/app/routes/conflict*.tsx` + +## Context + +Today the identity of the company behind a declared name is decided by a heuristic. `B_distinctive` +(ADR-0009) publishes a link when the name is *structurally* distinctive — a digit, a Latin token, or ≥3 +content words. It fails in both directions: it publishes on a bare name coincidence, and it withholds +everything else. Of the 101 links published today, **36 rest on name coincidence alone**. + +The Trade Register answers the identity question directly, over a public endpoint with no authentication: +`GET https://portal.registryagency.bg/CR/api/Deeds/{ЕИК}`. Either the declarant is named in the company's +live deed, or facts they declared match the ones registered against that ЕИК. + +Four things must be on the table before the decision, because each of them changes what is being decided. + +**1. This is an expansion, not a tightening.** The measured surface grows from 101 links to **329 links / +311 people / 239 companies**. „From held / published / withdrawn = 163 / 65 / 101" means **264 links we +deliberately withhold today become named public claims**. At 329 links a 1% error rate is three falsely +named people. The precision bar has to go **up**, not stay flat — which is why the acceptance gate below is +a hand-labelled sample and not the control totals. + +**2. The second rung is a heuristic used to assert — stated precisely.** What the registry evidence +establishes is **the identity of the company**, not that the official owns it: the ownership claim comes +from the official's own filed declaration and is not a heuristic at all. Rung 2 („Документ") confirms that +*the company this declared name refers to is the same legal entity as the winner we matched*, by finding +the declarant's full name inside that company's live fields. A homonym failure therefore does not invent an +ownership claim — it **attaches the official to the wrong company's ЕИК, contracts and money**. That is a +different error from the one the adversarial reading suggests, and it is still a false public claim about a +named person. + +A three-name subset match against free text, in a register that carries no ЕГН, is **not deterministic**. +ADR-0007's decision list rules it out twice: item 2 (published only when the official↔company↔ЕИК +resolution is *deterministic*) and item 3 (heuristics may *withhold* or *triage*, **never assert**). #279 +replaces a weaker heuristic with a stronger one and promotes it to grounds for assertion. That is the +decision this ADR exists to make explicit; it cannot be smuggled in as an implementation detail. + +**3. The shape of the access is the permitted one.** Spec §3.3 permits „bounded per-ЕИК … thousands, not +~900k" and forbids bulk scraping; ADR-0007 decision 1 already anticipated the „per-ЕИК lookup only, later +leg". The crawl is one lookup per candidate ЕИК — on the order of hundreds — with a closed candidate set: +the crawler never follows a link out of a deed. This is that later leg, not the bulk reuse. + +**4. The endpoint rate-limits, and the limiter is a stated preference.** An earlier spike against this same +API triggered **HTTP 429 at roughly 50 cumulative requests ending in a burst**, and the block was then +*sustained* — every subsequent request returned 429, including simple `/Deeds/{eik}` calls that had worked +seconds earlier. There is no `Retry-After` and no `X-RateLimit-*` header, so a client can only avoid +tripping the limit, never pace against a published one; 25 spaced requests were fine, so pacing matters more +than total volume. #279 §3's „~400 companies, about 20 minutes" is therefore feasible only at its stated +1 request / 3 seconds with no burst, and only if the crawl is resumable. The limiter is the operator's only +available way to express a rate preference, and tuning around it empirically is precisely the posture spec +§3.3's „NEVER bulk-scrape" exists to forbid. This is the evidence behind decision 7's crawl hygiene. + +**5. Parts of the source description in #279 were wrong, and were corrected by live probing** (9 sequential +requests, 2026-08-04 — well inside the pacing above, so no 429 was encountered): + +- `?entryDate=` **is** honoured — a bogus-parameter control returned byte-identical output to the + plain request, so the difference is the parameter, not noise. `…/Fields/{ident}/History` **does** return + JSON, not application HTML. #279 §3 states the opposite for both. +- A wrong `subUIC` yields **HTTP 500 `GL_ERROR_L`**, not 404 — so a 500 can never be read as „no history" + or „outside the register". +- `legalForm` 4 = ООД and 10 = ЕООД (as #279 says); the nomenclature endpoint does **not** settle the enum + — `legalForm=4` and `legalForm=10` return a byte-identical catalogue. +- The envelope's `fullName` carries the legal form (`"ПИМК" ООД`) while `CR_F_2_L` is bare. The joint-stock + bar can therefore be derived from the ЗТРРЮЛНЦ-mandated suffix, at zero extra requests. +- Erasure is **structural, not textual**. Entities inside one field are separated by + `
` blocks delimited by `
`. + An erased entity is such a block carrying an erasure marker. **Correction, on implementation:** this ADR + first named that marker `field-text--erased`. Re-measured against the full live deed for ЕИК 115536179, + that class occurs **zero** times; the register emits `
` (with + ``) and the erased container carries **no `field-text` paragraph at + all**. The parser therefore treats *either* spelling as erasure — honouring both costs nothing, while + assuming one costs a wrong publish. `fieldOperation` is **not** the signal: it reads 2 on both erased + fields in that deed but 1 on the erased history records W0 sampled, so it is an undocumented enum we do + not rely on. +- Erased versions are **live in the current deed**: the first company sampled carries a fully-erased + `CR_F_23_L` dated 2013-07-16, which read naïvely becomes „latest ownership entry: 2013-07-16" and feeds + the refutation rule. +- Seats move. The same company's `CR_F_5_L` shows entry dates 2010-11-04 and 2014-01-23. +- **An ЕИК that is not a търговец answers `HTTP 200` with a ZERO-BYTE body** — not the 404, and not the + HTML, that §3 predicts. Measured on Община София (`000696327`): empty on two consecutive requests, + while a real company returned its full 34,398-byte deed in the same window, so it is the register's + answer rather than an outage. This is what the „извън ТР" rung actually looks like on the wire. The + distinction that keeps R6 honest is therefore the **status**, not the empty body: empty under 200 is + a documented negative and may be cached permanently; empty under 5xx is a failure and stays transient. + +The as-of capability that `?entryDate=` unlocks would fix the two weakest rungs, but it **re-baselines every +control number in #279 §10**. It is therefore deliberately out of scope here and becomes a separate change +under a new rules version — which decision 6 makes the only sanctioned way the surface may move. + +## Decision + +### 1. The evidence ladder replaces the publish tiers + +For every resolved link (person × ЕИК) with a fetched deed, **the first matching rung wins**: + +| # | Outcome (data label) | Condition | Effect | +|---|---|---|---| +| 1 | **„Бар: акционерна форма"** — joint-stock bar | the company is a joint-stock form (АД / ЕАД / КДА) | never published, whatever follows | +| 2 | **„Документ"** — documented | the declarant's full name appears in a live `CR_F_7_L` / `CR_F_18_L` / `CR_F_19_L` / `CR_F_23_L` | published; the role (owner vs manager only) is kept for the label | +| 3 | **„Потвърдено"** — confirmed | the normalized declared seat equals the registered seat for that ЕИК, **or** the declarant wrote the ЕИК in the declaration | published | +| 4 | **„Оборена"** — refuted | *own* stake only: the person appears in no live field, **and** the latest entry date across the live ownership fields is strictly before the first declared year | link withdrawn | +| 5 | **„Неизвестна"** — unknown | anything else | held | +| 6 | **„Извън ТР"** — outside the register | the ЕИК is not in the register (ДЗЗД, БУЛСТАТ associations) | held | + +The Bulgarian labels are the persisted vocabulary — they reach the data, the audit and the methodology +page — so they are fixed here rather than translated at each layer. + +Erased versions are skipped everywhere the live state is read — otherwise the date on an erased record can +„certify" a state that is not in force. + +`publish_tier` carries the evidence kind. **`B_distinctive` leaves the publishing path entirely**; +name distinctiveness survives only as an ordering signal for the review queue. Rung 3's seat leg takes the +declared seat **only from declarations by the same person for the same company**: 4.9% of company-name keys +carry more than one distinct declared seat, so a company-only key would let one person's seat confirm +another person's link. + +This supersedes ADR-0009 (the tier ladder) and ADR-0015 (the name census, whose only job was to unblock +`C_hold`; `tr-census.mjs` and its `promote()` are deleted). ADR-0028's holding survives — a declared ЕИК +*is* the identity — but it is now rung 3 rather than a tier of its own, it is subject to rung 1, and its +census-exemption clause is moot. + +### 2. A heuristic may ground an assertion here — the argued exception to ADR-0007 + +Rung 2 asserts on a name match. We accept it, on these grounds and no wider: + +- **It is a full-subset match, patronymic included.** Partial (2-of-3) matching is refused outright: of 301 + matches measured, 46 were two-token only — precisely the homonym risk. A declarant name with fewer than + three tokens can never earn „Документ". +- **It is scoped to a single registry entity.** Tokens are matched only within one `record-container` + block, after erased blocks are dropped and after HTML entities are decoded. Matching against a whole + field's text would combine the given name of one person with the surname of another — the defect that + ships a libel. +- **The company identity behind it passed preventive control.** The candidate ЕИК comes from the + deterministic exact-name resolution of ADR-0008/#226, not from fuzzy matching. + + > **Superseded on this point by [ADR-0035](0035-registry-evidence-must-also-establish-the-company.md).** + > Deterministic is not the same as correct: that resolution ranges over PROCUREMENT WINNERS only, so an + > official whose real company never bid resolves to a same-named winner, and a homonym in the winner's + > deed completes a link false in both halves. Rung 2 now requires a corroborator for the COMPANY — + > declared ЕИК, a matching declared seat, or a distinctive фирма — and the uncorroborated remainder is + > withheld as `document_uncorroborated` and counted. +- **It is corroborated, and that is the whole difference.** A bare TR name is spec §4's *weakest* join — + no ЕГН, no birthdate, not even ADR-0026's `(name, ведомство)` grain. The registry-graph spike that + explored this API concluded, for its own use, that no person-derived edge may reach a user-visible + surface at all, and it named the one cheap corroborator that would change that: *an officer who also + appears as a declarant with a declared stake in the same ЕИК, confirmed by an independent source*. That + is exactly what rung 2 is. The register supplies the name-in-this-company fact; the official's own filing + supplies the stake. Neither alone would publish. **The corroboration is what licenses the assertion, so + the rule must never be extended to a name-only join** — matching a person across two companies, or + treating a registry name as an identity in its own right, stays forbidden. +- **The filters that can only withhold are retained, not removed.** `nameGloballyUnique` and + `nameDistinctiveness` stay in the pipeline as an **AND-gate on the weakest rung only** („Потвърдено"). + They cost near-zero recall and preserve ADR-0017's outcome even though its subject — the name-distinctive + tier — is gone. +- **Homoglyphs are not folded.** `company-name-key.ts` deliberately does not fold Cyrillic↔Latin; person + names take the same posture. A Latin letter inside a name means no „Документ", and the occurrence is + counted rather than silently dropped. + +**The residual collision rate is estimated, not measured, and we say so.** Bulgarian names are three-part +by statute (ЗГР чл. 9) and the API renders full triples consistently. A triple of common components — +Георги · Иванов · Петров — is on the order of 10⁻⁵ of ~3.2M men, so roughly 29 people nationally share it; +but the population that can produce a false link is company officers, a ~1% slice, which squares the +probability to ~10⁻⁴ per colliding pair. That is an argument for plausibility, not a measurement, and the +residual concentrates on exactly the common names. It is therefore **not** the basis on which this +publishes — the hand-labelled sample of decision 7 is. Hyphenated surnames count as one token; Latin-script +names are counted the same way but flagged, since a three-token foreign name is not a patronymic triple. + +The tie-breaker for every rung is the repo's own sentence, from +`packages/shared/src/company-name-key.ts`: **„When in doubt the key stays MORE specific (a recall miss is +safe; an over-merge is not)."** + +This **amends ADR-0007 decisions 2 and 3**: a published claim may now rest on a disclosed, bounded, +entity-scoped full-name match against a public register, in addition to deterministic facts. Everything +else in ADR-0007 stands — in particular that every heuristic is disclosed on the methodology page and +labelled in the data, which decision 7 makes a launch condition rather than a follow-up. + +### 3. The joint-stock bar is a union of three independent signals + +The bar exists because the shareholder book is not public (so the claim is unverifiable) and a parcel of +listed shares is not a material ownership conflict (the „11 Trace shares → €88M" trap). It fires if **any** +of the following says joint-stock: + +1. `closelyHeldForm` on the **declared** name (`load.mjs`) — a different, earlier stage; it **stays**, and + is not redundant with the two below; +2. the фирма suffix taken from the deed envelope's `fullName`, through that same tested predicate; +3. the numeric `legalForm` code from the deed. + +**An unknown `legalForm` code withholds and is reported** — it never falls through to publication. The enum +is known only from four observed values and the nomenclature endpoint does not settle it (Context 5), so +fail-open here would be a bar that silently stops barring. `КДА` — which rung 1 explicitly requires barring +— is absent from both `JOINT_STOCK` and `FORM_TOKENS` in `classify.mjs` today and is added. + +**The named open question is ЕАД.** #279 §3 lists 5 = АД, and the spike's catalogue-derived table also +reads 5 as АД/ЕАД jointly — but ООД and ЕООД turned out **not** to share a code (4 and 10), so a separate +ЕАД code is the likelier reading and no observation settles it. A single-owner joint-stock company is +precisely the shape a declarant is most likely to hold and rung 1 most needs to bar, so this is the one +place where a fail-open enum would do real damage. Signal 2 (the ЗТРРЮЛНЦ suffix on `fullName`) covers it +independently of the code, which is why the bar is a union and not a lookup. + +### 4. Termination is reconciled against the live deed — reversing ADR-0021 E11 for own stakes + +ADR-0021 E11 marks an ownership link `withdrawn` when the company is absent from the person's latest +ownership filing. That is an inference from silence, and its commonest cause is a finished mandate rather +than a sale. Before the withdrawal takes effect, every terminated **own** stake is reconciled against the +live deed: a person still named in a live ownership field (`CR_F_18/19/23_L`) has not divested, and the +link surfaces. Family stakes are never reconciled — the relative's name is neither stored nor checked +(ADR-0010 decision 4, ADR-0032 decision 2), so declared termination applies to them directly. + +**Phase 1 uses the reconciliation only to avoid withdrawing the link.** The „и към днешна дата" labels of +#279 §7, and the post-period contracts they license, are **deferred to phase 2 behind an LIA addendum**: +those labels assert a present tense about a named person, on evidence whose freshness is bounded by the +cache refresh cycle, and they change the claim's shape enough to need the balancing assessment updated +first. The derived live status is therefore recomputed on every run and **never sealed**. + +ADR-0021 **E10 is untouched and strengthened** — see decision 7. + +### 5. The deed cache — amends ADR-0010 + +A deed contains third-party personal data: addresses of natural persons, and the names of people who are +not office holders. + +- **Extraction stays inside ADR-0010 decision 3.** `CR_F_5_L` yields only the „Населено място:" segment; + no parser function returns an address. Decision 3 is re-affirmed, not overridden. +- **Storage is what changes, and that is ADR-0010 decision 6.** Raw deeds are cached under git-ignored + `scratch/tr/`, behind the same refuse-to-run guard as the declaration cache, with a **35-day retention** + (one refresh cycle plus slack) and a purge step in the same job. Decision 6's scope extends from „raw + declaration XML … deleted post-spike" to a second source with a stated TTL. +- **The cache index holds no name at all** — only ЕИК, dates, codes and verdicts, plus a body hash instead + of any content excerpt. Names exist solely in the raw JSON files, are read only to produce a boolean, and + never enter a public table, a response, or a log. ЕГН was absent from every payload examined, and the + index additionally refuses any ten-digit run in a text column — sound because an ЕИК is 9 or 13 digits, + never 10. +- **The deed's beneficial-ownership fields are deliberately not used.** A deed also carries `CR_F_550_L` + (действителни собственици, чл. 63 ЗМИП) and the control fields `CR_F_537/538_L`. Using them would make a + beneficial-ownership claim, which ADR-0007 decision 1 parked after CJEU C-37/20. Rung 2 reads the four + fields #279 names and no others; widening it is a new decision, not an improvement. +- **The sealed `matched_fact` is a closed vocabulary** (`seat:`, `role:owner:`, `eik`), + enforced by a pattern check in the audit. Without that, the matched *name* eventually gets stored there + as a convenience, which is exactly what #279 §9 forbids. + +### 6. Monotonicity is a gate, not a store — and #279 §8 is corrected + +#279 §8 requires the evidence seal to be kept „forever" and recomputation to be strictly additive. As +written that is false, and the implementation must not pretend otherwise: + +- §7's labels flip. A person who leaves the register must lose the label *and* the post-period contracts it + licensed. (Phase 1 avoids this by not shipping the labels at all — decision 4.) +- A permanent seal and a live, expiring cache contradict each other. +- A **court-annulled entry** (чл. 29 ЗТРРЮЛНЦ) invalidates the evidence without any rules change. It is + wired to the correction path of ADR-0031 and named as a ground in the suppression runbook. + +Therefore: seals are **re-derived deterministically** on every run — a seal is written for *every* link, +including held ones, so the review queue explains itself — and monotonicity is enforced as a **gate**. The +audit compares against the pre-wipe export and raises a hard finding when a previously published link +disappears under an **unchanged** `rules_version`; under a changed version it degrades to a printed diff. +Removal remains an intentional event, and each ground has an expressible mechanism — a gate that hard-fails +the only removals it sanctions is a deadlock, not a rail: + +| Ground | Mechanism | Why not one of the others | +|---|---|---| +| The rules changed | a `rules_version` bump | — | +| The evidence is void (court annulment, wrong person) | ADR-0031 suppression — `status` flips to `suppressed`, so the audit reads the current status and treats it as declared | the link is correctly built; only its publication is wrong | +| The input was wrong | `scripts/cacbg/link-corrections.jsonl`, fingerprinted like the suppression list; `load.mjs` flags the key in the pre-wipe snapshot | correcting the input *unbuilds* the link, so a suppression on it matches nothing and trips the unused-entry rail | + +Both non-rules grounds are one-shot, version-controlled, and reviewed in git; neither is silent — the audit +prints every declared removal with the ground that licensed it. An acknowledgement that matches nothing +fails the build, because a stale one would pre-clear a *future* disappearance of that same link. + +### 7. Launch gates + +None of these are follow-ups. + +- **Precision is proven by a hand-labelled sample, not by the control totals.** A human verifies against the + portal: every reconciled link, every refuted link, and random samples of „Документ" and „Потвърдено". + **Pass mark: zero wrong-company and zero wrong-person errors.** The control numbers of #279 §10 are a + reproducibility check, and two things had to be settled before they could serve as one. + + **The §5/§10 discrepancy is a dropped histogram category, not a disputed measurement.** §5's rungs carry + their own control counts: 4 barred + 281 document + 102 seat + 3 ЕИК + 21 refuted + 156 unknown = 567, + plus the 4 links whose ЕИК is not in the register at all (§5 scopes the ladder to links „с изтеглен акт", + so those reach no rung) = **571**. §10's identity row is 281 / 102 / 156 / 21 / 4 / 4 = **568**. The + difference is exactly the „3 по ЕИК". + + The cause is visible in the labels. The identity row names its second bucket **„седалище"** — one leg of + rung 3 — while the evidence row two lines below names the same rung **„потвърдено"** (251 / 78). Rung 3 + has two legs; the row tallied one and presented the result as a partition of the resolved set. And the + stated total 568 equals that row's sum exactly, which makes it a figure derived from the histogram rather + than measured independently. So both move together: the bucket becomes **„потвърдено: 105"** and the + resolved total becomes **571**. F8 measures against those and reports if its own count disagrees — the + reconciliation is not permitted to adopt whichever reading makes it pass. + + **Not a discrepancy, and not to be „fixed":** „извън ТР" is 4 in §10 and 3 in §3/§11 because the units + differ — 3 ЕИК that are not in the register, carrying 4 links between them. §10's row counts links. + + The numbers must also be re-measured on top of the §12 phantom-row fix, which changed the corpus — that + fix has landed as #281 (87 phantom rows across 15 sets; 256,286 announced − 87 = 256,199, covered exactly + by 255,582 fetched + 617 missing at source). +- **Every anti-false-zero control is a positive control.** „Sofia seat confirmations: 0" is indistinguishable + from a broken normalizer without one; so is a joint-stock bar with no marginal effect over + `closelyHeldForm`, and a matcher that always returns false (ADR-0027's lesson). +- **A partial cache must fail closed.** A missing *or* incomplete registry cache makes the loader throw. An + 80%-restored cache would otherwise yield roughly 80 published links — above the ship floor of 50 — and so + would ship a decimated surface and wipe the rest from production. The ship floor rises accordingly and is + passed explicitly rather than defaulted. +- **Crawl hygiene.** „Outside the register" is permanent only from a documented positive response; 429, + 5xx and timeouts are transient and are never cached as a negative. A 429 stops the run rather than + marking anything, and is never retried — „5 retries with growing backoff" and „429 stops the run" are + consistent only if retries exclude 429. Given Context 4, the crawl is sequential, paced, resumable, and + ends the run on the first 429 rather than backing off into it. **The deed's returned UIC must equal the + requested ЕИК**, or the deed is refused: an ЕИК is TEXT everywhere because public bodies' identifiers are + exactly the `000…` shape that loses its leading zeros to a numeric round-trip, and the failure mode of + losing them is fetching, and then publishing against, a different company's deed. +- **The methodology page carries the rule verbatim** (ADR-0021 E10): the ladder, the ≥3-token requirement, + the entity-boundary rule, the disclosure that identity rests on a name match **without ЕГН**, the homonym + and seat caveats, the lookup date, the refresh cadence and the retention. E10's existing promise that + held links „се показват едва след като регистърът стане достъпен" is finally kept by this change. +- **The PII rails are asserted, not assumed** — over the shipped dump and over the loader's own output. + +## Consequences + +- The surface roughly triples, and 264 links that are withheld today become named public claims. The + failure mode of the whole change is a **wrong-company** attribution, not a wrong publication decision. +- `tr-census.mjs` and the open-data census pipeline of ADR-0015 are removed. Its premise — that a name + proven nationally unique identifies the company — is the premise this ADR abandons. +- The pipeline gains a hard dependency on an external registry. Every path that lacked one before now has a + fail-closed branch: no cache, partial cache, unknown legal form, rate limit. +- We accept a slower, more fragile refresh in exchange for evidence. Decisions stay a pure, zero-network + function of declarations, cached deeds and contracts, but they no longer run on the 6-hourly ETL cycle + (#279 §9 assumes they do; they do not — the loader needs `node:sqlite`, the declaration corpus and the + full winner set, none of which exist inside a Worker). Cadence becomes two scheduled workflows: decisions + daily, registry lookups monthly. **Superseded by [ADR-0034](0034-registry-lookups-and-decisions-share-one-monthly-run.md):** + the split rests on the decision run being able to work from the cache alone, and it cannot — the strongest + rung compares the declarant's name against the deed text itself, which the index deliberately does not + store, so the raw deeds must be present; and they must not survive the runner. One monthly job, therefore. + Nothing else in this ADR is affected. +- Two capabilities are deliberately left on the table and become separate changes under new rules versions: + as-of evidence via `?entryDate=`, and the §7 „и към днешна дата" labels with their post-period contracts. +- The seat rung is structurally capped and we accept the cap: a declared seat is captured **only** from the + ООД/ЕООД holdings table of an *asset* declaration (`parse.mjs`), so it can never rescue a link declared + solely in an interests declaration. This is probably consistent with the 102 measured for the seat leg + specifically (not the 105 of the whole rung, whose other three come from the ЕИК leg) — rung 1 bars the + joint-stock cases anyway — but it is confirmed against the corpus before launch, not assumed. + +### Measured outcome — recorded on completion + +This ADR is accepted on the design. Exactly one amendment is permitted afterwards: this subsection is +filled in with the measured result, and nothing else in the file is rewritten (repo convention — an +accepted ADR is superseded, not edited). To be recorded: + +- the realized ladder split against the #279 §10 control row **as corrected above** (281 / 105 / 156 / 21 / + 4 / 4 = 571), with each number naming the SQL that produced it, and the gap between identified (281 + 105) + and surfaced (329) attributed bucket by bucket — 57 links on the corrected reading, 54 on the row as + written, so the figure itself distinguishes the two; +- the true candidate-ЕИК count (#279's „~400" is an assumption; the figure is `COUNT(DISTINCT eik)` over + all links, not just published ones); +- the hand-labelled sample result; +- the marginal effect of the registry joint-stock bar over `closelyHeldForm`, and the count of unknown + `legalForm` codes encountered. diff --git a/docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md b/docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md new file mode 100644 index 00000000..c9afd007 --- /dev/null +++ b/docs/adr/0034-registry-lookups-and-decisions-share-one-monthly-run.md @@ -0,0 +1,63 @@ +# ADR-0034 — Справките в ТР и решенията текат в едно месечно задание + +- **Статус:** Прието +- **Дата:** 2026-08-11 +- **Обхват:** конвейерът за свързани лица — `related-persons-data.yml`, `scripts/cacbg/load.mjs`, + `scripts/tr/fetch-deeds.mjs`. Заменя единствено твърдението за каденцията в последствията на + [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md); всичко останало в него стои. + +## Контекст + +ADR-0033 записа каденцията така: **„решения ежедневно, справки в регистъра месечно"**. Разделянето +почиваше на едно предположение — че решаващият ход може да работи само върху кеша, а обхождането е +отделен, по-рядък ход. Прегледът на PR-а показа, че това предположение не се държи. + +Двете страни на противоречието: + +1. **Суровите актове не бива да преживяват изпълнителя.** Кешираният акт носи имена на трети лица — + съдружници и управители без публична длъжност, и адреса на дружеството. Пренасянето му между ходове + изисква хранилище (кеш на Actions или артефакт), чието изгонване е политика за капацитет, не за + задържане: 35-дневният срок по решение 5 на ADR-0033 се изпълнява от `purgeExpired` на изпълнителя и + няма никаква власт над копие в хранилището на GitHub. Това беше находка B3 на прегледа. +2. **Решението не може да се вземе без суровите актове.** Най-силното стъпало сравнява трите имена на + декларатора със самия текст на записа. Индексът на кеша нарочно не пази нито едно име (§9), така че + „кеширай само индекса" премахва точно данните, от които `evidenceVerdict` се нуждае. + +Двете заедно значат: **всеки ход, който решава, трябва и да обхожда.** А ежедневно решение тогава значи +~400 заявки дневно към чужд публичен регистър, което спец. §3.3 („НИКОГА не обхождай ТР масово"; +КЗЛД, CJEU C-200/23) не разрешава да се приема с лека ръка — ограничителят на регистъра е единственият +начин операторът да изрази предпочитание за темп и ние не се настройваме около него емпирично. + +## Решение + +**Едно задание, месечно.** `related-persons-data.yml` изпълнява последователно: зареждане на списъка с +кандидати (`load.mjs --emit-candidates`, върху захвърляемо копие) → обхождане на регистъра → решение и +публикуване (`load.mjs`) → одит (`audit.mjs`) → изтриване на суровите актове в стъпка `if: always()`. +`scratch/tr` **не се кешира никога**; кешира се само `scratch/cacbg/raw` (публични декларации). + +`related-persons-tr-refresh.yml` е премахнат — нямаше какво да разделя. + +**Отхвърлена алтернатива: пренасяне на актовете между ходове** (кеш или артефакт с правило за +жизнен цикъл). Работи технически, но връща B3: срокът на задържане престава да е наш и става политика на +чуждо хранилище. R2 със собствено правило за изтриване остава разумен вариант, ако някога се наложи — +днес не се налага. + +**Отхвърлена алтернатива: ежедневно решение.** Единственият начин да се запази е обхождането да издава +**присъди по (връзка, ЕИК)** — булеви стойности без нито едно име — така че само те да пресичат +границата между ходовете. Това е коректната посока и остава на масата, но е промяна в дизайна на +доказателствения слой, не настройка на график, и не влиза в тази промяна. + +## Последствия + +- Публикуваната повърхност се преизчислява **месечно**, не ежедневно. Практическата цена е по-малка, + отколкото звучи: входът също се движи бавно (декларациите се подават годишно, вписванията в ТР — + рядко), а датата на справката се показва на всяка връзка, така че давността е видима за читателя, а не + подразбираща се. +- **Спешна корекция не чака графика.** Свалянето на връзка минава през ADR-0031 и ръчния ход, а при + жива експозиция — през пряк `UPDATE` върху обслужващата D1; вж. + [ръководството](../runbooks/related-persons-suppression.md). Каденцията определя кога се преоткриват + връзки, не кога може да се махне една. +- Едно задание значи един режим на отказ: ако обхождането спре при 429, гейтът за покритие на `load.mjs` + отказва частичния кеш и ходът не публикува нищо. Това е желаната посока — по-скоро без обновяване, + отколкото с орязана повърхност — и е същият гейт, който вече пазеше разделения вариант. +- #279 §9 продължава да описва ежедневни решения. Този ADR е записът, че не е така, и защо. diff --git a/docs/adr/0035-registry-evidence-must-also-establish-the-company.md b/docs/adr/0035-registry-evidence-must-also-establish-the-company.md new file mode 100644 index 00000000..4ba01f70 --- /dev/null +++ b/docs/adr/0035-registry-evidence-must-also-establish-the-company.md @@ -0,0 +1,81 @@ +# ADR-0035 — Регистровото доказателство трябва да установи и ДРУЖЕСТВОТО, не само лицето + +- **Статус:** Прието +- **Дата:** 2026-08-12 +- **Обхват:** доказателственото стъпало 2 („Документ") — `scripts/tr/evidence.mjs`, `scripts/cacbg/load.mjs`. + Заменя единствено твърдението в решение 2 на + [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md), че стъпало 2 не носи допълнителен + гейт; всичко останало в него стои. + +## Контекст + +ADR-0033 записа за стъпало 2: „по-силното стъпало не е гейтнато — регистърът е назовал това лице в ТОВА +дружество, което прави ключа по име без значение". Прегледът на PR-а показа, че изводът важи само ако +„ТОВА дружество" е дружеството, което лицето е декларирало. Не е гарантирано. + +Веригата, по която се стига до акта: + +1. `resolveEntity` (`load.mjs`) свежда декларираното фирмено наименование до **едно ЕИК на ПЕЧЕЛИВШ** — + участник в обществена поръчка. Дружества, които никога не са участвали, не съществуват за резолвера. +2. `nameGloballyUnique` проверява уникалност **само в множеството на участниците**, не в Търговския + регистър. „Единствено" тук значи „единственото сред печелившите". + +Оттук следва сценарият, който прегледът описа и който възпроизведохме: + +- Длъжностното лице действително притежава «АЛФА ООД» с ЕИК **A**, което никога не е участвало в поръчка. +- Друго, несвързано «АЛФА ООД» с ЕИК **B** е печеливш и единственият участник с това наименование. + Резолверът избира **B**. +- Актът на **B** съдържа **съименник** на декларатора — същите три имена, което в България не е рядкост. + Стъпало 2 съвпада и публикува връзка, **невярна и в двете си половини**. + +Регистровото доказателство доказва, че *някой с това име* е съдружник в **B** — не че *това лице* е. +Тоест стъпалото **премести** риска от съвпадение на имена в регистъра, вместо да го затвори, а точно +затварянето му е основанието на #279. Съществуващият карантинен механизъм за двусмислени ключове покрива +само сблъсък печеливш-срещу-печеливш; случаят печеливш-срещу-непечеливш не беше нито обработен, нито +записан като приет остатъчен риск. + +## Решение + +**Преди стъпало 2 да твърди, нещо извън фирменото наименование трябва да каже, че дружеството е +декларираното.** Съвпадението по име остава необходимо, но вече не е достатъчно само по себе си. + +Приема се, ако е изпълнено **поне едно** от: + +| Потвърдител | Защо стига | +|---|---| +| деклариран **ЕИК** | националният идентификатор по ЗТРРЮЛНЦ решава дружеството еднозначно, независимо от споделена фирма (ADR-0028) | +| декларирано **седалище**, съвпадащо с вписаното | близнак в друго населено място отпада; същият факт, на който публикува стъпало 3, включително темпоралната гаранция R10 | +| **отличително** фирмено наименование (`nameDistinctiveness`) | национален близнак е малко вероятен изобщо да съществува | + +Съвпадение по име, което не мине гейта, връща **нов, отделен въздържащ се резултат +`document_uncorroborated`** — никога не публикува, не носи роля и не носи `matched_fact`. + +**Защо отделен вид, а не пропадане към `unknown`.** „Съвпаднахме лице, но не установихме дружеството" и +„не съвпаднахме нищо" са различни факти за една връзка. Печатът се пише за всяка връзка, включително +задържаните, именно за да е прегледаема опашката (migration 0006); сливането им я заслепява точно там, +където решението има нужда от числото. + +**Третият потвърдител е граница, не доказателство.** Затова се **брои** (`document_uncorroborated` в +обобщението на зареждането) — това е числото, което F8 чете, за да реши дали гейтът да се стегне до +първите два. + +**Отхвърлена алтернатива: строго потвърждаване** (само ЕИК или седалище). По-висока точност, но +декларирано седалище се извлича само от таблицата ООД/ЕООД на **имуществена** декларация — никога от +декларация за интереси — така че цената в покритие е неизвестна преди F8 и можеше да свали повърхността +под прага за публикуване. Остава на масата и е точно решението, което измереният остатък ще информира. + +## Последствия + +- Стъпало 2 вече е **гейтнато**, което ADR-0033 решение 2 изрично казваше, че не е. Това е единственото + твърдение оттам, което този ADR заменя; наборът от стъпала, редът им и правилото „първото съвпадение + печели" стоят. +- Гейтът се проверява по подразбиране **затворено**: `companyNameDistinctive` е `false`, ако не бъде + подадено. За разлика от `nameGloballyUnique` — чието разрешаващо подразбиране е ограничено до + най-слабото стъпало — този гейт пази основното публикуващо стъпало. +- Стъпала 2 и 3 споделят **една** реализация на съвпадението по седалище (`matchDeclaredSeat`), заедно с + R10. Две копия на „какво брои за съвпадение по седалище" рано или късно се разминават за това кои връзки + могат да се публикуват. +- Покритието пада с неизмерена величина — по построение точно върху връзките, за които дружеството е + най-слабо установено. Това е приетата посока: по-скоро задържана вярна връзка, отколкото публикувана + невярна, назоваваща действително лице. +- Методологичната страница оповестява правилото дословно (ADR-0021 E10). diff --git a/docs/adr/README.md b/docs/adr/README.md index 0bd84ab6..0d63413f 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -16,15 +16,15 @@ | [0006](0006-eop-wins-dedup.md) | Dedup на два източника: EOP печели по `contract_number` | Прието | | [0007](0007-scope-and-certainty-bar.md) | Свързани лица: обхват и праг на сигурност (само 100% детерминистични съвпадения) | Прието | | [0008](0008-deterministic-name-to-eik-resolution.md) | Детерминистично разрешаване име→ЕИК (авто-публикуване; ръчна опашка само при двусмислие) | Прието | -| [0009](0009-name-uniqueness-guard-and-publish-tiers.md) | Пазач за уникалност на името + нива на публикуване (A_seat / B_distinctive / C_hold) | Прието | +| [0009](0009-name-uniqueness-guard-and-publish-tiers.md) | Пазач за уникалност на името + нива на публикуване (A_seat / B_distinctive / C_hold) | Заменено от [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) | | [0010](0010-pii-posture.md) | Позиция за лични данни: без ЕГН/адреси; третите лица — само за вътрешна проверка | Прието | | [0011](0011-host-scoped-tls-pinning.md) | TLS pinning само за хоста (счупена верига на register.cacbg.bg), не глобален байпас | Прието | | [0012](0012-crawler-and-persistence-architecture.md) | Архитектура на обхождането и съхранението (resumable; кеш по xml_file + ControlHash) | Прието | | [0013](0013-two-declaration-templates.md) | Два шаблона декларации (имущество + интереси) в един парсер | Прието | | [0014](0014-match-output-layers-and-interpretation.md) | Слоеве на съвпадението и тълкуване (собственост/контрол, времеви, собствена институция) | Прието | -| [0015](0015-tr-name-uniqueness-census.md) | Преброяване за уникалност на имена от ТР — промотира глобално уникални tier-C връзки | Прието | +| [0015](0015-tr-name-uniqueness-census.md) | Преброяване за уникалност на имена от ТР — промотира глобално уникални tier-C връзки | Заменено от [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) | | [0016](0016-free-text-entity-resolution.md) | Разрешаване на субекти от свободен текст (деклариран ЕИК + извличане от проза) | Прието | -| [0017](0017-name-collision-tier-gate.md) | Гейт срещу колизия на имена извън отличаващото ниво | Прието | +| [0017](0017-name-collision-tier-gate.md) | Гейт срещу колизия на имена извън отличаващото ниво | Заменено от [ADR-0033](0033-registry-evidence-replaces-name-distinctiveness.md) | | [0018](0018-folder-discovery-and-republication-dedup.md) | Откриване на папки от индекса + dedup на препубликувани декларации (ControlHash) | Прието | | [0019](0019-private-interest-vs-ex-officio-classification.md) | Разделяне на частен финансов интерес от служебни борд-роли (multi-declarant tell) | Прието | | [0020](0020-conflict-explorer-surface-posture.md) | Повърхност на експлорера — само interest_links, noindex до одобрение, произход на всеки ред | Прието | @@ -40,5 +40,8 @@ | [0030](0030-family-ownership-withheld-nameless-aggregate.md) | Дял на свързано лице не влиза в поименната повърхност (v1): събира се и се одитира, но се отчита само като безименен сбор (GDPR C-37/20; заменя ADR-0023) | Заменено от [ADR-0032](0032-family-ownership-published-under-public-interest.md) | | [0031](0031-suppressions-version-controlled-fingerprinted.md) | Свалянията на връзки са версиониран списък с HMAC-отпечатък (salt = CI secret), прилаган при зареждане — не служебна таблица; не изтича сигнала „кой е свален" към прод | Прието | | [0032](0032-family-ownership-published-under-public-interest.md) | Дял на свързано лице се публикува на поименната повърхност наравно със собствения — на основание надделяващ обществен интерес (ЗДОИ чл. 41и; C-184/20 забранява само поименните данни); близкият никога не се назовава, връзката не се твърди; заменя ADR-0030 | Прието | +| [0033](0033-registry-evidence-replaces-name-distinctiveness.md) | Връзка се публикува само срещу проверим факт от Търговския регистър — доказателственият ред заменя нивата по отличителност на името; евристиката вече обосновава твърдение (изрично изключение от ADR-0007), барът за акционерни форми е обединение от три сигнала, монотонността е гейт, не хранилище; заменя ADR-0009/0015/0017 | Прието (дизайн) | +| [0034](0034-registry-lookups-and-decisions-share-one-monthly-run.md) | Справките в ТР и решенията текат в едно месечно задание — суровите актове не преживяват изпълнителя, а решението не може да се вземе без тях; заменя твърдението за каденцията в ADR-0033 | Прието | +| [0035](0035-registry-evidence-must-also-establish-the-company.md) | Стъпало 2 („Документ") публикува само ако извън фирменото наименование нещо установи, че дружеството е декларираното — деклариран ЕИК, съвпадащо седалище или отличителна фирма; иначе `document_uncorroborated`, което се брои; заменя твърдението в ADR-0033, че стъпало 2 не е гейтнато | Прието | Свързан проектен документ: [spec/related-persons-foundation.md](../spec/related-persons-foundation.md). diff --git a/docs/core-scope.md b/docs/core-scope.md index ccf892fa..b1edd55d 100644 --- a/docs/core-scope.md +++ b/docs/core-scope.md @@ -58,7 +58,7 @@ | `review` | ≥10× процедурната оценка — задържан, но маркиран. | реалната EUR стойност | | `value_low` | Нула/отрицателна, или дребна (<1000 EUR **и** <5% от оценката) — задържан, но маркиран. | реалната EUR стойност | | `value_suspect` | >2 млрд. EUR, или >200× оценката (при оценка ≥1000 EUR) — **repair-ва се до процедурната оценка**. | процедурната оценка в EUR | -| `annex_suspect` | Анекс е вдигнал стойността ≥100× или до отрицателна — пада към `signing`/`current`. | `signing`/`current` в EUR | +| `annex_suspect` | Анекс е вдигнал стойността ≥100× или до отрицателна; или стъпка ≥10× при сбор ≥5× — пада към `signing`/`current`. | `signing`/`current` в EUR | > **Кога `amount_eur` е `NULL`** (и редът е изключен от сумите): когато няма надеждна EUR стойност — > чуждовалутен ред без покрит ECB курс (липсва latest-prior rate до 10 дни преди подписването), diff --git a/docs/deploy.md b/docs/deploy.md index ac3e845c..e296eda8 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -170,8 +170,17 @@ domain таблиците и преизчислява rollup-ите + FTS. > [scripts/precompute.sql](../scripts/precompute.sql) — не изпращайте FTS съдържание през dump. > Промени в схемата след първото зареждане се прилагат out-of-band, за всяка среда: -> `SIGMA_D1_NAME=sigma-stage wrangler d1 migrations apply sigma-stage --remote`. Деплоите не мигрират -> и не презареждат данни. +> `SIGMA_D1_NAME=sigma-stage wrangler d1 migrations apply sigma-stage --remote`. Деплоите **не презареждат +> данни**. +> +> **Уточнение (беше неточно):** тук пишеше и „деплоите не мигрират". Вече не е вярно — +> [deploy.yml](../.github/workflows/deploy.yml) прилага изрично `0003`, `0009` и `0010` с +> `d1 execute --file` при всеки деплой. Причината е, че ledger-ът на wrangler на тези бази е празен +> (базовата схема е създадена out-of-band), тъй че `d1 migrations apply` би се сблъскал с `0000`. И трите +> са безопасни за повторно прилагане: `CREATE … IF NOT EXISTS` и, при `0010`, тригери и индекси, които +> се пресъздават до същото състояние. Отделно от тях има и стъпки, които сондират таблицата и добавят +> само липсващите колони — за `0002` и за колоните на анексите. Останалите схемни промени наистина си +> остават out-of-band. ## 3. Конфигуриране на GitHub Environments diff --git a/docs/etl.md b/docs/etl.md index 6c624620..4f34a507 100644 --- a/docs/etl.md +++ b/docs/etl.md @@ -116,12 +116,27 @@ node scripts/load-eop.mjs --from=YYYY-MM-DD --to=YYYY-MM-DD --no-ocds Първоначалният backfill и ежедневният refresh ползват едни и същи staging таблици, mapper-и и SQL. Различават се по прозореца от дати и режима на derive: -- **голямо или първоначално догонване:** CLI прозорец + пълен derive; -- **малък steady-state refresh:** gap-aware прозорец + slice derive. +- **първоначално зареждане или пълно презареждане:** прозорец от началото на емисията + пълен derive; +- **догонване и steady-state refresh:** gap-aware прозорец + slice derive. `--derive=full` пуска amendment rollup, FX, NUTS, пълна нормализация и precompute. `--derive=slice` -пуска scoped refresh SQL-а. По подразбиране catch-up логиката избира full derive за големи -празнини и slice derive за малки. +пуска scoped refresh SQL-а. + +Пълният derive **презижда** доменните таблици от staging — `normalize-raw.sql` започва с +`DELETE FROM contracts` — тоест всичко извън заредения прозорец отпада и не се връща. Затова той е +допустим само когато прозорецът стига до началото на емисията (или когато още няма корпус). Понеже +gap-aware прозорецът по устройство покрива само опашката, `--catchup` върху **вече зареден** корпус +прави slice derive, колкото и голяма да е празнината. Изключението е първото пускане: когато няма +никакви заредени дни, догонването взима прозорец от началото на емисията и пълен derive — там няма +какво да се загуби. + +Пълен derive с частичен прозорец върху вече зареден корпус `import.mjs` отказва да продължи — преди +зареждането — вместо да изтрие историята. Отказът важи за **действащия** режим, не само за изрично +подадения: без `--catchup` подразбиращият се derive е `full`, тъй че и `import.mjs --from=2026-06-01` +получава същия отказ. Проверката пита за всяка таблица, която `normalize-raw.sql` изпразва (виж +`@full-clear` там), не само за `contracts` — корпус без договори, но с попълнени `tenders` или +`bidders` е точно състоянието, което половинчат пробег оставя. Ако някоя от тях не може да бъде +прочетена, отказът пак важи: проверка, която не може да провери, не бива да пуска нататък. ## Доменна нормализация (derive) @@ -215,6 +230,14 @@ CZK). `normalize-raw.sql` пази нативната записана стой присвоява verdict `value_flag` и безопасната за сумиране `amount_eur` на всеки доменен договор. Всички прагове се сравняват в евро след FX нормализация. +> **Изключение — prefer-EOP dedup на анексите (#286).** `derive-amendments.sql` / `refresh-slice.sql` +> **изтриват** OCDS редове от `raw_amendments`, когато EOP близнак покрива същия `(unp, contract_number)`. +> Това е единственият пазач срещу двойно броене, защото промоцията в `amendments` е безусловна — затова +> DELETE-ът, а не read-time филтър. Инвариантът е записан и се налага от integrity проверката +> `amendment-twin-dedup` (`scripts/integrity-checks.mjs`): нито една двойка `(unp, contract_number)` не +> носи едновременно EOP и OCDS сервиран ред. По slice пътя близнакът може да се раздели между прозорци, +> затова там dedup-ът сверява и кумулативната сервирана таблица (виж коментарите в `refresh-slice.sql`). + Verdict-ът се определя от две прогнозни стойности, като всеки флаг ползва тази, която избягва *собствената* си посока на false positive: @@ -230,8 +253,22 @@ Verdict-ът се определя от две прогнозни стойнос CASE-ът се оценява по ред; печели първото съвпадение: -1. **`value_suspect`** — записана стойност неправдоподобно висока: `eff > 2 000 000 000` **или** - (`procEst >= 1000` **и** `eff > 200 * procEst`). Редът се **поправя, не се хвърля**: `amount_eur` +1. **`value_suspect`** — записана стойност неправдоподобно висока. Три отделни повода, всеки от + които стига сам по себе си: + - `eff > 2 000 000 000` — извън всякакъв мащаб за български договор; + - `procEst >= 1000` **и** `eff > 200 * procEst` — общият праг за неправдоподобно надвишаване; + - **стотинки лента** (#247, #298): пропуснат десетичен знак вдига стойността точно ~100 пъти. + Хваща се на две места, защото грешката може да е спрямо коя да е от двете прогнози: + - спрямо **процедурната**: `procEst >= 1000` **и** `95 * procEst <= eff <= 105 * procEst`; + - спрямо **прогнозата по позиция**: `ownEst >= 1000` **и** `procEst >= 1000` **и** + `eff >= 10 * procEst` **и** `95 * ownEst <= eff <= 105 * ownEst`. + + Лентата е тясна нарочно — измерването върху корпуса показва изолирано струпване при ~100x и + нула договора между 105x и 200x. Второто рамо иска и `eff >= 10 * procEst`, защото при рамкови + и единични цени прогнозата по позиция е ЕДИНИЧНА цена и цял call-off законно я надхвърля + стократно; условието за процедурата отсява точно тези случаи. + + Редът се **поправя, не се хвърля**: `amount_eur` става `procEst` (собственият документиран бюджет на поръчката), а показваната нативна сума става нативната процедурна прогноза. Пада до NULL (изключва се) само когато процедурата няма прогноза, от която да се поправи. Guard-ът `procEst >= 1000` пази редовете, чиято *прогноза* е грешката @@ -243,11 +280,17 @@ CASE-ът се оценява по ред; печели първото съвп големите легитимни рамкови call-off-и (малък дял от огромен таван, но голям в абсолютна стойност) извън този флаг. 3. **`annex_suspect`** — анекс е тласнал `current_value` до отрицателна или до `>= 100x` подписаната - стойност, докато подписаната е разумна. Договорът **се връща към подписаната стойност** за - `amount_eur`; надутата текуща стойност се потиска. -4. **`review`** — надхвърляне в сива зона: `eff >= 10 * procEst`. Запазен, флагнат и **брои се по + стойност, докато подписаната е разумна; или единична анексна стъпка е скочила `>= 10x`, а сборът е + свършил `>= 5x` над подписаната (сбъркано число в анекс — само стъпката не стига, защото има + вериги, при които по-късен анекс сваля грешката обратно под подписаната стойност). Договорът + **се връща към подписаната стойност** за `amount_eur`; надутата текуща стойност се потиска. +4. **`annex_total_suspect`** (#305) — водещият анекс е удвоил договора в една стъпка (ЗОП чл. 116 + ограничава единично изменение до +50%), а текстът на основанието не дава сигнал, по който + стойността да се поправи. Договорът **се връща към подписаната стойност**, както при + `annex_suspect`. +5. **`review`** — надхвърляне в сива зона: `eff >= 10 * procEst`. Запазен, флагнат и **брои се по face value**. -5. **`ok`** — всичко останало, брои се по `eff`. +6. **`ok`** — всичко останало, брои се по `eff`. Водещият принцип е **поправка пред изключване**: само наистина невъзстановими редове напускат тоталите. Където записаната стойност е недвусмислен боклук, заместваме с най-добрия документиран @@ -286,11 +329,51 @@ cross-check показват изпуснати десетични запета 6. пуска `refresh-slice.sql`; 7. пуска reconciliation gate-а (#97, `runIntegrityChecks`) върху обслужвания D1 в стъпка `integrity-gate` и проваля стъпката при дрейф (виж [`integrity-gate.md`](integrity-gate.md)). - FX-проверката нарочно **не** е включена тук — Worker-ът не зарежда курсове (проследено в #154). + FX-проверката нарочно **не** е включена тук (проследено в #154). Старата обосновка „Worker-ът не + зарежда курсове" вече не важи — от #158 стъпката `load-fx` тегли липсващите курсове преди derive-а. Worker-ът е нарочно ограничен до малък скорошен прозорец. Ако базата е далеч назад, логва, че прозорецът е capped, и оставя голямото догонване на CLI-то (`pnpm run import --catchup`). +### Лъжливата грешка „hung" при всеки успешен цикъл + +Всяка **успешно завършила** инстанция на Workflow-а оставя и запис от ниво `error`: + +> The Workers runtime canceled this request because it detected that your Worker's code had hung +> and would never generate a response. + +Тя пада на същата милисекунда, на която инстанцията получава своя `end` — тоест **след** като +всички стъпки са върнали `success` и изходът е записан. Извикването се отчита с +`outcome: exception`, докато самата инстанция е `status: complete`. Нищо не се губи. + +Измерено на `sigma-etl-stage` (2026-08-06) с три инстанции, при които варирахме всичко, което е +в наши ръце: + +| Пуск | Стъпки | Време | Неизчерпани тела на отговори | „hung" | +| --- | --- | --- | --- | --- | +| крон, прозорец от 6 дни | 30 | 91 s | да | да | +| празен прозорец (`today` в бъдещето) | 5 | 1,6 s | да (един 403) | да | +| един пълен ден | 30 | 26 s | нула | да | + +Продължителност, брой стъпки, обем към D1, изход от четенията и кой `return` се ползва — грешката +е неизменна спрямо всяко от тях. Затова към днешна дата я третираме като страничен продукт на +средата (`executionModel: "stateless"`), а не като дефект в нашия код. + +**Това е измерване, не присъда.** Обходът на четенията, направен заедно с тези опити, пропусна +един жив теч в `packages/ingest/src/fx.ts` — тоест „проверихме всичко наше" не се оказа вярно. +Отваряйте въпроса наново, ако: грешката спре да пада точно на `end` времето; появи се инстанция +със `status: complete`, но с непълни стъпки; или се намери в наш код изчакване, което не приключва. + +**Следствие за наблюдението:** „няма грешки в таблото" не е показател за здраве на този Worker. +Здравето се чете от `etl_refresh_complete` — записът, който `run()` оставя след успешното +изчистване на преходния staging. Точната истина винаги може да се провери и през +`GET /accounts/{acct}/workflows/{name}/instances/{id}`, който дава списъка със стъпките. + +Предупреждението „An RPC result was not disposed properly" се появява само при крон-цикли с +многодневен прозорец и не се възпроизведе при контролираните еднодневни пускания. В `apps/etl` и +`packages/ingest` няма нищо, което държи RPC резултат, дължащ `dispose` — тоест източникът е под +нас, в свързването. + **Текущо ограничение на Worker-а:** той стейджва само вложения OCDS файл. Coercion-ът на плоския базов JSON все още живее в CLI loader-а и трябва да се извади в споделени Worker-safe helper-и, преди Worker-ът да стейджва и базови contracts, tenders и annexes. Големите backfill-и остават diff --git a/docs/implementation-plans/286-ocds-amendment-unp.md b/docs/implementation-plans/286-ocds-amendment-unp.md new file mode 100644 index 00000000..e21755d4 --- /dev/null +++ b/docs/implementation-plans/286-ocds-amendment-unp.md @@ -0,0 +1,179 @@ +# Implementation Plan: #286 — OCDS amendments don't link to any contract (OCID instead of УНП) + +## Executive Summary + +| Field | Value | +|---|---| +| Ticket | [midt-bg/sigma#286](https://github.com/midt-bg/sigma/issues/286) — labels `data-quality`, `etl`, `priority: high` | +| Problem | OCDS-sourced rows in `amendments` store the **OCID** (`ocds-e82gsb-245534`) in `unp` instead of the **УНП** (`00044-2022-0146`), so **none** of them join to a contract. Measured live on `sigma-dev`: 4,800 OCDS rows, **0 linked**. | +| Root cause | `packages/ingest/src/ocds.ts` sets `unp: rel.ocid`. The УНП is **not present anywhere structured** in the OCDS feed — it must be recovered by bridging OCDS `tender.id` → EOP `tenderId` (`raw_tenders.tender_id` / `tenders.eop_tender_id`) → УНП (`source_id`). | +| Approach | (1) Ingest captures `tender_ext_id` + fixes value semantics; (2) SQL bridge rewrites the OCDS `unp` from the existing lots-bridge pattern; (3) prefer-EOP dedup so twins don't double-count and the value trap can't fire. | +| Complexity | Medium — small code, but the correctness traps (value semantics + cross-source dedup) are the whole point. | +| Risk | Medium — touches the value pipeline that feeds headline `current_value`; a naive fix understates contracts by millions (issue's €5.8M case). Mitigated by parts 2+3 shipping together and a real-corpus before/after. | +| Status | **Draft — validated against the live `sigma-dev` corpus.** | + +> **The two halves are inseparable.** Fixing the join key *without* fixing the OCDS value semantics is worse than the bug: once linked, the stale OCDS "before" value competes with the correct EOP value on the same `published_at` and the tiebreak can pick it. Key-alignment **and** value-correctness land in the same PR. + +--- + +## 1. Problem, verified on real data + +The УНП is genuinely absent from the OCDS releases. Pulling the real feed +`storage.eop.bg/open-data-2026-03-05/…OCDS.json` and walking all 28 +`contractAmendment` releases: the `NNNNN-YYYY-NNNN` УНП pattern appears **zero +times** as a structured field (once, incidentally, in free-text `rationale`). A +real release carries only: + +- `ocid: "ocds-e82gsb-425867"` and `tender.id: "425867"` (the EOP internal + procedure id = the OCID suffix) +- `contracts[].id: "180821"` (the contract number) +- `buyer.identifier.id: "000024948"` (authority EIK) +- `contracts[].value` — the **pre-amendment** value + +The EOP base "договори" feed, by contrast, carries **both** join anchors: +`uniqueProcurementNumber` (→ `unp`, the УНП) **and** `tenderId` (→ +`tender_ext_id`, the same id space as OCDS `tender.id`). + +### Reproduction on `sigma-dev` (live) + +| check | count | +|---|---| +| `amendments` rows with `source LIKE 'ocds:%'` | 4,800 | +| …with `unp LIKE 'ocds-%'` | 4,800 (all) | +| …that link via `contracts.tender_id = 't:' \|\| unp` | **0** | +| `eop:%` rows that link | 174,635 | + +### The value trap (contract 90029, live) + +| source | unp | value_before | value_after | published_at | +|---|---|---|---|---| +| `eop:annexes` | `00044-2022-0146` | 21,602,081.98 | **27,435,415.31** | 2026-03-05 | +| `ocds:` | `ocds-e82gsb-245534` | null | **21,602,081.98** | 2026-03-05 | + +`ocds.ts:378-379` writes `value_after = c.value.amount` (the *before* value) and +`value_before = null`. `derive-amendments.sql:42-50` selects `current_value` as +the latest non-null `value_after` with `ORDER BY published_at DESC, natural_key +DESC`. Same date ⇒ the tiebreak can pick OCDS's 21.6M and drop €5.8M. Twin +samples confirm OCDS values are generally stale/partial (e.g. contract 188980: +OCDS 111,163 vs EOP 217,416), so the OCDS value is never trustworthy as an +"after". + +--- + +## 2. Why the bridge is the right recovery (proven in-repo + on data) + +The exact bridge already exists for OCDS **lots** — +`scripts/normalize-raw.sql:1110-1133`: + +``` +-- The bridge is OCDS tender.id -> EOP tenderId (raw_tenders.tender_id) -> UNP -> domain lots. +-- ocid is a surrogate and is never treated as the UNP. +JOIN raw_tenders rt ON rt.tender_id = rl.tender_id +``` + +OCDS **amendments** never got the same treatment: `releaseToAmendments` doesn't +read `rel.tender.id`, even though `raw_amendments.tender_ext_id` already exists +as a column (`work-staging-schema.sql:183`) and EOP amendments populate it +(`base.ts:300`). + +Read-only validation of the bridge against `sigma-dev` (all OCDS `unp` are +`ocds-e82gsb-`; `tenders.eop_tender_id` maps to `source_id`/УНП): + +| step | count | +|---|---| +| OCDS amendments whose `tender.id` bridges to a tender (УНП recovered) | 4,797 / 4,800 | +| …that link to a contract with matching `contract_number` | **4,782** | +| …that have an **EOP twin** for the same `(unp, contract_number)` | 4,741 (~99%) | +| genuinely **OCDS-only** annexes (net-new) | ~41 | + +So ~99% of OCDS amendments duplicate an existing EOP annex. Linking them without +dedup would ~double `annex_count` on those contracts and arm the value trap. +Per-annex twin matching is unreliable (twin dates and values differ), so dedup is +done at the contract level: **prefer EOP; use OCDS only where EOP has no annex.** + +--- + +## 3. The fix (three parts, one PR) + +> **Shipped implementation — where it diverged from the plan below (this section is the original +> planning snapshot).** Part 2's bridge landed in `scripts/derive-amendments.sql` (mirrored in +> `scripts/refresh-slice.sql`), **not** `normalize-raw.sql` — `derive-amendments.sql` is the *first* +> consumer of `raw_amendments` on every path, so the rewrite has to happen there or its own rollup still +> joins on the OCID. Part 3's dedup shipped as a `DELETE FROM raw_amendments` (not an inline +> `WHERE NOT EXISTS` filter), and `promote-amendments.sql` is unchanged. On the incremental path the +> slice dedup additionally reconciles against the cumulative served `amendments` table (full path rebuilds +> it wholesale, so it needs no equivalent). See the review thread on the PR for the reasoning. + +### Part 1 — Ingest (`packages/ingest/src/ocds.ts`, `apps/etl` re-exports) +- Add `tender_ext_id` to `AmendmentStagingRow` and `AMENDMENT_STAGING_COLS`. +- In `releaseToAmendments`, set `tender_ext_id: clean(rel.tender?.id)`. +- Value semantics: store the release value as `value_before`; leave + `value_after = null`. OCDS cannot know the after-value, so it must never drive + `current_value`. (Verify `contractUpdate` separately before treating it the + same as `contractAmendment`; default to the conservative before-only mapping.) +- Unit tests updated in `packages/ingest/src/ocds.test.ts`. + +### Part 2 — SQL bridge (`scripts/normalize-raw.sql`) +- Before `derive-amendments.sql` / `promote-amendments.sql`, rewrite OCDS + amendment `unp` from the bridge, mirroring the lots block: + ```sql + UPDATE raw_amendments + SET unp = ( + SELECT rt.unp FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL + ) + WHERE source LIKE 'ocds:%' + AND tender_ext_id IS NOT NULL + AND EXISTS (SELECT 1 FROM raw_tenders rt WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL); + ``` + Fallback to `raw_contracts.tender_ext_id` where a tender row is absent. `ocid` + stays only as a surrogate, never a key. Mirror the same step in + `scripts/refresh-slice.sql` (the scoped slice path). + +### Part 3 — Prefer-EOP dedup (`derive-amendments.sql` + `promote-amendments.sql`) +- Include an OCDS amendment only when no EOP amendment exists for the same + `(unp, contract_number)`: + ```sql + AND NOT (source LIKE 'ocds:%' AND EXISTS ( + SELECT 1 FROM raw_amendments e + WHERE e.source LIKE 'eop:%' AND e.unp = raw_amendments.unp + AND e.contract_number = raw_amendments.contract_number)) + ``` + This kills the double-count and the value trap in one move while still + surfacing the ~41 OCDS-only annexes. OCDS-only rows keep `value_after = null`, + so they increment `annex_count` and become visible/linked without inventing a + `current_value`. + +--- + +## 4. Testing — proving we solved the real problem + +1. **Unit** (`packages/ingest/src/ocds.test.ts`): realistic fixture with + `tender.id` + value asserts `tender_ext_id` captured, `value_before` set, + `value_after` null. +2. **End-to-end SQL** (`packages/db/src/refresh-slice.test.ts` harness runs the + *actual* `normalize-raw → derive-amendments → promote-amendments` scripts via + the sqlite3 CLI). New case seeds: + - an EOP tender+contract (УНП `X`, `tenderId T`) with one EOP annex, + - an OCDS twin (`ocds-…-T`, same `contract_number`), + - an OCDS-only annex (`tenderId T2`, no EOP annex). + Assert: OCDS-only links & counts once; twin is dropped; the 90029 scenario + keeps `current_value = 27.4M` (never 21.6M). +3. **Repro regression**: encode the issue's SQL (`ocds:%` linked rows > 0, + no contract understated) as assertions. +4. **Real-corpus before/after**: rebuild a local slice from the public feeds and + run the repro queries; optionally dry-run-validate the bridge against + `sigma-dev` read-only. Expected: linked OCDS rows 0 → ~41 net (4,782 matched, + 4,741 deduped), zero contracts understated. + +--- + +## 5. Scope boundaries + +- **Out of scope:** OCDS *contracts* also store the OCID in `unp` + (`ocds.ts:323`); whether they reach the served domain at all is a separate + question — note it, don't fold it in. +- **Adjacent:** #248 (annex plausibility) and PR #285 (`value_delta` sign) touch + the same table but are independent defects. +- **Migration:** none — `raw_amendments.tender_ext_id` already exists; the change + is ingest + ETL SQL + tests only. diff --git a/docs/implementation-plans/305-amendment-value-double-count.md b/docs/implementation-plans/305-amendment-value-double-count.md new file mode 100644 index 00000000..7c67c492 --- /dev/null +++ b/docs/implementation-plans/305-amendment-value-double-count.md @@ -0,0 +1,119 @@ +# Implementation Plan: #305 — Amendment value double-count (a new *total* is added to the old value) + +## Executive Summary + +| Field | Value | +|---|---| +| Ticket | [midt-bg/sigma#305](https://github.com/midt-bg/sigma/issues/305) — labels `data-quality`, `etl`, `priority: high` | +| Problem | When an EOP annex announces a new **total** contract value, ЦАИС ЕОП puts that total in the *change* field, so `currentContractValue = lastContractValue + newTotal`. Sigma stores `value_after` verbatim, so the served value is **doubled** (the old value is counted twice). | +| Root cause | **Source data defect, faithfully stored.** `base.ts:313-315` maps `value_before ← lastContractValue`, `value_after ← currentContractValue`, `value_delta ← contractValueDifference` with **no arithmetic**. The bug is that, for a subset of annexes, `contractValueDifference` (→ `value_delta`) holds the **new total**, not the increment — and the feed's `currentContractValue` is `before + that total`. Verified: `value_after = value_before + value_delta` on **100%** of delta-carrying rows on `sigma-dev`. | +| Why existing flags miss it | A double-count is only ~2× signed value. `value_flag = 'annex_suspect'` needs ≥5× aggregate (+ a ≥10× per-step); #299 the same; the estimate-based flags need ≥10×/≥200×. 2× < 5× ⇒ classed **`ok`** ⇒ enters the `amount_eur` canonical value base and inflates every rollup. (`normalize-raw.sql:882-948`.) | +| Scale (real corpus) | On the issue's 2020→2026 local rebuild: 7,335 price-raising annexes, **686 at ≥100% growth**, ~666 unflagged, **€475.4M**. Independently reproduced on `sigma-dev`: **686 at ≥100% growth**, ~526 unflagged. The named records (145652, 189325, 84818) confirm. | +| Complexity | Medium. Detection is the hard part (a source-text heuristic with false-positive risk); the plumbing (flag + exclude, then optional correct) mirrors the existing `annex_suspect` machinery. | +| Risk | Medium. A naive correction that trusts a text heuristic can mis-restate genuine >100% increases; a flag-only tier is safe and ships first. The signature has a **blind spot** (a new-total *lower* than the old value hides below +100%), so the fix must not be sold as complete. | +| Status | **Draft — investigated with real DB calls + code trace.** | + +> **Two independently-true facts frame the fix.** (1) The source is internally *consistent* — `value_after = value_before + value_delta` always — so the correction can be expressed purely as *"when `value_delta` is a **total**, the true `value_after` is `value_delta`, not `value_before + value_delta`."* (2) The defect is `value_flag = 'ok'`, so it is inside the aggregated value base; the minimal safe fix is to move it *out* of that base (a new verdict), exactly as #299 did for its case. + +--- + +## 1. Problem, verified on real data + +### 1a. The mechanism (code, on `main`) +- `packages/ingest/src/base.ts:313-315` — EOP annexes map three source keys straight to columns, no math: + - `value_before ← lastContractValue` + - `value_after ← currentContractValue` + - `value_delta ← contractValueDifference` (the only signed field) +- `scripts/derive-amendments.sql:147-155` (mirrored in `scripts/normalize-raw.sql:752-759`): `contracts.current_value` = the latest amendment's non-null `value_after`. So the doubled `value_after` becomes the served contract value. +- `scripts/promote-amendments.sql:42-44`: `value_before/after/delta` copied verbatim into served `amendments`. + +**Conclusion:** the doubled figure is not computed by Sigma; it arrives in `currentContractValue`. For the affected annexes the authority entered the **new total** into `contractValueDifference`, and the feed's `currentContractValue = lastContractValue + newTotal`. Sigma stores it faithfully. + +### 1b. The named records (queried on `sigma-dev`, read-only) + +| contract | source | value_before | value_after | value_delta | doubled? | +|---|---|---|---|---|---| +| 145652 (УНП 00010-2023-0006) | `eop:annexes:2024-06-21` | 442,000 | **981,240** | 539,240 | yes — `value_delta` (539,240) is the announced new total; true `value_after` = 539,240 | +| 189325 (УНП 00210-2024-0024) | `eop:annexes:2025-10-07` | 77,000,000 | **154,000,000** | 77,000,000 | yes — exact 2× (currency-change annex) | +| 84818 (УНП 00080-2023-0001) | `eop:annexes:2026-07-17` (EUR) | 76,769,540.87 | **153,539,081.74** | 76,769,540.87 | yes — exact 2× | + +Caveats found in verification: 84818 has **6** amendment rows (only the 2026-07-17 EUR annex is the doubled one — earlier BGN annexes are consistent); the issue treated it as one. Absolute counts differ from the issue because `sigma-dev` (31,543 amendments) is a superset of the issue's local rebuild (26,921). + +### 1c. The math signature and its blind spot +- `value_delta = value_after − value_before` holds on **100%** of delta-carrying rows (`sigma-dev`), i.e. the source is self-consistent. The defect is semantic: `value_delta` is sometimes a *total*, not an *increment*. +- A double-count where `newTotal ≥ before` produces growth **≥ 100%** (686 rows). The exactly-+100% subset (`value_after = 2×before`) is the "same total re-stated" / currency-change case. +- **Blind spot:** if `newTotal < before`, the double-count yields growth **< 100%** and hides among clean rows. The +100% line is a *safety threshold*, not a proof of cleanliness — the fix must say so. + +--- + +## 2. Why the existing flags don't catch it + +`scripts/normalize-raw.sql:882-948` (mirrored in `refresh-slice.sql`), evaluated top-down against `eff_eur = EUR(COALESCE(current_value, signing_value))`: +- `value_suspect`: `eff_eur > 2e9`, or `> 200 × proc_est_eur`, or the стотинки band — estimate-relative, ignores a 2× overrun. +- `annex_suspect` (`:937-945`, the #299/#248 rule): `current_value/signing_value ≥ 100`, **or** (`≥ 5` **and** a per-step `value_after ≥ 10 × value_before`). A double-count is ~2× signed ⇒ below 5×. +- `review`: `eff_eur ≥ 10 × proc_est_eur`. +- else `ok`. + +A ~2× inflation clears none of these gates → `ok` → `amount_eur` takes `COALESCE(current_value, signing_value)` (`normalize-raw.sql:818-830`), so the doubled value is summed everywhere. Pinned by `packages/db/src/value-flag-annex-step-sql.test.ts` (the 5× floor is the smallest firing case; 4.9× stays `ok`). + +**Downstream consumers currently inflated** (all via the shared `amount_eur` base — `precompute.sql:16-19`): contract-list totals/sort/buckets and CSV export (`queries/contracts.ts:318,61-62,432,475`); `company_totals.won_eur`, `authority_totals.spent_eur`, `home_totals.value_eur` (`precompute.sql:42-60`); the contract detail value strip and the **amendment timeline**, which reads `amendments.value_after` unrepaired (`queries/details.ts:451-460,670-684`). *(The `/anomalies` #239 and `/overruns` #171 signals named in the issue are not on `main` yet — they will inherit the fix once they land.)* + +--- + +## 3. Detection + +The correction hinges on classifying each price-raising annex as **increment** vs **total**. Layer the signals; never rely on free text alone. + +1. **Arithmetic gate (necessary, cheap, high-recall / low-precision):** `value_before > 0 AND value_after ≥ 2 × value_before` (equivalently `value_delta ≥ value_before`). A single annex whose *increment* is ≥ the entire prior value is implausible; a double-count always lands here. Catches the 686. Does **not** catch the sub-100% blind spot (out of scope for v1, documented). +2. **Text confirmation (raises precision):** the основание free-text carries a number equal to `value_delta` in a *total* context — keywords `обща|общата|крайна|краен|възлиза|става` near the figure (the issue found 355 such records). Available fields at the raw/derive stage: `raw_amendments.description` (`changeDescription`), `reason` (`changeReason`), `circumstances` (`changeReasonDescription`) — see §5 note. Parse Bulgarian number formats (`1 234 567,89` / `1234567.89`), compare to `value_delta` within a small relative tolerance. +3. **Currency-change tell (special-case, very high precision):** exactly-+100% rows whose text mentions `евро|валута|EUR|лева в евро` are currency re-denominations with the total doubled (e.g. 189325). Treat as confirmed total. + +Classification: +- **Confirmed total** = gate (1) AND (text (2) or (3)). → correct (Tier 2) and/or flag. +- **Suspected total** = gate (1) only (no text confirmation). → flag-only (Tier 1); do not silently rewrite the value. + +--- + +## 4. The fix + +Two tiers. Ship Tier 1 first (safe, immediate); Tier 2 is the higher-value correction and needs the text heuristic hardened by tests. + +### Tier 1 — Flag and exclude from aggregates (minimal, safe, ships first) +Mirror the `annex_suspect` machinery so the ~475M/€ inflation leaves every rollup immediately, without trusting any text parse. +- Add a new `value_flag` verdict, e.g. **`annex_total_suspect`**, in `scripts/normalize-raw.sql` (and the `refresh-slice.sql` mirror + its reconciliation re-flag), placed **before** the `ELSE 'ok'`: fires when the contract's current-value-driving annex satisfies the arithmetic gate (§3.1) and the source-consistency check (`value_after ≈ value_before + value_delta`). +- Route it through the existing suspect fallback: `amount_eur`/`trusted_native` fall back to `signing_value` (`normalize-raw.sql:818-830`) and `precompute.sql:36` NULLs `current_value_eur` — so these contracts drop out of totals/CSV/pages exactly like `annex_suspect`. +- Emit a diagnostic count (like #286's diagnostics) so the flagged volume is observable in the ETL log. +- **Per-amendment flag (the issue's specific gap):** the contract flag does not fix the timeline row. Add a per-amendment marker so `queries/details.ts` can render the row as "suspected re-stated total" and suppress its `+%`. Options: a `value_flag`/`total_restated` column on served `amendments` (schema migration + carry through `promote-amendments.sql`), or recompute the same predicate in the details query. Prefer the column (single source of truth, avoids duplicating the heuristic in TS). + +### Tier 2 — Correct the value (higher value, needs text confirmation) +For **confirmed totals** (§3), restate the amendment at the **raw/derive stage** (where `value_before`, `value_delta`, and all three text fields coexist — see §5): +- Because the source is self-consistent, the correction is simply **`value_after := value_delta`** (the announced new total) and **`value_delta := value_after_old − value_before`** *no* — restate as: `corrected_after = value_delta_source` (the total); `corrected_delta = corrected_after − value_before` (the true increase). Keep the raw source values immutable in `raw_amendments`; write corrected values on the way to served `amendments` (a `promote`/derive transform), plus a `total_restated = 1` marker. +- `contracts.current_value` then derives from the corrected `value_after`, so headline value, deltas, and the timeline are all right — not merely excluded. +- Keep Tier-1 flagging for the **suspected-but-unconfirmed** set (gate only, no text) so nothing inflates while remaining un-restated. + +### Fix location (decided) +Raw/derive stage, **not** ingest and **not** the served query layer: +- Ingest (`base.ts`) must stay a faithful mirror of the source (per `docs/etl.md` non-destructive-staging stance) — do not mutate `raw_amendments`. +- The correction/flag needs `value_before`, `value_delta`, and the основание text on one row, which is true in `raw_amendments` and consumed by `derive-amendments.sql` / `normalize-raw.sql` / `promote-amendments.sql`. Implement there; keep `derive-amendments.sql` and `refresh-slice.sql` in lockstep (a drift guard already exists for the #286 bridge block — extend the pattern). + +--- + +## 5. Schema / data note (blocking for Tier 2) +Only `description` survives to served `amendments`; `reason` and `circumstances` are dropped at `promote-amendments.sql:9-11` (served DDL `0000_init.sql:169-182`). The text heuristic therefore must run at the **raw/derive** stage where all three exist (`work-staging-schema.sql:178-180`). If any per-row flag or corrected value must be *visible* to the app, add the column(s) to the served `amendments` table (migration) and carry them through `promote-amendments.sql` + the `refresh-slice.sql` amendments promotion. + +--- + +## 6. Testing — proving the fix +1. **Unit — number/keyword parser** (`packages/ingest` or a new `packages/db` SQL-driven test): Bulgarian number formats; total-context keywords vs increment phrasing; the currency-change tell. Fixtures from the real examples (145652 "възлезе на 539 240.00 лв."; 189325 currency change; a genuine >100% *increment* that must NOT be corrected). +2. **End-to-end SQL** (extend `packages/db/src/refresh-slice.test.ts` / a new `amendments-total-suspect.test.ts`): run the real `derive-amendments.sql → normalize-raw.sql → promote-amendments.sql` and assert: (a) a confirmed-total annex is restated (`value_after = value_delta`, `current_value` correct, `total_restated = 1`); (b) a suspected-only annex is flagged `annex_total_suspect` and excluded from `amount_eur`; (c) a genuine >100% increment stays `ok` and untouched (guard against false positives); (d) the exactly-+100% currency case restates to no real growth. +3. **Flag-coverage regression**: assert the new verdict count on a seeded corpus, and that `value-flag-annex-step-sql.test.ts`'s existing cases are unchanged. +4. **Real-corpus before/after**: rebuild the 2020→2026 corpus (local work-DB, then optionally ship), and report flagged/corrected counts + the EUR removed from `company_totals`/`authority_totals`/`home_totals`. Target: the 686 (≥100%) restated or flagged, headline totals drop by the double-counted amount, zero genuine-increment regressions. + +--- + +## 7. Scope boundaries & risks +- **In scope:** EOP annexes (the sole driver — OCDS rows carry `value_after = NULL` and never drive `current_value`, per #286). +- **Blind spot (out of scope for v1, must be documented):** double-counts where the new total is *lower* than the old value (growth < 100%) — not detectable by the arithmetic gate. Text-only detection could reach them but at higher false-positive cost; defer. +- **False-positive risk (the main hazard):** a genuine annex that legitimately more-than-doubles a contract. Tier 1 only *flags* (reversible, excludes from aggregates); Tier 2 *rewrites* and must require text confirmation + be unit-tested against a real >100%-increment fixture. When uncertain, prefer flag over rewrite. +- **Adjacent / not this:** #299 (`c44a7ee`) and #248 handle the ≥10×/≥5× mis-keying case (kept); #245 (EUR double-conversion) and #304/#247 (стотинки) are separate. This defect is the sub-5× band those cannot reach. +- **Consumer follow-through:** once `/anomalies` (#239) and `/overruns` (#171) land, confirm they read a value base that already excludes/corrects these (they will, if they use `amount_eur` + `value_flag`). diff --git a/docs/implementation-plans/306-amendment-contract-namespace-link.md b/docs/implementation-plans/306-amendment-contract-namespace-link.md new file mode 100644 index 00000000..3b23ff19 --- /dev/null +++ b/docs/implementation-plans/306-amendment-contract-namespace-link.md @@ -0,0 +1,94 @@ +# Implementation Plan: #306 — annexes don't link to a contract (contract number in a different namespace) + +## Executive Summary + +| Field | Value | +|---|---| +| Ticket | [midt-bg/sigma#306](https://github.com/midt-bg/sigma/issues/306) — labels `data-quality`, `etl`, `priority: high` | +| Problem | 1,937 EOP annexes (7.2%) don't link to any contract. #286 fixed the OCDS half (procedure axis: OCID vs УНП). This is the other axis: the **contract number**. The annex carries an internal annex-side number (`148846`, `2886`); the contract carries the buyer's filing number (`Д-226`, `388-2020`). The exact `(unp, contract_number)` join fails, so the annex drops out of every annex→contract→company/authority rollup. | +| Root cause | Two genuinely unrelated identifiers in different namespaces — not dirty strings. Measured: `TRIM`+`UPPER` saves 0, digits-only 11, substring 19. String normalisation is a dead end. | +| Approach | Link by **value**: an annex's `value_before` is the contract's value at amendment time, so it equals the target contract's `signing_value`. Rewrite `raw_amendments.contract_number` to the resolved contract when `value_before` **exactly** (< 0.5 стотинка), currency-matched, uniquely matches one contract on the procedure; propagate across a chain sharing the annex-side number. | +| Complexity | Medium — the resolver is a self-contained CTE; the subtlety is full-vs-slice parity (the #305 bug class) and the slice's windowed staging. | +| Risk | Low on the full path (measured 99.99% precision), gated to leave ambiguous/no-match annexes unlinked. | +| Status | **Full-derive resolver implemented as a gated, standalone script (`scripts/resolve-amendment-contracts.sql`) that runs BEFORE `derive-amendments.sql`, full-derive path only. Slice/daily resolver deferred (documented below).** Validated against the live `sigma-dev` corpus; hardened per the PR #308 review (ordering blocker, group-contradiction rule, cumulative-candidate dedup, EIK guard, provenance columns). | + +--- + +## 1. Problem, verified on real data (`sigma-dev`) + +Reproduced the issue's breakdown exactly on the served corpus (an amendment `a` links iff `contracts c` exists with `c.tender_id = 't:'||a.unp AND c.contract_number = a.contract_number`): + +| check | #306 | live `sigma-dev` | +|---|---|---| +| EOP annexes unlinked | 1930 | **1937** | +| empty contract_number | 0 | 0 | +| empty УНП | 0 | 0 | +| УНП not in `tenders` | 0 | 3 | +| procedure has no contract | 12 | 15 | +| procedure has contracts, none match | 1918 | **1919** | +| procedure has exactly 1 contract (safe) | 1012 | **1012** | +| price-changing | 554 | **555** | + +## 2. The value anchor (the insight beyond the issue) + +The issue proposed linking only the 1,012 single-contract procedures by УНП. Measuring `value_before` → `signing_value` unlocks the multi-contract cases too. On the **already-linked** annexes (ground truth), when `value_before` uniquely matches one contract's `signing_value`: + +| gate | unique matches | correct | precision | +|---|---|---|---| +| within 1% | 9061 | 8975 | 99.08% | +| **exact (< 0.5 стотинка)** | **9349** | **9348** | **99.99%** | + +Exact-cent match wins on **both** precision and recall — the 1% band manufactures ambiguity and admits the #305 "matched a smaller sibling's value" errors. Example `00011-2020-0002` (multi-lot): the unlinked `2886` annexes carry `value_before=56000`, which exactly matches contract `388-2020` (lot 2, signing 56000), **not** `387-2020` (64000) — a case УНП-only linking cannot resolve. + +### Recovery under the shipped gate (exact + currency + unique + chain-propagation) + +| tier | rule | links | +|---|---|---| +| single-contract, value-confirmed | procedure has 1 contract, `value_before` == its `signing_value` | 775 | +| multi-contract, value-unique | `value_before` exactly matches one of ≥2 contracts | 604 | +| **combined (before chain propagation)** | | **~1,379 (71%)** | + +Left **unlinked by design**: value-ambiguous (matches 2+, ~115), no-match (~193, target not yet ingested — the #249 class), no-tender/no-contract (~18). An honest gap beats a wrong contract on a transparency site. + +> **Re-measured on the live `sigma-dev` corpus (PR #308 review).** Read-only reproduction against the served corpus confirms the headline numbers hold under the revised rule: **1,937** unlinked, **1,563** recovered (1,377 direct + 186 propagated), precision **9,348/9,349 = 99.99%**. Two review inferences were corrected by the data: the `value_before IS NULL/≤0` class is **empty** (all 1,937 unlinked annexes carry a value; the `1,379+326≠1,937` gap is tier-estimate rounding, not a value-less class), and the only anchor-disagreement group in the whole corpus is the correct 00026 lot-base case (hence the §3.4 rule revision). **Still owed:** the served corpus is deduped one-row-per-contract, so it does not exercise the cumulative-staging path (§3.1) — that must be re-measured on raw cumulative staging via a local full backfill before the dedup fix is fully validated end-to-end. + +## 3. Fix — full-derive path (`scripts/resolve-amendment-contracts.sql`, run before `derive-amendments.sql`) + +The resolver is a **standalone script** that `import.mjs` runs from `runFullDerive` / `runWorkBackfill` **before** `derive-amendments.sql` — i.e. **before** the #286 prefer-EOP dedup DELETE and its diagnostics. Ordering is load-bearing (PR #308 review, todorkolev #1 blocker): rewriting an EOP annex onto a contract that already kept an OCDS twin, if done *after* the dedup, resurrects the twin (`annex_count = 2` on a one-annex contract) and trips the `amendment-twin-dedup` integrity gate (#303), failing the whole derive. Running first — and above the #286 diagnostics — keeps the dedup the sole twin guard and its dropped/excess counts honest bounds. The resolver reads only `source LIKE 'eop:%'` rows + `raw_contracts`, so it is independent of the OCDS bridge and safe to run first. + +1. `contract_candidates` — `raw_contracts` **deduped to one row per `(unp, contract_number)`** (`ROW_NUMBER() … ORDER BY source DESC, id DESC = 1`), mirroring normalize-raw. `raw_contracts` is cumulative (the same contract recurs across daily buckets; the collapse happens later in normalize-raw), so without this `COUNT(*) OVER` would count staging **rows**, not contracts — mass fail-close on a real rebuild, or a match to a superseded value (PR #308 review, nikimilenkov HIGH 2). +2. `grp` — **all** EOP annexes with no `(unp, contract_number)` contract, grouped by the shared annex-side number `(unp, annex_cnum)` = one contract's chain. Value-less members (`value_before` NULL/≤0) are included so they can inherit the chain target (review MEDIUM 2). +3. `vmatch` — join to `contract_candidates` on `unp`, exact value (`ABS(signing_value − value_before) < 0.005`), **explicit** currency on both sides (blank ≠ blank; review LOW 1), null-tolerant contractor-EIK match (a value collision onto a different contractor is refused for free; review MEDIUM 5). `COUNT(*) OVER (PARTITION BY amendment_id)` flags uniqueness. +4. **Group rule** (reviews todorkolev #2, nikimilenkov MEDIUM 1 & 2, revised against the live corpus): a member's OWN unique (`n_match = 1`) exact match always applies — it is individually trustworthy (the 99.99% figure) and is **not** voided when annex-number siblings point elsewhere, because an annex number can be a **lot-base** shared across contracts (live corpus: `20РП-У50А015` → …-Л01 @ 22569.98 **and** …-Л03 @ 28557.50, each annex exactly-uniquely matching its own lot — the sole disagreement group in the whole corpus, and it is correct). Members with **no** own unique match (value-less admin steps, or later steps whose cumulative value matches no `signing_value`) inherit one agreed group target — but **only** when the direct members agree; disagreement withholds *propagation*, never the direct hits. A member that is itself value-ambiguous (`n_match ≥ 2`) carries its own contradicting evidence and never links, directly or by inheritance. + + > An earlier revision refused the whole group on disagreement (nikimilenkov MEDIUM 1 as first stated). Measured on the live corpus that dropped 2 confirmed-correct multi-lot links and prevented **zero** wrong links (the only disagreement group is the benign 00026 lot-base case), so the rule keeps direct hits and gates only propagation. +5. `UPDATE raw_amendments SET contract_number = resolved, contract_number_raw = , link_method = 'value_anchor'` in place — like the #286 УНП bridge, but **preserving provenance** (review MEDIUM 4). The rollup, `promote-amendments.sql`, and the serving join then link with no further change; the original annex number stays in `contract_number_raw`, which also keeps it in the amendment `natural_key` so a resolved row never collides with a native annex sharing `document_number` on the target (review MEDIUM 3). + +Diagnostic printed by wrangler: `annexes_value_linked`, `eop_annexes_still_unlinked` (complementary predicates over the same mismatch population). + +Idempotent: after the rewrite the row links by number, so a re-run's `grp` no longer selects it and the provenance columns are never re-stamped. + +## 4. Slice / daily path — **implemented** (corpus-safe resolver inside `refresh-slice.sql`) + +Originally deferred: the standalone `resolve-amendment-contracts.sql` is **not** run on the slice path (`runSliceDerive`), because it would execute against `refresh-slice.sql`'s **windowed** `raw_contracts`, where "unique on the procedure" means "unique **in the window**" — a corpus-ambiguous annex looks unique in a narrow window and mislinks, and the measured 99.99% precision (a full-corpus number) would not carry (PR #308 review, nikimilenkov HIGH 1). + +The daily/slice + Worker path now runs a corpus-safe equivalent **inside `refresh-slice.sql`** (search `#306: slice-safe value-anchor resolver`), addressing PR #308 review todorkolev "дневните обновявания": the cron runs only `refresh-slice.sql`, so the fix had to live there or stay inert in production. It applies the same value/currency/EIK anchor, chain rules, and provenance stamping as the full path, differing only in the candidate source: + +- Candidate contracts are drawn from the **served `contracts` table** (the whole corpus; `unp` via the `tender_id` suffix, contractor ЕИК via the winning `bidders.eik_normalized`) **UNIONed** with this window's `raw_contracts`, so uniqueness is asked corpus-wide. It is intentionally **not** under a byte-identical lockstep marker — the candidate source differs by construction. Scans are bounded to procedures with an EOP annex in this window (`window_unps`), keeping the served read a keyed lookup (`idx_contracts_tender_id`). +- Resolved targets land in `refresh_touched_contracts` **for free**: the resolver rewrites the target `contract_number` onto `raw_amendments` in the setup batch, and the existing `@refresh-batch amendments` touch join (`raw_amendments` → `contracts` on `contract_number`) then scopes prior-window targets into it. +- It runs in the setup batch **before** the prefer-EOP dedup DELETE, the same ordering the full path uses before `derive-amendments.sql`'s dedup (review todorkolev #1 blocker — otherwise a rewrite resurrects an OCDS twin and trips `amendment-twin-dedup` #303). +- The "is this annex's number already a real contract?" question is asked over a **value-agnostic** `all_contract_numbers` CTE (every contract number on the procedure, from window `raw_contracts` **and** served `contracts`, regardless of `signing_value`) — **not** over `contract_candidates` (which requires `signing_value > 0` for value matching). This keeps the slice path identical to the full path's `NOT EXISTS … raw_contracts`: an annex that matches a **zero-value** contract by number links by number, and is never value-linked to a neighbour (review todorkolev discrepancy). + +**Deployment (review todorkolev blocker).** `refresh-slice.sql` now writes `contract_number_raw` + `link_method` into served `amendments`, but the deployed DBs are populated out-of-band so `wrangler d1 migrations apply` can't run. `deploy.yml` gains an idempotent step (probe `pragma_table_info`, `ALTER` only the missing columns, malformed response fatal) **before** the Worker deploys — otherwise the first cron after release crashes on the missing columns. If #307 merges first, fold these two columns into its provenance step instead. + +Provenance keeps slice and full keys aligned: the amendment `natural_key` (winners dedup + promotion) uses the raw annex number via `COALESCE(NULLIF(contract_number_raw,''), contract_number, '')`, and the served `amendments` promotion carries `contract_number_raw` + `link_method`. A resolved annex re-emitted in a later slice window re-resolves deterministically to the same key — an honest, never-double-counted link. Where a target contract and its namespace-mismatched annex arrive in the **same** window, the annex still links (window `raw_contracts` is in the candidate union); the residual best-effort gap is only a target never served or staged, which the next full rebuild closes (#286 precedent). + +## 5. Blast radius (auto-corrects on a full rebuild) + +Everything is rebuilt from scratch by `precompute.sql` (DELETE/INSERT), so it self-corrects: `company_totals`, `authority_totals`, `sector_totals`, `home_totals`, `facet_counts`, `flow_pairs`, `cpv_division_stats`, `search_index`, and each contract's `annex_count`/`current_value`/timeline. Overall annex counts and sums on the annexes themselves are unchanged. Two things to verify after the run: (1) `cpv_division_stats` p95/p99 bands shift for affected divisions; (2) **merge-order dependency** — ~23 newly-linked annexes have `value_after ≥ 2× value_before` and should trip the #307 double-count flags (`annex_total_suspect`/`value_suspect`). Those flags do **not** exist on this branch (they are #307, in review); if this merges before #307, the newly-linked ≥2× annexes enter the aggregates **unflagged** until #307 lands. Track the merge order explicitly. + +**Migration numbering (PR #308 review todorkolev "сблъсък на номера").** #307 adds `0006_amendment_restated.sql` + `0007_amendment_value_suspect.sql`; this branch's provenance migration is therefore numbered **`0008_amendment_provenance.sql`** to sit after both. This assumes #307 merges first (it should — the ≥2× flags above depend on it). If #308 lands first instead, renumber to `0006` and have #307 shift to `0007`/`0008`. + +## 6. Tests + +`packages/db/src/amendments-contract-resolve.test.ts` runs the real `resolve-amendment-contracts.sql` + `derive-amendments.sql` (the full-derive composition) against SQLite and pins: single-contract link (00017 shape), multi-lot value disambiguation (00011 shape), chain propagation + `current_value` = last step, value-less chain member inherits (MEDIUM 2), ambiguous-in-chain member stays unlinked (todorkolev #2), whole-group refusal on disagreeing anchors (MEDIUM 1), cumulative-staging duplicate counted once (HIGH 2), EIK guard (MEDIUM 5), value-ambiguous / no-match / currency-guard / blank-vs-blank-currency (LOW 1) → unlinked, twin-ordering (the resolver runs before the prefer-EOP dedup, no `annex_count = 2` — todorkolev #1 blocker), gate (derive-alone does not resolve — HIGH 1), idempotency, natural-key collision avoidance (MEDIUM 3), provenance through promote into served `amendments` (MEDIUM 4), already-linked untouched, and the diagnostic counts. diff --git a/docs/review-testing.md b/docs/review-testing.md index 3ab4b106..0cbd4b0a 100644 --- a/docs/review-testing.md +++ b/docs/review-testing.md @@ -43,6 +43,26 @@ - Нов workspace с тестове се добавя и в `coverage-baseline.json` (и получава `vitest.config.ts` с `sharedCoverage(...)`); `packages/api-contract` е освободен, докато няма тестове. +### `scripts/` е извън ratchet-а — прегледано изключение, не пропуск + +Ratchet-ът покрива само `apps/*` и `packages/*`. `scripts/` (~7 хил. реда, сред които клеветнически +критичната логика на свързаните лица — доказателственият ред, съвпадението по имена, гейтовете при +публикуване) не се измерва от него. Записано е тук, за да е решение, а не дупка, която някой открива +по-късно. + +**Защо:** тези файлове са `.mjs`, пускат се от три отделни runner-а извън `turbo run test` и голяма част +от тях са I/O обвивки около мрежа и SQLite, за които процент покритие би измервал колко от обвивката е +докосната, а не дали правилото е вярно. + +**Какво пази качеството вместо процент:** и трите набора са в CI и са задължителни — +`node --test scripts/*.test.mjs`, `node --import ./scripts/cacbg/register-ts.mjs --test +scripts/cacbg/*.test.mjs scripts/tr/*.test.mjs` — а гейтовете, при които грешка означава поименно +твърдение за реален човек, се проверяват **мутационно**: тестът трябва да пада при обърнато условие, +иначе не е гейт. Виж ADR-0027 за защо тест, който минава и преди реализацията, е фалшив гейт. + +**Кога това трябва да се преразгледа:** ако `scripts/` започне да носи логика, която не е нито +чиста функция с мутационно проверен тест, нито тънка I/O обвивка. + ## Integrity gate - Пуска се върху обслужвания D1 след `precompute` в `ship-domain.mjs` и след `runSliceDerive()` в diff --git a/docs/runbooks/related-persons-suppression.md b/docs/runbooks/related-persons-suppression.md index ac2b140c..7d407cd3 100644 --- a/docs/runbooks/related-persons-suppression.md +++ b/docs/runbooks/related-persons-suppression.md @@ -4,6 +4,21 @@ A contested, corrected, or legally-challenged link on the свързани-ли the version-controlled suppression list. See [ADR-0031](../adr/0031-suppressions-version-controlled-fingerprinted.md) for why this is a git list keyed on an HMAC fingerprint, not a DB row. +## Grounds that require a takedown + +Two of these are specific to the Trade Register evidence (#279, ADR-0033) and are easy to miss, because +neither shows up as a bug: the pipeline is working correctly and the claim is still wrong. + +- **A court-annulled entry (чл. 29 ЗТРРЮЛНЦ).** A registry entry set aside by a court is void, but the + deed we cached may still carry it, and the rules that read it are unchanged — so a rules-version bump + will NOT remove the link. This is an evidence-invalidation ground, and suppression is the mechanism. +- **A match against the wrong person (a namesake).** The register carries no ЕГН, so a three-name match + inside one entity is strong evidence of identity but not proof. If the named official is not the person + in the act, the link is a false public claim about them and must come down immediately — do not wait for + a rules change or a re-crawl. +- The ordinary grounds are unchanged: a contested link under review, a correction the source has since + published, or a legal challenge. + ## What you need - The exact `link_key`. Self links are `pid|eik`; a close-relative link is `pid|eik|family`. Get it from the @@ -46,6 +61,38 @@ wrangler d1 execute --remote \ Then do the Standard takedown steps above **in the same shift**. The direct UPDATE is undone by the next ship (which rebuilds the table); only the git list makes it permanent. +## Correcting a wrong INPUT (a different list — read this before reaching for a suppression) + +Suppression keeps a *built* link out of the public surface. It is the wrong tool when the link should never +have been built at all — a misparsed declaration row, a stake attributed to the wrong filing, an entity +that was never declared. There you fix the input, and the link stops existing. + +That removal still has to get past the monotonicity gate ([ADR-0033](../adr/0033-registry-evidence-replaces-name-distinctiveness.md) +decision 6), which hard-fails on a link that was published last run and is not published now. And you +cannot use a suppression to clear it: correcting the input unbuilds the link, so the fingerprint would +match nothing and `load.mjs` fails on the unmatched-entry rail instead. The two removals fail in opposite +directions, which is why there is a second list. + +1. Compute the fingerprint exactly as above (same salt, same function, same `key_version`). +2. Append one line to `scripts/cacbg/link-corrections.jsonl`: + + ```json + {"fp":"","key_version":"1","reason":"the declaration row was misparsed — ","signal_ref":"","corrected_at":"2026-08-11"} + ``` + +3. Land it **in the same PR as the input fix**. The loader flags that key in the pre-wipe snapshot and the + audit reads it as a declared removal — printed, never silent, and attributed to this ground rather than + flattened into "removed". + +**An acknowledgement is one-shot. Delete the line in the next change.** Once the corrected link stops being +published it also stops appearing in the prior set, so the entry matches nothing and the build fails on the +same rail suppressions use. That is deliberate: a stale acknowledgement would sit in the list and silently +pre-clear a *future* disappearance of that same link — the exact regression the gate exists to catch, with +nobody having decided it. + +Do not use this list to clear a removal you do not understand. If a published link vanished and you cannot +name why, that is the gate doing its job; find the cause. + ## Fail-closed behaviour `load.mjs` aborts the build (non-zero exit) on any of three conditions — each would otherwise silently @@ -65,6 +112,30 @@ If a ship fails on any of these, fix the cause; do **not** empty the list or dro A rotated salt invalidates every existing fingerprint, so it is a coordinated change guarded by `key_version`: re-derive `fp` for each entry from its original `link_key` under the new salt (you need the plaintext keys, kept out-of-band, e.g. in the incident tickets), **bump every entry's `key_version`** to the -new value, replace the file in one commit, and update both CI secrets (`SUPPRESSION_SALT` and -`SUPPRESSION_KEY_VERSION`). Because the loader refuses any entry on a non-current `key_version`, a -half-finished rotation fails the build loudly instead of silently un-suppressing. +new value, replace the file in one commit, and update **both** CI settings — but note they are different +kinds: `SUPPRESSION_SALT` is a repository **secret**, while `SUPPRESSION_KEY_VERSION` is a repository +**variable** (`${{ vars.SUPPRESSION_KEY_VERSION || '1' }}` in `related-persons-data.yml`). The version is +not sensitive — it is a counter — and looking for it under Secrets is a dead end during an incident. +Because the loader refuses any entry on a non-current `key_version`, a half-finished rotation fails the +build loudly instead of silently un-suppressing. + +## Verifying a takedown actually worked + +A takedown that silently failed looks exactly like one that succeeded: the entry sits in the list, the +build is green, and the link is still public. The list is applied at LOAD time, so nothing changes on the +served surface until the next data run ships — check the surface, not the commit. + +1. **The loader saw it.** The run refuses an entry matching no built link (the B3 unused-entry rail), so a + green run already proves the fingerprint matched something. A run that fails with „suppression matched + NO link" means the `link_key` was wrong — a family link needs its `|family` suffix. +2. **The row is gone from the served D1**, which is the only copy a reader can reach: + ``` + wrangler d1 execute "$SIGMA_D1_NAME" --remote \ + --command "SELECT status FROM interest_links WHERE link_key = ''" + ``` + Expect `suppressed`, or no row at all. Anything else means the ship did not carry the decision. +3. **The page is gone**, allowing for cache: the link's page must 404 and the official's page must not + list it. `publicCache(3600)` means a reader can still see it for up to an hour after the write — if it + is still there beyond that, the takedown did not land. +4. **It stays gone.** Re-run the next scheduled load and repeat step 2: the list is what makes a takedown + survive a rebuild, and this is the only step that proves the survival rather than assuming it. diff --git a/docs/spec/related-persons-foundation.md b/docs/spec/related-persons-foundation.md index 8c50fa94..61d399f5 100644 --- a/docs/spec/related-persons-foundation.md +++ b/docs/spec/related-persons-foundation.md @@ -70,6 +70,13 @@ Of #128's five checkboxes: **1** (shared owners) contingently unblocked via §3. **Per-ЕИК public lookup** (`portal.registryagency.bg`) is free, no fee. Recoverable per ЕИК: **управители, съдружници/собственици на капитала + stakes, капитал, legal form, seat.** **ЕГН masked** → owner identity name-based. Run **only** for the bounded set (bidders + resolved declared/donor companies) — thousands, not ~900k. **Hard constraints (DPA-verified):** **NEVER bulk-scrape TR** (КЗЛД ruled bulk provision unlawful; CJEU C-200/23 climate). **Never store scraped ЕГН**, even if leaked in a document image. Phase-0 confirms endpoint, fields, and rate limits. +**Phase-0 is CLOSED — measured 2026-08-04/05 (#279, ADR-0033).** Endpoint: `GET portal.registryagency.bg/CR/api/Deeds/{ЕИК}`, public JSON, no authentication. Facts worth recording because three of them contradict what the issue predicted: +- **The register rate-limits, and the block is sustained.** HTTP 429 at ~50 cumulative requests ending in a burst, then 429 to everything — no `Retry-After`, no quota header. So a 429 is an instruction to stop, never a transient to retry through: the crawler ends the run, marks nothing, and resumes later. The implemented pace is 1 request / 3 s, sequential, over a **closed** candidate set the crawler cannot extend (it never follows a link out of a deed). +- **„Not in the register" is an HTTP 200 with a ZERO-BYTE body**, not a 404 and not HTML. Verified on Община София (`000696327`): empty twice while a real company returned its full deed in the same window. Permanence therefore keys on the **status**, not the empty body — empty under 200 is an answer and may be cached; empty under 5xx is a failure and stays transient. +- **`fetch` does not work against this host.** The identical request returns 500/empty via undici and 200 with the full deed via `node:https`. +- Field values are HTML fragments inside the JSON envelope, and one field routinely holds several people separated by `
`; erasure is marked structurally (`erasure-text-inline`), not textually. Matching must happen **inside one entity**, or one person's given name combines with another's surname. +- **ЕГН is absent** from every payload examined, and is never stored either way. Raw deeds live only under git-ignored `scratch/tr/` with a 35-day retention; the cache index holds no name at all. + ## 4. Data model New domain tables, built by `normalize-raw.sql` from persistent staging; `interest_links` written by a **JS resolver pass** (§5), NOT by fuzzy SQL. @@ -182,6 +189,7 @@ The public site MUST carry a plain-language methodology page. It is part of the 1. **Sources + legal basis** — CACBG declarations (ЗСП чл. 75, Сметна палата); ЕРИК (Изборен кодекс); targeted TR per-ЕИК (public company facts). Each with a direct source link. 2. **What is shown and what is not** — officials' own declared holdings; **family holdings are NOT published** (v1); addresses/ЕГН never shown. 3. **The matching rule, verbatim** — declared company **full name** (incl. legal form) is normalized and matched **exactly** against the winner's registered name, which is paired 1:1 with its ЕИК in the procurement source; because Bulgarian trade names are **nationally unique** (ЗТРРЮЛНЦ/ТЗ), an exact match is the same legal entity. Ambiguous cases (truncated names, no exact match) are **excluded or human-confirmed, never guessed**. Every link shows its full evidence chain. + **Superseded in part by #279 / ADR-0033:** a name match is no longer *sufficient* to publish. Every link additionally rests on a checkable **Търговски регистър** fact, and the methodology page carries the six-rung evidence ladder verbatim, together with the disclosures that make it honest: the register carries **no ЕГН**, so a three-name match is not proof of identity and a namesake is possible; the match must therefore be a **full three-token subset inside a single registry entity**; seats move, so a registered seat only confirms a period it predates; erased entries are skipped; and every published link shows its **entry number and lookup date**. 4. **Certainty & framing** — only deterministic exact matches (and human-confirmed cases) appear; each is a „**модел за проверка, не обвинение**"; the tool shows *declared* links only and **does not claim to find hidden ownership** (it is a floor, not the full picture). 5. **Temporal meaning** — a declaration is a point-in-time snapshot; how overlap with a contract date is (and isn't) interpreted. 6. **Known gaps** — the recall holes (§5), self-reporting limits, sources it cannot see. diff --git a/docs/spec/related-persons-lia.md b/docs/spec/related-persons-lia.md index 1309c42a..07523940 100644 --- a/docs/spec/related-persons-lia.md +++ b/docs/spec/related-persons-lia.md @@ -128,6 +128,40 @@ v Poland* (2428/05) приема онлайн публикуване на дек регистър. Приемаме този оспорим елемент съзнателно. **Резервен план при оспорване:** един филтър (сваляне на семейните връзки към еднолични дружества), не спиране на повърхността. +## 9а. Допълнение: справки в Търговския регистър (#279, ADR-0033) + +Нов **източник** и нова **обработка**, затова оценката се допълва, а не само се препотвърждава. + +**Каква е обработката.** За всяко дружество-кандидат правим една справка в публичния Търговски регистър и +четем живия акт. Актът съдържа **лични данни на трети лица** — имена на съдружници, собственици и +управители, които не заемат публична длъжност, както и адрес на дружеството. + +**Цел и основание.** Целта е тясна и проверима: да се установи **самоличността на дружеството** зад +декларирано име, преди срещу конкретно лице да бъде публикувано твърдение. Това е мярка, която *намалява* +риска за субектите на данни — досега самоличността се решаваше по евристика за отличителност на името, а +сега — по документ. Основанието е същият надделяващ обществен интерес по ЗДОИ чл. 41и, ал. 4, приложен към +по-точна, а не към по-широка публикация. + +**Минимизиране — какво НЕ правим.** +- Имената от акта се ползват **само в булеви сравнения** и не влизат в никаква публична таблица, отговор или + лог. Индексът на кеша не съдържа нито едно име — само ЕИК, дати, кодове и хеш на тялото. +- Адресът **не се извлича**: от полето за седалище се чете единствено сегментът „Населено място". +- Полетата за **действителен собственик** (`CR_F_550_L`) и за контрол (`537/538`) **не се ползват** — това + би било претенция за действителна собственост, оставена настрана след CJEU C-37/20 (ADR-0007, т. 1). +- Суровите актове се пазят само служебно, в git-игнорирана директория, със **срок 35 дни**. +- Обхватът е **затворен**: справка се прави само за ЕИК, които вече са резултат от съпоставянето; обхождащият + никога не следва връзка навън от акт. Това не е масово изтегляне (спец. §3.3). + +**Остатъчен риск — съименник.** Регистърът **не съдържа ЕГН**, затова съвпадението по трите имена е силно, но +не абсолютно доказателство за самоличност. Смекчаване: изисква се пълно съвпадение и на трите имена в **един и +същ запис** на акта; имена с инициали или с латински букви не се приемат; фамилни съвпадения не се ползват +никъде. Грешката тук не измисля твърдение за собственост — тя **закача длъжностното лице за друго дружество**, +което е също толкова невярно и се сваля по реда на ADR-0031 (виж runbook-а: „съвпадение с грешно лице"). + +**Отложено съзнателно.** Етикетите „и към днешна дата" по т. 7 на #279 **не се пускат** в тази фаза: те +твърдят сегашно време за поименно лице, а давността им е ограничена от цикъла на опресняване. Влизат само +след отделен преглед на настоящата оценка. + ## 10. Статус и преглед Draft за правен преглед; изпраща се паралелно с пускането, не като блокер (ADR-0032, т. 7). Ревизира се при промяна diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index d1d9f709..75f100cb 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -252,6 +252,9 @@ export interface ContractValueTimeline { currentEur: number | null; deltaPct: number | null; // (current − signing) / signing, when both present suspect: boolean; // value_/annex_suspect/review → render with an unverified-value label + // annex_total_suspect → the current value is a KNOWN exact 2× double-count. currentEur is blanked (—) + // rather than shown as a labelled doubled figure — a known-wrong number is worse than an honest gap (#307). + currentValueDoubled: boolean; } export interface ContractLotRow { @@ -282,6 +285,8 @@ export interface AmendmentEntry { description: string | null; // recorded reason/notes, when the source carries them valueAfterEur: number | null; // the contract value after this annex deltaEur: number | null; // value_after − value_before + restated: boolean; // #305 Tier-2: value_after was text-corrected from a double-counted total + suspect: boolean; // #305 residual: an uncorrectable double-count — value_after/delta suppressed, row marked } export interface ContractDetail { @@ -705,6 +710,15 @@ export interface ConflictLink { firstContractYear: string | null; lastContractYear: string | null; sourceUrl: string | null; // a representative declaration URL — provenance, never a fabricated value + // Trade Register evidence (#279, ADR-0033). A link only reaches this DTO when its identity rests on a + // checkable registry fact, so these describe WHICH fact — the surface's whole point is that every shown + // link can explain itself. `registryRole` is the role the register records, NOT a claim about who owns + // what: the ownership claim comes from the official's own declaration. + evidenceKind: 'document' | 'confirmed'; // the only two rungs that publish + registryRole: 'owner' | 'manager' | null; // set only for evidenceKind='document' + registryEntryNumber: string | null; // TEXT — a fieldEntryNumber exceeds the exact-integer range + registryEntryDate: string | null; // the registry entry the evidence rests on + registryLookupDate: string; // when the deed was read — the freshness bound on the claim } /** One contract of a linked winner, marked by whether it was signed during the declared-stake window. diff --git a/packages/db/migrations/0000_init.sql b/packages/db/migrations/0000_init.sql index 90f98dac..57002fd3 100644 --- a/packages/db/migrations/0000_init.sql +++ b/packages/db/migrations/0000_init.sql @@ -125,13 +125,13 @@ CREATE TABLE contracts ( bids_received INTEGER, contract_kind TEXT, -- Доставки / Услуги / Строителство awarded_to_group INTEGER, -- this AWARD went to an обединение (per-contract, distinct from bidders.is_consortium) - value_flag TEXT NOT NULL DEFAULT 'ok', -- ok | review | value_low | value_suspect | annex_suspect (data-quality verdict; assigned in scripts/normalize-raw.sql) + value_flag TEXT NOT NULL DEFAULT 'ok', -- ok | review | value_low | value_suspect | annex_suspect | annex_total_suspect (data-quality verdict; assigned in scripts/normalize-raw.sql) date_flag TEXT NOT NULL DEFAULT 'ok', -- ok | signed_after_publication (non-destructive date-quality verdict) amount_eur REAL, -- canonical EUR, SAFE TO SUM; populated for all flags (value_suspect repaired to the procedure estimate); NULL only when no trustworthy EUR figure (FX-rateless foreign / value_suspect w/o estimate / no signing+current) fx_converted INTEGER NOT NULL DEFAULT 0, -- 1 = amount_eur came from a foreign-currency market rate fx_rate REAL, -- EUR per 1 unit of `currency` for foreign rows (amount × fx_rate = amount_eur) signing_value_eur REAL, -- signing_value in EUR (peg/fx); NULL for value_suspect — for the contract value timeline - current_value_eur REAL, -- current_value in EUR; NULL for value_suspect/annex_suspect (suspect annex suppressed) + current_value_eur REAL, -- current_value in EUR; NULL for value_suspect/annex_suspect/annex_total_suspect (suspect annex suppressed) lot_id TEXT, -- domain lot id ('lot:'||УНП||':'||raw) when the award is lot-scoped; soft-links lots(id) document_number TEXT, -- Номер на документ published_at TEXT, -- Публикуван на diff --git a/packages/db/migrations/0006_amendment_restated.sql b/packages/db/migrations/0006_amendment_restated.sql new file mode 100644 index 00000000..5f158067 --- /dev/null +++ b/packages/db/migrations/0006_amendment_restated.sql @@ -0,0 +1,17 @@ +-- #305 Tier-2 text-based value correction (packages/ingest/src/amendment-total.ts). Some ЦАИС ЕОП +-- annexes put the announced NEW TOTAL into the change field, doubling value_after. The основание text +-- resolves each: a restated total drives the corrected value_after (and current_value), a genuine +-- increment is confirmed correct. The served amendments row records the outcome so the UI can mark a +-- corrected row and the refresh-slice reconciliation can skip text-treated annexes when arithmetic-flagging. +-- +-- value_restated = 1 when value_after was rewritten to the text-confirmed true total, else 0. +-- value_treatment = the raw treatment label ('total_restated' / 'unchanged_restated' / +-- 'genuine_increment', NULL when the text carried no signal). Kept alongside +-- value_restated because the slice reconciliation re-classifies from the served +-- amendments and must skip confirmed-genuine increments (value_restated stays 0 there). +-- Additive columns. On the live stage DB (whose migration ledger is empty — base schema imported +-- out-of-band) these are applied by the column probe in .github/workflows/deploy.yml, which ALTERs only +-- when the column is missing; on a fresh ledger `d1 migrations apply` runs this file exactly once. SQLite +-- has no `ADD COLUMN IF NOT EXISTS`, so do not replay this file against a DB that already has the columns. +ALTER TABLE amendments ADD COLUMN value_restated INTEGER NOT NULL DEFAULT 0; +ALTER TABLE amendments ADD COLUMN value_treatment TEXT; diff --git a/packages/db/migrations/0007_amendment_value_suspect.sql b/packages/db/migrations/0007_amendment_value_suspect.sql new file mode 100644 index 00000000..faa1b195 --- /dev/null +++ b/packages/db/migrations/0007_amendment_value_suspect.sql @@ -0,0 +1,14 @@ +-- #305 residual: a contract flagged value_flag = 'annex_total_suspect' has its current_value excluded +-- from every aggregate, but the served amendments row still carried the DOUBLED value_after — so the +-- amendment timeline kept showing the untrusted figure. Only text-corrected rows (value_restated) had +-- their served value rewritten; the ~183 flag-only doubles (non-exact >2×, no основание signal) did not. +-- We do NOT know their true total, so we MARK the row and let the UI SUPPRESS the untrusted figure — +-- never invent a number. +-- +-- value_suspect = 1 when the served amendment is a suspected double-count NOT already text-treated +-- (value_treatment IS NULL), else 0. A restated/genuine row (value_treatment set) is +-- never also suspect. The UI blanks value_after/delta for a suspect row. +-- Additive column. Applied on the live stage DB by the column probe in .github/workflows/deploy.yml (ALTER +-- only when missing); on a fresh ledger `d1 migrations apply` runs it once. SQLite has no `ADD COLUMN IF +-- NOT EXISTS`, so do not replay this file against a DB that already has the column. +ALTER TABLE amendments ADD COLUMN value_suspect INTEGER NOT NULL DEFAULT 0; diff --git a/packages/db/migrations/0008_amendment_provenance.sql b/packages/db/migrations/0008_amendment_provenance.sql new file mode 100644 index 00000000..fc8d4727 --- /dev/null +++ b/packages/db/migrations/0008_amendment_provenance.sql @@ -0,0 +1,9 @@ +-- #306: provenance for value-anchor-linked annexes. The resolver in +-- scripts/resolve-amendment-contracts.sql rewrites a namespace-mismatched annex's contract_number to its +-- target contract; unlike the #286 OCDS bridge (whose OCID survives in tender_ext_id + source), the annex +-- number would otherwise be destroyed with no trace, making the 99.99%-precision claim unauditable in +-- production and any "this annex isn't ours" complaint uninvestigable (review nikimilenkov MEDIUM 4). +-- Keep the original annex-side number and stamp the link method so value-linked rows stay enumerable on the +-- served side. NULL on both columns = the row linked by contract_number directly (or is unlinked). +ALTER TABLE amendments ADD COLUMN contract_number_raw TEXT; +ALTER TABLE amendments ADD COLUMN link_method TEXT; diff --git a/packages/db/migrations/0009_interest_link_evidence.sql b/packages/db/migrations/0009_interest_link_evidence.sql new file mode 100644 index 00000000..e91869d1 --- /dev/null +++ b/packages/db/migrations/0009_interest_link_evidence.sql @@ -0,0 +1,50 @@ +-- Trade Register evidence seal for a свързани-лица link (#279 §8, ADR-0033). +-- +-- WHY A SIDE TABLE and not columns on interest_links, since both were on the table: +-- 1. SQLite's ALTER TABLE ... ADD COLUMN has no IF NOT EXISTS, and migrations here are applied by a +-- bare `wrangler d1 execute --file` with no applied-migrations tracking (see the workflow steps), +-- so a re-apply MUST be a no-op. `CREATE TABLE IF NOT EXISTS` is; ADD COLUMN is not. +-- 2. scripts/cacbg/load.mjs rebuilds the CACBG tables from 0003 ALONE. A column added here would have +-- to be duplicated into 0003 and kept in step with it forever, and the day the two diverge the +-- loader silently drops the evidence for every link it writes. +-- 3. §8 describes the seal as an attached artefact of a link, which is what this is. +-- +-- A seal is written for EVERY link, not only published ones. The seals on held and withdrawn links are +-- what make the review queue reviewable — without them „why is this one hidden?" has no answer. +-- +-- PII rail: `matched_fact` is a CLOSED VOCABULARY — 'seat:' | 'role:owner:' | +-- 'role:manager:' | 'eik'. It must NEVER carry the matched name. The registry deed's names are +-- read only to produce a boolean and never leave git-ignored scratch (ADR-0033 decision 5); storing one +-- here would put third-party personal data on the served surface, which #279 §9 forbids outright. The +-- audit enforces the vocabulary with a pattern check, because a schema cannot. + +CREATE TABLE IF NOT EXISTS interest_link_evidence ( + link_key TEXT PRIMARY KEY REFERENCES interest_links(link_key), + -- ADR-0033 decision 1, plus document_uncorroborated (ADR-0035: a name match whose COMPANY nothing + -- corroborated). CHECKed because the read gate filters on this column — SURFACED_OWNERSHIP admits + -- exactly 'document' and 'confirmed' — so an unlisted value is either a link that silently stops + -- surfacing or, if it collides with a publishing name, one that surfaces unproven. + evidence_kind TEXT NOT NULL + CHECK (evidence_kind IN ('document','confirmed','document_uncorroborated','refuted', + 'bar_joint_stock','unknown','outside_tr')), + -- owner | manager | NULL — only meaningful for evidence_kind='document'. The card renders this as + -- „вписан като …", so a stray value becomes a public claim about a named person's registry role. + registry_role TEXT CHECK (registry_role IS NULL OR registry_role IN ('owner','manager')), + matched_fact TEXT, -- the closed vocabulary above. NEVER a name. + -- TEXT, not INTEGER: a fieldEntryNumber like 20130716101007 is already 14 digits and exceeds the + -- exact-integer range once it round-trips through JSON/JS. + entry_number TEXT, + entry_date TEXT, -- ISO date of the registry entry the evidence rests on + lookup_date TEXT NOT NULL, -- when the deed was fetched — the freshness bound on the claim + rules_version TEXT NOT NULL, -- evidence.mjs RULES_VERSION; §8's monotonicity gate keys on this + -- live | terminated_owner_still | terminated_manager_still | terminated. + -- RE-DERIVED on every run and deliberately NOT part of the seal's permanence: it asserts a present + -- tense about a named person, and its freshness is bounded by the cache refresh cycle (ADR-0033 R3). + live_status TEXT NOT NULL + CHECK (live_status IN ('live','terminated_owner_still','terminated_manager_still', + 'terminated')), + sealed_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- The surface filters on evidence_kind ('document' | 'confirmed' publish); the audit scans by kind too. +CREATE INDEX IF NOT EXISTS idx_ile_kind ON interest_link_evidence(evidence_kind); diff --git a/packages/db/migrations/0010_publishing_gate_constraints.sql b/packages/db/migrations/0010_publishing_gate_constraints.sql new file mode 100644 index 00000000..b3e9d4cf --- /dev/null +++ b/packages/db/migrations/0010_publishing_gate_constraints.sql @@ -0,0 +1,123 @@ +-- The publishing-gate constraints, for EVERY database — freshly built and already-deployed alike (#279 §2). +-- +-- WHY THIS MIGRATION OWNS THEM RATHER THAN 0003 (todorkolev, #309). The obvious move is to add the CHECKs +-- to `interest_links` in 0003, where the table is declared. That does not work and is worse than not +-- trying: 0003 is ALREADY APPLIED everywhere, `CREATE TABLE IF NOT EXISTS` is a no-op against an existing +-- table, and `ship-related-persons.mjs` wipes ROWS (`DELETE FROM`, WIPE_ORDER), never definitions. So an +-- in-place CHECK would exist only on databases built after the edit — putting two different schemas under +-- one name, with the constraint absent precisely on the served database. That is where a hand-run +-- `UPDATE … SET status='published '` during an incident actually lands. +-- +-- So 0003 stays byte-identical to what was applied, and enforcement lives here, reached by both paths: +-- a fresh chain runs 0007 after 0003, and a deployed database gets it as a retrofit. One shape, one +-- mechanism. `packages/db/src/migrations.test.ts` holds the two shapes to the same rejections and the same +-- acceptances, so this stays true rather than merely intended. +-- +-- WHY TRIGGERS AND NOT A TABLE REBUILD: SQLite cannot add a CHECK in place, so the textbook route is +-- create-copy-drop-rename. That route is WRONG here, and a real `wrangler d1 migrations apply` proved it: +-- `interest_link_evidence` references interest_links(link_key), D1 enforces foreign keys, and +-- `PRAGMA defer_foreign_keys` does not survive the statement-by-statement execution wrangler performs — +-- the rebuild aborts with SQLITE_CONSTRAINT_FOREIGNKEY and the Durable Object rolls back. Dropping the +-- child first would work mechanically but would strip every evidence seal, and since the read gate +-- REQUIRES a seal that empties the public surface until the next monthly data run. +-- +-- A BEFORE INSERT/UPDATE trigger that RAISEs enforces the identical invariant for every writer — including +-- the hand-run UPDATE above, which no CHECK on a legacy table would ever have covered — with no rebuild, +-- no FK exposure and no seal loss. 0006 keeps its own CHECKs: it is NEW in this change, has never been +-- applied to a served environment, so declaring them there edits no applied history. +-- +-- WHY IT IS SAFE TO RE-RUN: migrations here are applied by a bare `wrangler d1 execute --file` with no +-- applied-migrations tracking (see related-persons-data.yml / deploy.yml), so re-application MUST be a +-- no-op. Everything below is `IF NOT EXISTS` over statements that converge. + +-- ── declarations ──────────────────────────────────────────────────────────────────────────────────── +-- The table-level `UNIQUE (xml_file, control_hash)` did not constrain what it was written for. SQLite +-- counts NULLs as DISTINCT and `control_hash` is genuinely optional at the source (the register omits +-- on some declarations), so every hashless declaration was mutually unique and re-imported +-- as a NEW row on each run, double-counting the stakes it carries. `xml_file` is also not unique across +-- folders — the register reuses basenames per year — which load.mjs already handles by namespacing the +-- declaration id with the folder. +-- +-- Replaced by an expression index over COALESCE(control_hash, '') plus folder_year: NULL-proof, and the +-- same natural key `id` already encodes. The column stays NULLABLE on purpose — NOT NULL would convert an +-- optional source field into a run-stopping loader failure, and a fabricated placeholder hash would +-- assert an integrity check nobody performed. +-- +-- No table rebuild is needed for this one: the table-level `UNIQUE (xml_file, control_hash)` declared in +-- 0003 STAYS (0003 is not edited — see the header), and the correct index is simply added alongside it. +-- The stale constraint is strictly WEAKER than the index — it treats NULLs as distinct, so it accepts a +-- superset — and therefore rejects nothing the index accepts. The index governs. +-- +-- Duplicates ALREADY stored under the old non-constraint would block the index, so they are collapsed +-- first, keeping the earliest row per natural key. +-- +-- The DELETE below changes live data, so it ANNOUNCES itself first (cefothe, #309): the count is emitted +-- before the rows go, and a run that removes nothing says so. Without it the only evidence a deployment +-- silently dropped declarations would be a row count nobody recorded beforehand. It needs no explicit +-- transaction — `wrangler d1 execute --file` runs the file as one implicit transaction, so a failure +-- anywhere below rolls the DELETE back with it rather than leaving a half-collapsed table. +-- The subtraction is parenthesised deliberately: `||` binds TIGHTER than `-` in SQLite, so without it +-- this reads as ('…' || countA) - (countB || '…') — two strings coerced to numbers — and reports a +-- meaningless figure instead of the row count. It did, before a real apply showed „notice: -2". +SELECT 'migration 0007: collapsing ' || + ((SELECT COUNT(*) FROM declarations) - + (SELECT COUNT(*) FROM (SELECT 1 FROM declarations + GROUP BY xml_file, folder_year, COALESCE(control_hash, '')))) || + ' duplicate declaration row(s) before the natural-key index' AS notice; +DELETE FROM declarations WHERE id NOT IN ( + SELECT MIN(id) FROM declarations GROUP BY xml_file, folder_year, COALESCE(control_hash, '') +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_declarations_natural_key + ON declarations(xml_file, folder_year, COALESCE(control_hash, '')); + +-- ── interest_links: status / interest_class ───────────────────────────────────────────────────────── +-- These two ARE the publishing gate: the surface is `status = 'published'` AND a surfaced +-- `interest_class`. A value that merely resembles one — 'published ' with a trailing space, the case +-- #279 §2 names — passes every writer and then fails the gate silently, hiding a link that should show. +-- +-- One trigger per operation, since SQLite triggers are per-event. `RAISE(ABORT)` rolls back the +-- statement, so a bad write fails loudly at its source instead of surfacing later as a missing link. +DROP TRIGGER IF EXISTS trg_interest_links_status_ins; +CREATE TRIGGER trg_interest_links_status_ins +BEFORE INSERT ON interest_links +WHEN NEW.status NOT IN ('published','internal','held','withdrawn','suppressed') + OR NEW.interest_class NOT IN ('private_ownership','family_ownership','ex_officio_board','management_role') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_links.status/interest_class outside the publishing-gate enum'); +END; + +DROP TRIGGER IF EXISTS trg_interest_links_status_upd; +CREATE TRIGGER trg_interest_links_status_upd +BEFORE UPDATE ON interest_links +WHEN NEW.status NOT IN ('published','internal','held','withdrawn','suppressed') + OR NEW.interest_class NOT IN ('private_ownership','family_ownership','ex_officio_board','management_role') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_links.status/interest_class outside the publishing-gate enum'); +END; + +-- ── interest_link_evidence: evidence_kind ─────────────────────────────────────────────────────────── +-- The read gate filters on this column — SURFACED_OWNERSHIP admits exactly 'document' and 'confirmed' — +-- so an unlisted value either silently stops a link surfacing or, if it collides with a publishing name, +-- surfaces one that was never proven. 0006 declares this as a CHECK for new databases; the trigger is +-- the same rule for those provisioned before it. +DROP TRIGGER IF EXISTS trg_ile_kind_ins; +CREATE TRIGGER trg_ile_kind_ins +BEFORE INSERT ON interest_link_evidence +WHEN NEW.evidence_kind NOT IN ('document','confirmed','document_uncorroborated','refuted', + 'bar_joint_stock','unknown','outside_tr') + OR (NEW.registry_role IS NOT NULL AND NEW.registry_role NOT IN ('owner','manager')) + OR NEW.live_status NOT IN ('live','terminated_owner_still','terminated_manager_still','terminated') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_link_evidence enum outside the ADR-0033/0035 vocabulary'); +END; + +DROP TRIGGER IF EXISTS trg_ile_kind_upd; +CREATE TRIGGER trg_ile_kind_upd +BEFORE UPDATE ON interest_link_evidence +WHEN NEW.evidence_kind NOT IN ('document','confirmed','document_uncorroborated','refuted', + 'bar_joint_stock','unknown','outside_tr') + OR (NEW.registry_role IS NOT NULL AND NEW.registry_role NOT IN ('owner','manager')) + OR NEW.live_status NOT IN ('live','terminated_owner_still','terminated_manager_still','terminated') +BEGIN + SELECT RAISE(ABORT, 'CHECK failed: interest_link_evidence enum outside the ADR-0033/0035 vocabulary'); +END; diff --git a/packages/db/src/amendments-bridge-lockstep.test.ts b/packages/db/src/amendments-bridge-lockstep.test.ts new file mode 100644 index 00000000..2f99e840 --- /dev/null +++ b/packages/db/src/amendments-bridge-lockstep.test.ts @@ -0,0 +1,42 @@ +// Issue #286 — the tender.id → УНП bridge UPDATE is duplicated in the full path (derive-amendments.sql) +// and the incremental path (refresh-slice.sql). Both must recover the SAME УНП, or a daily slice refresh +// on the production Worker would silently regress to the #286 bug while every functional test stayed green +// (the Worker runs refresh-slice.sql, not derive-amendments.sql). The prefer-EOP dedup DELETE deliberately +// diverges — the slice path additionally reconciles against the cumulative served `amendments` — so only +// the marked bridge block is held byte-identical here. This is the repo's established drift-guard form +// (see search-sql.test.ts / precompute-cohort.test.ts). +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); + +const START = '-- @bridge-lockstep start'; +const END = '-- @bridge-lockstep end'; + +function bridgeBlock(file: string): string { + const sql = readFileSync(resolve(root, file), 'utf8'); + const start = sql.indexOf(START); + const end = sql.indexOf(END, start); + expect(start, `no "${START}" marker in ${file}`).toBeGreaterThanOrEqual(0); + expect(end, `no "${END}" marker in ${file}`).toBeGreaterThan(start); + return sql.slice(start + START.length, end).trim(); +} + +describe('OCDS amendment bridge lockstep (issue #286)', () => { + it('the bridge UPDATE is byte-identical between derive-amendments.sql and refresh-slice.sql', () => { + const derive = bridgeBlock('scripts/derive-amendments.sql'); + const slice = bridgeBlock('scripts/refresh-slice.sql'); + + // Sanity: the extracted span really is the bridge UPDATE, not an empty/misplaced marker range. + expect(derive).toContain('UPDATE raw_amendments'); + expect(derive).toContain('raw_amendments.tender_ext_id'); + expect(derive).toContain('ORDER BY rt.unp LIMIT 1'); + expect(derive).toContain('ORDER BY rc.unp LIMIT 1'); + + // The guard: exact byte-equality. Deleting the block from refresh-slice.sql fails on the missing + // marker above; changing the recovery in either file fails here. + expect(slice).toBe(derive); + }); +}); diff --git a/packages/db/src/amendments-contract-resolve.test.ts b/packages/db/src/amendments-contract-resolve.test.ts new file mode 100644 index 00000000..f0b57d13 --- /dev/null +++ b/packages/db/src/amendments-contract-resolve.test.ts @@ -0,0 +1,634 @@ +// Issue #306 — EOP annexes whose annex-side number is in a different namespace than the contract number +// (the annex carries an internal number like 148846; the contract carries the buyer's filing number like +// Д-226), so the (unp, contract_number) join drops them out of every annex→contract rollup. The resolver in +// scripts/resolve-amendment-contracts.sql links them by the exact, currency- and contractor-matched +// value_before → signing_value anchor (measured 99.99% precision on the real corpus), uniquely, with chain +// propagation, leaving ambiguous / no-match annexes honestly unlinked. +// +// The resolver is a SEPARATE script that runs BEFORE derive-amendments.sql on the full-derive path only +// (review todorkolev #1 blocker: running after the prefer-EOP dedup resurrects OCDS twins; review +// nikimilenkov HIGH 1: the slice path's windowed raw_contracts can't answer "unique on the procedure"). +// These tests run the REAL scripts against SQLite via the sqlite3 CLI, exactly as runFullDerive composes +// them — resolver, then derive — so the value anchor, dedup, group rule, guards, and provenance are +// exercised as shipped. +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const initSchema = resolve(root, 'packages/db/migrations/0000_init.sql'); +// #305: promote-amendments.sql writes value_restated/value_treatment/value_suspect into served amendments. +const restatedMigration = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const valueSuspectMigration = resolve( + root, + 'packages/db/migrations/0007_amendment_value_suspect.sql', +); +const provenanceMigration = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +const workStagingSchema = resolve(root, 'scripts/work-staging-schema.sql'); +const resolveAmendments = resolve(root, 'scripts/resolve-amendment-contracts.sql'); +const deriveAmendments = resolve(root, 'scripts/derive-amendments.sql'); +const promoteAmendments = resolve(root, 'scripts/promote-amendments.sql'); + +function sqlite(dbPath: string, sql: string): void { + execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8', stdio: 'pipe' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function readScriptCapture(dbPath: string, path: string): string { + return execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + encoding: 'utf8', + }); +} + +// The full-derive composition: value resolver first (review todorkolev #1), then the derive rollup. +function runFullDerive(dbPath: string): void { + readScript(dbPath, resolveAmendments); + readScript(dbPath, deriveAmendments); +} + +// The numeric rows the resolver prints; the #306 diagnostic is its only 2-column row +// (annexes_value_linked | eop_annexes_still_unlinked). +function diagRows(out: string): number[][] { + return out + .split('\n') + .map((line) => line.trim()) + .filter((line) => /^\d+(\|\d+)*$/.test(line)) + .map((line) => line.split('|').map(Number)); +} + +let dir: string; +let db: string; + +beforeEach(() => { + dir = mkdtempSync(resolve(tmpdir(), 'amendments-resolve-')); + db = resolve(dir, 'work.sqlite'); + readScript(db, initSchema); + readScript(db, restatedMigration); + readScript(db, valueSuspectMigration); + readScript(db, provenanceMigration); + readScript(db, workStagingSchema); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe('#306 amendment→contract value-anchor resolver', () => { + it('links a single-contract procedure whose annex number differs (00017 shape) by exact value', () => { + // UNP with ONE contract "ОП-3-016/…" signing 5800; annex carries internal number 11725, value_before + // 5800 (exact). The number join fails; the value anchor links it. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00017-2020-0041','ОП-3-016/03.02.2021г.',5800,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, value_delta, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00017-2020-0041','11725','2026-03-05','A1',5800,5800,0,'BGN');`, + ); + runFullDerive(db); + + // The annex's contract_number is rewritten to the real filing number → it now links. + expect( + sqliteJson<{ contract_number: string; contract_number_raw: string; link_method: string }>( + db, + "SELECT contract_number, contract_number_raw, link_method FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([ + { + contract_number: 'ОП-3-016/03.02.2021г.', + contract_number_raw: '11725', + link_method: 'value_anchor', + }, + ]); + // The contract picks up the annex in its rollup. + expect( + sqliteJson<{ annex_count: number }>( + db, + "SELECT annex_count FROM raw_contracts WHERE contract_number='ОП-3-016/03.02.2021г.'", + ), + ).toEqual([{ annex_count: 1 }]); + }); + + it('disambiguates a multi-contract (multi-lot) procedure by value (00011 shape)', () => { + // Two contracts on one procedure: 387-2020 (signing 64000) and 388-2020 (signing 56000). An unlinked + // annex carries internal number 2886, value_before 56000 → must link to 388-2020, NEVER 387-2020. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','387-2020',64000,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','388-2020',56000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, value_delta, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00011-2020-0002','2886','2026-03-05','A1',56000,55552.5,NULL,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '388-2020' }]); + expect( + sqliteJson<{ contract_number: string; annex_count: number }>( + db, + "SELECT contract_number, annex_count FROM raw_contracts WHERE unp='00011-2020-0002' ORDER BY contract_number", + ), + ).toEqual([ + { contract_number: '387-2020', annex_count: 0 }, + { contract_number: '388-2020', annex_count: 1 }, + ]); + }); + + it('propagates the resolved target across a chain sharing the annex number, so current_value is the last step', () => { + // Three annexes share internal number 2886; only the FIRST carries value_before = signing (56000), the + // later two carry the running cumulative. All must attach to 388-2020, and current_value must be the + // LAST step's value_after (53580.21), not the first. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','387-2020',64000,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','388-2020',56000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00011-2020-0002','2886','2026-03-05','A1',56000,55552.5,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00011-2020-0002','2886','2026-03-06','A2',55552.5,55306.05,'BGN'), + ('eop:annexes:2026-03-05','2026-03-07','00011-2020-0002','2886','2026-03-07','A3',55306.05,53580.21,'BGN');`, + ); + runFullDerive(db); + + // All three annexes now carry the resolved contract_number. + expect( + sqliteJson<{ n: number }>( + db, + "SELECT COUNT(*) AS n FROM raw_amendments WHERE unp='00011-2020-0002' AND contract_number='388-2020'", + ), + ).toEqual([{ n: 3 }]); + // annex_count = 3, current_value = the latest step's after-value. + expect( + sqliteJson<{ annex_count: number; current_value: number }>( + db, + "SELECT annex_count, current_value FROM raw_contracts WHERE contract_number='388-2020'", + ), + ).toEqual([{ annex_count: 3, current_value: 53580.21 }]); + }); + + it('propagates the target to a VALUE-LESS chain member so the chain does not break (review MEDIUM 2)', () => { + // A1 carries value_before = signing (unique anchor → 388-2020); A2 is an administrative annex with NO + // value_before (a term/scope change). A2 shares the annex number 2886, so it must INHERIT 388-2020 — + // else the chain breaks and annex_count under-counts. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','387-2020',64000,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00011-2020-0002','388-2020',56000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00011-2020-0002','2886','2026-03-05','A1',56000,55552.5,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00011-2020-0002','2886','2026-03-06','A2',NULL,NULL,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ document_number: string; contract_number: string; link_method: string | null }>( + db, + 'SELECT document_number, contract_number, link_method FROM raw_amendments ORDER BY document_number', + ), + ).toEqual([ + { document_number: 'A1', contract_number: '388-2020', link_method: 'value_anchor' }, + { document_number: 'A2', contract_number: '388-2020', link_method: 'value_anchor' }, + ]); + expect( + sqliteJson<{ annex_count: number }>( + db, + "SELECT annex_count FROM raw_contracts WHERE contract_number='388-2020'", + ), + ).toEqual([{ annex_count: 2 }]); + }); + + it('does NOT attach an ambiguous chain member to the group target (review todorkolev #2)', () => { + // Procedure: C-1 @ 500, C-2 @ 500, C-3 @ 1000. Annex 999: A1 value_before 1000 → C-3 (unique anchor); + // A2 value_before 500 → matches C-1 AND C-2 (ambiguous). A2 must stay unlinked — it carries its own + // contradicting evidence — and must NOT corrupt C-3's current_value. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00050-2020-0001','C-1',500,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00050-2020-0001','C-2',500,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00050-2020-0001','C-3',1000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00050-2020-0001','999','2026-03-05','A1',1000,900,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00050-2020-0001','999','2026-03-06','A2',500,450,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ document_number: string; contract_number: string }>( + db, + 'SELECT document_number, contract_number FROM raw_amendments ORDER BY document_number', + ), + ).toEqual([ + { document_number: 'A1', contract_number: 'C-3' }, // unique anchor linked + { document_number: 'A2', contract_number: '999' }, // ambiguous → stays unlinked + ]); + // C-3 keeps A1 only; current_value is A1's after-value (900), NOT the ambiguous A2's 450. + expect( + sqliteJson<{ annex_count: number; current_value: number }>( + db, + "SELECT annex_count, current_value FROM raw_contracts WHERE contract_number='C-3'", + ), + ).toEqual([{ annex_count: 1, current_value: 900 }]); + }); + + it('links each annex of a lot-base group to its OWN lot by its own unique match (real 00026 shape)', () => { + // The annex-side number is a LOT-BASE shared across two lots (real corpus: 20РП-У50А015 → …-Л01 @ + // 22569.98 AND …-Л03 @ 28557.50). Each annex exactly-uniquely matches its OWN lot, so BOTH link — the + // "disagreement" between siblings is benign and must NOT void the individually-trustworthy direct hits. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00026-2020-0027','20РП-У50А015-Л01',22569.98,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00026-2020-0027','20РП-У50А015-Л03',28557.5,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00026-2020-0027','20РП-У50А015','2026-03-05','A1',22569.98,22000,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00026-2020-0027','20РП-У50А015','2026-03-06','A2',28557.5,28000,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ document_number: string; contract_number: string }>( + db, + "SELECT document_number, contract_number FROM raw_amendments WHERE unp='00026-2020-0027' ORDER BY document_number", + ), + ).toEqual([ + { document_number: 'A1', contract_number: '20РП-У50А015-Л01' }, + { document_number: 'A2', contract_number: '20РП-У50А015-Л03' }, + ]); + expect( + sqliteJson<{ contract_number: string; annex_count: number }>( + db, + "SELECT contract_number, annex_count FROM raw_contracts WHERE unp='00026-2020-0027' ORDER BY contract_number", + ), + ).toEqual([ + { contract_number: '20РП-У50А015-Л01', annex_count: 1 }, + { contract_number: '20РП-У50А015-Л03', annex_count: 1 }, + ]); + }); + + it('withholds PROPAGATION when the group disagrees, but keeps the direct hits (review MEDIUM 1, revised)', () => { + // Two direct hits disagree (lot-base spread), plus a value-less admin annex A3 sharing the number. The + // two direct hits still link to their own lots; A3 has no agreed target to inherit → stays unlinked. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00060-2020-0001','D-1',100,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00060-2020-0001','D-2',200,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00060-2020-0001','777','2026-03-05','A1',100,90,'BGN'), + ('eop:annexes:2026-03-05','2026-03-06','00060-2020-0001','777','2026-03-06','A2',200,180,'BGN'), + ('eop:annexes:2026-03-05','2026-03-07','00060-2020-0001','777','2026-03-07','A3',NULL,NULL,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ document_number: string; contract_number: string }>( + db, + "SELECT document_number, contract_number FROM raw_amendments WHERE unp='00060-2020-0001' ORDER BY document_number", + ), + ).toEqual([ + { document_number: 'A1', contract_number: 'D-1' }, // own unique match stands + { document_number: 'A2', contract_number: 'D-2' }, // own unique match stands + { document_number: 'A3', contract_number: '777' }, // no agreed target → no propagation + ]); + expect( + sqliteJson<{ contract_number: string; annex_count: number }>( + db, + "SELECT contract_number, annex_count FROM raw_contracts WHERE unp='00060-2020-0001' ORDER BY contract_number", + ), + ).toEqual([ + { contract_number: 'D-1', annex_count: 1 }, + { contract_number: 'D-2', annex_count: 1 }, + ]); + }); + + it('counts cumulative-staging duplicates of one contract as a SINGLE candidate (review HIGH 2)', () => { + // The same logical contract is staged from TWO daily EOP buckets (raw_contracts is cumulative). Without + // deduping candidates, the lone annex would see n_match = 2 and be refused; the resolver must collapse + // the duplicate to one candidate and link. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00070-2020-0001','FILING-7',12345,'BGN'), + ('eop:contracts:2026-03-06','2026-03-06','00070-2020-0001','FILING-7',12345,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-06','2026-03-06','00070-2020-0001','88001','2026-03-06','A1',12345,12000,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: 'FILING-7' }]); + }); + + it('applies an EIK guard — a value match onto a different contractor is refused (review MEDIUM 5)', () => { + // Same unp, value, currency, but the annex is contractor 222222222 and the only value-matching contract + // is contractor 111111111 → the row's own EIK contradicts the target → no link. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency, contractor_eik) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00080-2020-0001','FILING-8',9000,'BGN','111111111'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency, contractor_eik) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00080-2020-0001','44001','2026-03-05','A1',9000,8500,'BGN','222222222');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '44001' }]); // unchanged → EIK guard held + }); + + it('leaves a value-AMBIGUOUS annex unlinked (two contracts share the signing value)', () => { + // Both contracts on the procedure have signing 5000; the annex value_before 5000 matches BOTH — the + // resolver must refuse (an honest gap beats a coin-flip). + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00099-2020-0001','LOT-A',5000,'BGN'), + ('eop:contracts:2026-03-05','2026-03-05','00099-2020-0001','LOT-B',5000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00099-2020-0001','7777','2026-03-05','A1',5000,4800,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '7777' }]); // unchanged → stays unlinked + expect( + sqliteJson<{ total: number }>( + db, + "SELECT COALESCE(SUM(annex_count),0) AS total FROM raw_contracts WHERE unp='00099-2020-0001'", + ), + ).toEqual([{ total: 0 }]); + }); + + it('leaves a NO-MATCH annex unlinked (value_before matches no contract on the procedure)', () => { + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00099-2020-0002','30',700000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00099-2020-0002','24035','2026-03-05','A1',123456,120000,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '24035' }]); // unchanged + }); + + it('holds the exact-cent tolerance — a value 3 BGN off does NOT match (pins the 0.005 constant)', () => { + // signing 5000, value_before 5003 → 3.00 apart: far under a naive "within 5" band but far over the + // 0.5-стотинка exact gate. Must stay unlinked; this is the whole basis of the 99.99% precision claim. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00087-2020-0001','FILING-87',5000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00087-2020-0001','47001','2026-03-05','A1',5003,4800,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '47001' }]); // unchanged → 3 BGN is not an exact-cent match + }); + + it('applies a currency guard — a BGN-valued annex does not match a same-number EUR contract', () => { + // signing_value numerically equals value_before but the currencies differ → not a real value match. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00207-2020-0171','FILING-1',66820,'EUR'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00207-2020-0171','9001','2026-03-05','A1',66820,66000,'BGN');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '9001' }]); // unchanged → currency guard held + }); + + it('does not match blank-currency against blank-currency (review LOW 1)', () => { + // Both sides carry no explicit currency. Under so tight a gate the resolver must NOT silently agree via + // a 'BGN' default — an explicit currency is required on both sides. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00085-2020-0001','FILING-85',7500,''); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00085-2020-0001','45001','2026-03-05','A1',7500,7000,'');`, + ); + runFullDerive(db); + + expect( + sqliteJson<{ contract_number: string }>( + db, + "SELECT contract_number FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '45001' }]); // unchanged → no blank-vs-blank match + }); + + it('does NOT resurrect an OCDS twin: resolver runs before the prefer-EOP dedup (review todorkolev #1 blocker)', () => { + // Contract Д-226 already has an OCDS annex on it. An EOP annex carries the unlinked internal number + // 148846 and value_before = signing (5000). Because the resolver runs BEFORE the #286 prefer-EOP DELETE, + // the EOP annex is moved onto Д-226 first, the OCDS twin is then dropped, and annex_count = 1. If the + // order were reversed the twin would survive and annex_count would be 2 (the twin-dedup #303 gate would + // then hard-fail the whole derive). + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00090-2020-0001','Д-226',5000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('ocds:annexes:2026-03-05','2026-03-05','00090-2020-0001','Д-226','2026-03-05','ocds-e82gsb-1',5000,NULL,'BGN'), + ('eop:annexes:2026-03-05','2026-03-05','00090-2020-0001','148846','2026-03-05','148846-1',5000,4500,'BGN');`, + ); + runFullDerive(db); + + // Exactly one annex survives on Д-226 (the EOP one), and it drives current_value. + expect( + sqliteJson<{ annex_count: number; current_value: number }>( + db, + "SELECT annex_count, current_value FROM raw_contracts WHERE contract_number='Д-226'", + ), + ).toEqual([{ annex_count: 1, current_value: 4500 }]); + // No twin duplication: exactly one row lands on Д-226 for this procedure. + expect( + sqliteJson<{ n: number }>( + db, + "SELECT COUNT(*) AS n FROM raw_amendments WHERE unp='00090-2020-0001' AND contract_number='Д-226'", + ), + ).toEqual([{ n: 1 }]); + }); + + it('is GATED to the resolver script: derive-amendments.sql alone does not resolve (review HIGH 1)', () => { + // The slice path runs derive-amendments.sql WITHOUT the resolver. Running derive alone must leave a + // resolvable annex untouched — proving the link is gated to the full-derive-only resolver script. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00095-2020-0001','FILING-95',5800,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00095-2020-0001','11725','2026-03-05','A1',5800,5600,'BGN');`, + ); + readScript(db, deriveAmendments); // derive ONLY — no resolver + + expect( + sqliteJson<{ contract_number: string; link_method: string | null }>( + db, + "SELECT contract_number, link_method FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number: '11725', link_method: null }]); // untouched + }); + + it('is idempotent: a second full derive links nothing new and keeps annex_count stable', () => { + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00096-2020-0001','FILING-96',5800,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00096-2020-0001','11725','2026-03-05','A1',5800,5600,'BGN');`, + ); + runFullDerive(db); + const secondPass = readScriptCapture(db, resolveAmendments); + readScript(db, deriveAmendments); + + // Second pass over already-resolved staging: the row now links by number so grp is empty and no new + // rewrite happens. The diagnostic still reports the one value-linked row (link_method persists) and 0 + // still unlinked — i.e. nothing new linked, nothing left over. + const twoCol = diagRows(secondPass).filter((r) => r.length === 2); + expect(twoCol[twoCol.length - 1]).toEqual([1, 0]); + // contract_number stays resolved; annex_count stable at 1; provenance preserved. + expect( + sqliteJson<{ annex_count: number }>( + db, + "SELECT annex_count FROM raw_contracts WHERE contract_number='FILING-96'", + ), + ).toEqual([{ annex_count: 1 }]); + expect( + sqliteJson<{ contract_number_raw: string; link_method: string }>( + db, + "SELECT contract_number_raw, link_method FROM raw_amendments WHERE document_number='A1'", + ), + ).toEqual([{ contract_number_raw: '11725', link_method: 'value_anchor' }]); + }); + + it('carries provenance through promote into the served amendments table (review MEDIUM 4)', () => { + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00097-2020-0001','FILING-97',5800,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00097-2020-0001','11725','2026-03-05','A1',5800,5600,'BGN');`, + ); + runFullDerive(db); + readScript(db, promoteAmendments); + + // The served row is enumerable as value-linked, and the original annex number survives as provenance. + expect( + sqliteJson<{ contract_number: string; contract_number_raw: string; link_method: string }>( + db, + "SELECT contract_number, contract_number_raw, link_method FROM amendments WHERE unp='00097-2020-0001'", + ), + ).toEqual([ + { contract_number: 'FILING-97', contract_number_raw: '11725', link_method: 'value_anchor' }, + ]); + // The audit count nikimilenkov asked for is a plain query on the served side. + expect( + sqliteJson<{ n: number }>( + db, + "SELECT COUNT(*) AS n FROM amendments WHERE link_method='value_anchor'", + ), + ).toEqual([{ n: 1 }]); + }); + + it('a resolved annex does not collide with a native annex sharing document_number on the target (review MEDIUM 3)', () => { + // Contract FILING-98 already has a native annex whose document_number is 'DOC-1'. An unlinked annex 55501 + // resolves to FILING-98 and ALSO has document_number 'DOC-1'. Keying the resolved row on its original + // annex number keeps both rows distinct — the real annex is not silently dropped by the dedup. + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00098-2020-0001','FILING-98',5000,'BGN'); + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00098-2020-0001','FILING-98','2026-03-04','DOC-1',6000,5500,'BGN'), + ('eop:annexes:2026-03-05','2026-03-05','00098-2020-0001','55501','2026-03-05','DOC-1',5000,4500,'BGN');`, + ); + runFullDerive(db); + + // Both annexes survive on FILING-98 (the native one and the value-linked one). + expect( + sqliteJson<{ annex_count: number }>( + db, + "SELECT annex_count FROM raw_contracts WHERE contract_number='FILING-98'", + ), + ).toEqual([{ annex_count: 2 }]); + }); + + it('never touches an annex that already links by contract_number, and emits the #306 diagnostic', () => { + sqlite( + db, + `INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency) VALUES + ('eop:contracts:2026-03-05','2026-03-05','00001-2020-0001','Д-100',5000,'BGN'), -- annex links by value (unlinked number) + ('eop:contracts:2026-03-05','2026-03-05','00002-2020-0001','Д-200',9000,'BGN'); -- annex already links by number + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05','2026-03-05','00001-2020-0001','55501','2026-03-05','A1',5000,4500,'BGN'), + ('eop:annexes:2026-03-05','2026-03-05','00002-2020-0001','Д-200','2026-03-05','A2',9000,8000,'BGN');`, + ); + const out = readScriptCapture(db, resolveAmendments); + readScript(db, deriveAmendments); + + // A1 got resolved; A2 (already linked by number) is untouched and keeps NULL provenance. + expect( + sqliteJson<{ document_number: string; contract_number: string; link_method: string | null }>( + db, + 'SELECT document_number, contract_number, link_method FROM raw_amendments ORDER BY document_number', + ), + ).toEqual([ + { document_number: 'A1', contract_number: 'Д-100', link_method: 'value_anchor' }, + { document_number: 'A2', contract_number: 'Д-200', link_method: null }, + ]); + // Diagnostic: exactly 1 value-linked, 0 still unlinked. The resolver's only 2-column row. + const twoCol = diagRows(out).filter((r) => r.length === 2); + expect(twoCol[twoCol.length - 1]).toEqual([1, 0]); + }); +}); diff --git a/packages/db/src/amendments-ocds-link.test.ts b/packages/db/src/amendments-ocds-link.test.ts new file mode 100644 index 00000000..e86eb5bb --- /dev/null +++ b/packages/db/src/amendments-ocds-link.test.ts @@ -0,0 +1,243 @@ +// Issue #286 — OCDS amendments must link to their contract via the recovered УНП, without +// double-counting the EOP annexes or letting an OCDS "before" value understate current_value. +// +// Runs the REAL scripts (derive-amendments.sql → promote-amendments.sql) against SQLite via the +// sqlite3 CLI, exactly as the ETL does, so the bridge + prefer-EOP dedup + value semantics are +// exercised as shipped — not a hand-copied mirror. +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const initSchema = resolve(root, 'packages/db/migrations/0000_init.sql'); +// #305 Tier-2: promote-amendments.sql writes value_restated/value_treatment to served amendments. +const migration6 = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7 = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8 = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +const workStagingSchema = resolve(root, 'scripts/work-staging-schema.sql'); +const deriveAmendments = resolve(root, 'scripts/derive-amendments.sql'); +const promoteAmendments = resolve(root, 'scripts/promote-amendments.sql'); + +function sqlite(dbPath: string, sql: string): void { + execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8', stdio: 'pipe' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + // Enforce FK constraints (mirrors refresh-slice.test.ts) so the promotion is validated against the + // served schema, not run with FK checks silently off. + execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +// Captures sqlite3 stdout (the SELECT results a `.read` prints) so a test can assert the diagnostic +// numbers the ETL emits, not just the table state. +function readScriptCapture(dbPath: string, path: string): string { + return execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + encoding: 'utf8', + }); +} + +// The numeric rows derive-amendments.sql prints (its diagnostics + the final summary), split into integer +// arrays and identified by column count: 1 col = ocds_ambiguous_bridges, 2 = dropped/excess-over-eop, +// 4 = the run summary. sqlite3's default output separates columns by '|' and rows by newlines. +function diagRows(out: string): number[][] { + return out + .split('\n') + .map((line) => line.trim()) + .filter((line) => /^\d+(\|\d+)*$/.test(line)) + .map((line) => line.split('|').map(Number)); +} + +// EOP УНП for the twin contract (90029), the OCDS-only contract (55500), and the OCDS-only contract +// whose procedure is contract-only — a "synthetic tender" absent from raw_tenders (77700). +const UNP_TWIN = '00044-2022-0146'; +const UNP_ONLY = '00099-2022-0009'; +const UNP_SYNTH = '00077-2022-0007'; +// A fourth OCDS annex whose procedure bridges NOWHERE — its tender.id is in neither raw_tenders nor +// raw_contracts. On the live corpus ~3/4,800 OCDS amendments are like this (#286 links 4,797/4,800); +// they honestly keep the OCID in `unp` because they can't be keyed to a contract yet — NOT dropped. +const OCID_UNBRIDGED = 'ocds-e82gsb-666666'; + +let dir: string; +let db: string; + +beforeEach(() => { + dir = mkdtempSync(resolve(tmpdir(), 'amendments-ocds-')); + db = resolve(dir, 'work.sqlite'); + readScript(db, initSchema); // served `amendments` + `contracts` + readScript(db, migration6); // #305 Tier-2 value_restated/value_treatment on served amendments + readScript(db, migration7); // #305 residual value_suspect on served amendments + readScript(db, migration8); // #306 provenance columns on served amendments + readScript(db, workStagingSchema); // raw_* staging + + // Two EOP procedures: T1 (contract 90029) already has an EOP annex; T2 (contract 55500) has NONE. + sqlite( + db, + `INSERT INTO raw_tenders (source, fetched_at, tender_id, unp) VALUES + ('eop:tenders:2026-03-05', '2026-03-05', 'T1', '${UNP_TWIN}'), + ('eop:tenders:2026-03-05', '2026-03-05', 'T2', '${UNP_ONLY}'); + INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency, tender_ext_id) VALUES + ('eop:contracts:2026-03-05', '2026-03-05', '${UNP_TWIN}', '90029', 21602081.98, 'EUR', 'T1'), + ('eop:contracts:2026-03-05', '2026-03-05', '${UNP_ONLY}', '55500', 500000, 'EUR', 'T2'), + -- 77700's procedure is contract-only: it has NO raw_tenders row, so the bridge must fall back + -- to raw_contracts.tender_ext_id (T3) to recover the УНП. + ('eop:contracts:2026-03-05', '2026-03-05', '${UNP_SYNTH}', '77700', 300000, 'EUR', 'T3'); + -- EOP annex for 90029 carries the correct after-value (27.4M). + INSERT INTO raw_amendments (source, fetched_at, unp, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('eop:annexes:2026-03-05', '2026-03-05', '${UNP_TWIN}', '90029', '2026-03-05', 'E1', 21602081.98, 27435415.31, 'EUR'); + -- OCDS twin of the 90029 annex: unp is the OCID, value is the pre-amendment number stored as value_before. + INSERT INTO raw_amendments (source, fetched_at, unp, tender_ext_id, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('ocds:2026-03-05', '2026-03-05', 'ocds-e82gsb-245534', 'T1', '90029', '2026-03-05', 'O1', 21602081.98, NULL, 'EUR'); + -- OCDS-only annex for 55500: exists in NO EOP feed. This is the row #286 wants to make visible. + INSERT INTO raw_amendments (source, fetched_at, unp, tender_ext_id, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('ocds:2026-04-01', '2026-04-01', 'ocds-e82gsb-999999', 'T2', '55500', '2026-04-01', 'O2', 480000, NULL, 'EUR'); + -- OCDS-only annex for 77700 (contract-only procedure) — bridges via the raw_contracts fallback. + INSERT INTO raw_amendments (source, fetched_at, unp, tender_ext_id, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('ocds:2026-04-01', '2026-04-01', 'ocds-e82gsb-777777', 'T3', '77700', '2026-04-01', 'O3', 300000, NULL, 'EUR'); + -- OCDS annex whose tender.id (T_MISSING) is in NEITHER raw_tenders nor raw_contracts — unbridgeable. + INSERT INTO raw_amendments (source, fetched_at, unp, tender_ext_id, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('ocds:2026-04-01', '2026-04-01', '${OCID_UNBRIDGED}', 'T_MISSING', '66600', '2026-04-01', 'O4', 300000, NULL, 'EUR');`, + ); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe('OCDS amendment → contract linkage (issue #286)', () => { + it('bridges the УНП, drops EOP twins, and never understates current_value', () => { + readScript(db, deriveAmendments); + + // Bridge: every *bridgeable* OCDS amendment now carries the real УНП. The one unbridgeable procedure + // (tender.id in neither raw_tenders nor raw_contracts) honestly keeps its OCID — it is NOT dropped. + const stillOcid = sqliteJson<{ contract_number: string; unp: string }>( + db, + "SELECT contract_number, unp FROM raw_amendments WHERE source LIKE 'ocds:%' AND unp LIKE 'ocds-%'", + ); + expect(stillOcid).toEqual([{ contract_number: '66600', unp: OCID_UNBRIDGED }]); + + // Prefer-EOP dedup: the 90029 OCDS twin is gone; the OCDS-only rows survive with their УНП — + // 55500 bridged via raw_tenders, 77700 via the raw_contracts synthetic-tender fallback, and 66600 + // stays on its OCID (unbridgeable, but surfaced rather than silently deleted). + const ocds = sqliteJson<{ contract_number: string; unp: string }>( + db, + "SELECT contract_number, unp FROM raw_amendments WHERE source LIKE 'ocds:%' ORDER BY contract_number", + ); + expect(ocds).toEqual([ + { contract_number: '55500', unp: UNP_ONLY }, + { contract_number: '66600', unp: OCID_UNBRIDGED }, + { contract_number: '77700', unp: UNP_SYNTH }, + ]); + + // The value trap: 90029 rolls up the EOP after-value (27.4M), never the stale OCDS 21.6M. + const twin = sqliteJson<{ annex_count: number; current_value: number | null }>( + db, + "SELECT annex_count, current_value FROM raw_contracts WHERE contract_number = '90029'", + ); + expect(twin[0]?.annex_count).toBe(1); + expect(twin[0]?.current_value).toBe(27435415.31); + + // OCDS-only annex is now visible on its contract (annex_count = 1); current_value stays NULL because + // OCDS cannot know the after-value (honest — no fabricated figure). + const only = sqliteJson<{ annex_count: number; current_value: number | null }>( + db, + "SELECT annex_count, current_value FROM raw_contracts WHERE contract_number = '55500'", + ); + expect(only[0]?.annex_count).toBe(1); + expect(only[0]?.current_value).toBeNull(); + }); + + it('promotes served amendments, keeping only the unbridgeable OCID as the honest residual', () => { + readScript(db, deriveAmendments); + readScript(db, promoteAmendments); + + // The issue's mass symptom is gone (4,800 → the handful with no bridge). The one procedure that + // bridges nowhere is promoted with its OCID still in unp — surfaced honestly, not silently dropped + // and not fabricated onto a contract. promote-amendments.sql intentionally promotes every staged row. + const dead = sqliteJson<{ contract_number: string; unp: string }>( + db, + "SELECT contract_number, unp FROM amendments WHERE unp LIKE 'ocds-%'", + ); + expect(dead).toEqual([{ contract_number: '66600', unp: OCID_UNBRIDGED }]); + + // The served rows: one EOP annex (90029), two bridged OCDS-only annexes (55500, 77700) keyed by real + // УНП, and the unbridgeable OCDS residual (66600) still on its OCID. + const served = sqliteJson<{ contract_number: string; unp: string; source: string }>( + db, + "SELECT contract_number, unp, CASE WHEN source LIKE 'ocds:%' THEN 'ocds' ELSE 'eop' END AS source FROM amendments ORDER BY contract_number", + ); + expect(served).toEqual([ + { contract_number: '55500', unp: UNP_ONLY, source: 'ocds' }, + { contract_number: '66600', unp: OCID_UNBRIDGED, source: 'ocds' }, + { contract_number: '77700', unp: UNP_SYNTH, source: 'ocds' }, + { contract_number: '90029', unp: UNP_TWIN, source: 'eop' }, + ]); + }); + + it('refuses to bridge (keeps the OCID) when a tender.id resolves to more than one УНП', () => { + // The domain is 1-to-1 (one procedure = one УНП). Give T2 a SECOND distinct УНП in raw_tenders. Rather + // than pick one arbitrarily (which would mis-attribute every annex of the losing procedure), the bridge + // must REFUSE: 55500's annex keeps its OCID as an honest residual, and the ocds_ambiguous_bridges + // diagnostic reports exactly one refusal (review nikimilenkov LOW 1). + sqlite( + db, + `INSERT INTO raw_tenders (source, fetched_at, tender_id, unp) VALUES + ('eop:tenders:2026-03-05', '2026-03-05', 'T2', 'ZZ-9999-9999');`, + ); + const out = readScriptCapture(db, deriveAmendments); + + const row = sqliteJson<{ unp: string }>( + db, + "SELECT unp FROM raw_amendments WHERE source LIKE 'ocds:%' AND contract_number = '55500'", + ); + expect(row).toEqual([{ unp: 'ocds-e82gsb-999999' }]); + + // The single-column diagnostic row reports the one ambiguous refusal. + expect(diagRows(out).find((r) => r.length === 1)).toEqual([1]); + }); + + it('recovers the УНП from raw_tenders in preference to raw_contracts (COALESCE order)', () => { + // One tender_ext_id (TP) resolves to DIFFERENT УНП in the two tables. The bridge tries raw_tenders + // first, so it must recover UNP-FROM-TENDERS — swapping the two COALESCE arms would fail this (#286). + sqlite( + db, + `INSERT INTO raw_tenders (source, fetched_at, tender_id, unp) VALUES + ('eop:tenders:2026-03-05', '2026-03-05', 'TP', 'UNP-FROM-TENDERS'); + INSERT INTO raw_contracts (source, fetched_at, unp, contract_number, signing_value, currency, tender_ext_id) VALUES + ('eop:contracts:2026-03-05', '2026-03-05', 'UNP-FROM-CONTRACTS', 'PREC-1', 100, 'EUR', 'TP'); + INSERT INTO raw_amendments (source, fetched_at, unp, tender_ext_id, contract_number, published_at, document_number, value_before, value_after, currency) VALUES + ('ocds:2026-04-01', '2026-04-01', 'ocds-e82gsb-tp', 'TP', 'PREC-1', '2026-04-01', 'OP', 100, NULL, 'EUR');`, + ); + readScript(db, deriveAmendments); + + expect( + sqliteJson<{ unp: string }>( + db, + "SELECT unp FROM raw_amendments WHERE contract_number = 'PREC-1'", + ), + ).toEqual([{ unp: 'UNP-FROM-TENDERS' }]); + }); + + it('emits the residual diagnostics (dropped / excess-over-eop / ambiguous) for monitoring', () => { + // The base fixture has exactly one twin (90029: one OCDS annex vs one EOP annex) → dropped = 1, + // excess-over-eop = 0; and no ambiguous procedures → 0. These are the numbers the PR promises are + // "surfaced, not hidden" — pin them so a wrong CASE branch, or moving a diagnostic SELECT below the + // DELETE (which would zero the dropped/excess counts), fails CI (review nikimilenkov MEDIUM 4). + const out = readScriptCapture(db, deriveAmendments); + const rows = diagRows(out); + expect(rows.find((r) => r.length === 2)).toEqual([1, 0]); // dropped, excess-over-eop + expect(rows.find((r) => r.length === 1)).toEqual([0]); // ocds_ambiguous_bridges + }); +}); diff --git a/packages/db/src/amendments-slice-resolve.test.ts b/packages/db/src/amendments-slice-resolve.test.ts new file mode 100644 index 00000000..93dee1db --- /dev/null +++ b/packages/db/src/amendments-slice-resolve.test.ts @@ -0,0 +1,305 @@ +// Issue #306 (PR #308 review todorkolev "дневните обновявания") — the value-anchor resolver must also run on +// the daily/slice + Worker path, not only the full derive. `scripts/refresh-slice.sql` carries a corpus-safe +// resolver whose candidate contracts come from the served `contracts` table (the whole corpus) UNIONed with +// the current window's raw_contracts, so "unique on the procedure" is asked corpus-wide — closing the gap that +// kept the full-path resolver off the slice (windowed raw_contracts could only answer "unique in the window"). +// +// These tests run the REAL refresh-slice.sql against SQLite via the sqlite3 CLI, exactly as the CLI slice path +// and the Worker compose it: a first window promotes a served corpus, then a later window brings a +// namespace-mismatched annex (its annex-side number is in a different namespace than the contract's filing +// number). The annex links by the exact value_before → signing_value anchor, keeps its provenance, and rolls +// onto the prior-window target — while a corpus-ambiguous annex that would look unique in the window stays +// honestly unlinked. +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migrations = [ + 'packages/db/migrations/0000_init.sql', + 'packages/db/migrations/0001_flow_pairs_bidder_index.sql', + 'packages/db/migrations/0002_current_value_currency.sql', + 'packages/db/migrations/0003_related_persons_foundation.sql', + // #305: refresh-slice.sql writes value_restated/value_treatment/value_suspect into served amendments. + 'packages/db/migrations/0006_amendment_restated.sql', + 'packages/db/migrations/0007_amendment_value_suspect.sql', + 'packages/db/migrations/0008_amendment_provenance.sql', + // #279/ADR-0033: refresh-slice.sql reads interest_link_evidence. + 'packages/db/migrations/0009_interest_link_evidence.sql', +].map((p) => resolve(root, p)); +const workStagingSchema = resolve(root, 'scripts/work-staging-schema.sql'); +const refreshSlice = resolve(root, 'scripts/refresh-slice.sql'); + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function sqlite(dbPath: string, sql: string): void { + execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8', stdio: 'pipe' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +// Fresh transient staging for a new window, exactly as import.mjs / the Worker do between refreshes. +function resetStaging(dbPath: string): void { + const rows = sqliteJson<{ name: string }>( + dbPath, + "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'raw_%' ORDER BY name DESC", + ); + for (const row of rows) sqlite(dbPath, `DROP TABLE IF EXISTS "${row.name}";`); + readScript(dbPath, workStagingSchema); +} + +// One EOP procedure header (a tender) so contract promotion has an authority + tender to attach to. +function seedTender(dbPath: string, unp: string, authorityEik: string): void { + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject, + cpv_code, cpv_description, contract_kind, estimated_value, currency, authority_name, + authority_eik, authority_type, published_at) + VALUES + ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', '${unp}', 'T-${unp}', 'open', + 'Subject ${unp}', '45000000', 'Construction', 'works', 5000, 'BGN', 'Authority ${unp}', + '${authorityEik}', 'public', '2026-06-01');`, + ); +} + +// An EOP base contract. `cnum` is the contract's filing number; `signing` its signing value. +function seedContract( + dbPath: string, + opts: { unp: string; cnum: string; signing: number; authorityEik: string; contractorEik: string }, +): void { + const { unp, cnum, signing, authorityEik, contractorEik } = opts; + sqlite( + dbPath, + `INSERT INTO raw_contracts + (source, dataset_year, dataset_variant, fetched_at, needs_enrichment, document_number, + published_at, unp, tender_ext_id, procedure_type, procurement_subject, cpv_code, + cpv_description, contract_kind, estimated_value, procurement_currency, authority_name, + authority_eik, authority_type, contract_number, contract_date, signing_value, currency, + contract_subject, awarded_to_group, contractor_eik, contractor_name) + VALUES + ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-${cnum}', + '2026-06-01', '${unp}', 'T-${unp}', 'open', 'Subject ${unp}', '45000000', 'Construction', + 'works', 5000, 'BGN', 'Authority ${unp}', '${authorityEik}', 'public', '${cnum}', + '2026-06-02', ${signing}, 'BGN', 'Contract ${cnum}', 0, '${contractorEik}', 'Bidder ${unp}');`, + ); +} + +// A namespace-mismatched EOP annex: `annexCnum` is the annex-side internal number (matches no contract by +// number); it must link to a contract by the value_before anchor instead. +function seedAnnex( + dbPath: string, + opts: { + unp: string; + annexCnum: string; + valueBefore: number; + valueAfter: number; + authorityEik: string; + contractorEik: string; + }, +): void { + const { unp, annexCnum, valueBefore, valueAfter, authorityEik, contractorEik } = opts; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, contract_number, + contract_date, published_at, unp, authority_eik, authority_name, procurement_subject, + contract_kind, value_before, value_after, value_delta, currency, contractor_eik, description) + VALUES + ('eop:annexes:2026-06-08', 2026, 'eop', '2026-06-08T00:00:00Z', '1', 'AMD-${annexCnum}', + '${annexCnum}', '2026-06-02', '2026-06-09', '${unp}', '${authorityEik}', 'Authority ${unp}', + 'Subject ${unp}', 'works', ${valueBefore}, ${valueAfter}, ${valueAfter - valueBefore}, 'BGN', + '${contractorEik}', 'Namespace-mismatched annex');`, + ); +} + +let dir: string; +let db: string; + +beforeEach(() => { + dir = mkdtempSync(resolve(tmpdir(), 'amendments-slice-resolve-')); + db = resolve(dir, 'work.sqlite'); + for (const m of migrations) readScript(db, m); + readScript(db, workStagingSchema); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +interface AmendmentRow { + contract_number: string; + contract_number_raw: string | null; + link_method: string | null; +} +interface ContractRow { + contract_number: string; + annex_count: number; + current_value: number | null; +} + +describe('refresh-slice #306 value-anchor resolver', () => { + it('links a window annex to a prior-window served contract by the corpus value anchor', () => { + // Window 1: the base contract, filing number Д-226, is served on its own. + seedTender(db, 'UNP-NS', '123456786'); + seedContract(db, { + unp: 'UNP-NS', + cnum: 'Д-226', + signing: 1000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // Window 2: only the annex arrives, carrying the internal number 148846 — matches no contract by number, + // but its value_before (1000) is the served contract's signing_value. + resetStaging(db); + seedAnnex(db, { + unp: 'UNP-NS', + annexCnum: '148846', + valueBefore: 1000, + valueAfter: 1200, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + const amendments = sqliteJson( + db, + "SELECT contract_number, contract_number_raw, link_method FROM amendments WHERE unp='UNP-NS'", + ); + // Rewritten onto the target contract's filing number, with the annex-side number kept as provenance. + expect(amendments).toEqual([ + { contract_number: 'Д-226', contract_number_raw: '148846', link_method: 'value_anchor' }, + ]); + + // The prior-window target was touched and re-rolled: the annex now counts and drives current_value. + const contract = sqliteJson( + db, + "SELECT contract_number, annex_count, current_value FROM contracts WHERE tender_id='t:UNP-NS'", + ); + expect(contract).toEqual([{ contract_number: 'Д-226', annex_count: 1, current_value: 1200 }]); + }); + + it('leaves a corpus-ambiguous annex unlinked even when it is unique within the window', () => { + // Window 1: two served contracts on the SAME procedure, both signing_value 1000 — the annex value cannot + // pick between them across the corpus. + seedTender(db, 'UNP-AMB', '123456786'); + seedContract(db, { + unp: 'UNP-AMB', + cnum: 'Д-1', + signing: 1000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + seedContract(db, { + unp: 'UNP-AMB', + cnum: 'Д-2', + signing: 1000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // Window 2: a THIRD matching contract Д-3 (1000) plus the mismatched annex. A windowed-only resolver would + // see just Д-3 and mislink (n_match = 1); the corpus-aware resolver sees Д-1/Д-2/Д-3 and refuses. + resetStaging(db); + seedContract(db, { + unp: 'UNP-AMB', + cnum: 'Д-3', + signing: 1000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + seedAnnex(db, { + unp: 'UNP-AMB', + annexCnum: '148846', + valueBefore: 1000, + valueAfter: 1200, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // Unlinked: the annex keeps its raw annex-side number and no link_method. + const amendments = sqliteJson( + db, + "SELECT contract_number, contract_number_raw, link_method FROM amendments WHERE unp='UNP-AMB'", + ); + expect(amendments).toEqual([ + { contract_number: '148846', contract_number_raw: null, link_method: null }, + ]); + + // None of the three candidate contracts absorbed the annex. + const amended = sqliteJson<{ n: number }>( + db, + "SELECT COUNT(*) AS n FROM contracts WHERE tender_id='t:UNP-AMB' AND annex_count > 0", + ); + expect(amended).toEqual([{ n: 0 }]); + }); + + it('keeps an annex on its zero-value contract it matches BY NUMBER (never value-links to a neighbour)', () => { + // The full path excludes an annex whose (unp, contract_number) is a real contract, regardless of that + // contract's value. The slice path must agree: Д-1 has signing_value 0, Д-2 has 5000; the annex is numbered + // Д-1 with value_before 5000. It matches Д-1 by number, so it must stay on Д-1 — NOT get value-linked to Д-2 + // just because Д-1 fails the signing_value > 0 candidate filter (review todorkolev: the paths must agree). + seedTender(db, 'UNP-ZERO', '123456786'); + seedContract(db, { + unp: 'UNP-ZERO', + cnum: 'Д-2', + signing: 5000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // Window 2: the zero-value contract Д-1 arrives with the annex that carries its number. + resetStaging(db); + seedContract(db, { + unp: 'UNP-ZERO', + cnum: 'Д-1', + signing: 0, + authorityEik: '123456786', + contractorEik: '987654308', + }); + seedAnnex(db, { + unp: 'UNP-ZERO', + annexCnum: 'Д-1', + valueBefore: 5000, + valueAfter: 9000, + authorityEik: '123456786', + contractorEik: '987654308', + }); + readScript(db, refreshSlice); + + // The annex stays on Д-1 by number — not rewritten to the value-neighbour Д-2. + const amendments = sqliteJson( + db, + "SELECT contract_number, contract_number_raw, link_method FROM amendments WHERE unp='UNP-ZERO'", + ); + expect(amendments).toEqual([ + { contract_number: 'Д-1', contract_number_raw: null, link_method: null }, + ]); + + // Д-1 absorbs the annex; the value-neighbour Д-2 is untouched. + const perContract = sqliteJson<{ contract_number: string; annex_count: number }>( + db, + "SELECT contract_number, annex_count FROM contracts WHERE tender_id='t:UNP-ZERO' ORDER BY contract_number", + ); + expect(perContract).toEqual([ + { contract_number: 'Д-1', annex_count: 1 }, + { contract_number: 'Д-2', annex_count: 0 }, + ]); + }); +}); diff --git a/packages/db/src/amendments-sql.test.ts b/packages/db/src/amendments-sql.test.ts index 2d161410..9e54acdd 100644 --- a/packages/db/src/amendments-sql.test.ts +++ b/packages/db/src/amendments-sql.test.ts @@ -16,6 +16,14 @@ import { AMENDMENTS_SQL } from './queries/details'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); +// #305 Tier-2: AMENDMENTS_SQL selects am.value_restated, added to served amendments by this migration. +const migration6 = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +// #305 residual: AMENDMENTS_SQL also selects am.value_suspect, added by this migration. +const migration7 = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8 = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +// #279/ADR-0033: refresh-slice.sql + normalize-raw.sql read interest_link_evidence, so 0009 must be applied too. +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); @@ -65,6 +73,10 @@ function withDb(fn: (dbPath: string) => T): T { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, migration0); + readScript(dbPath, migration6); + readScript(dbPath, migration7); + readScript(dbPath, migration8); + readScript(dbPath, migration9); return fn(dbPath); } finally { rmSync(dir, { recursive: true, force: true }); diff --git a/packages/db/src/amendments-total-restated.test.ts b/packages/db/src/amendments-total-restated.test.ts new file mode 100644 index 00000000..1212ea33 --- /dev/null +++ b/packages/db/src/amendments-total-restated.test.ts @@ -0,0 +1,307 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +// #305 Tier-2 text-based value correction. Some ЦАИС ЕОП annexes put the announced NEW TOTAL into the +// change field, so the feed's value_after is doubled. The основание-text heuristic (computed in TS +// ingest, packages/ingest/src/amendment-total.ts) classifies each annex and — because the correction is +// computed BEFORE the SQL runs — the raw row lands with value_treatment + value_after_restated already +// set. This suite simulates that ingest output and drives the REAL derive → normalize/refresh-slice → +// promote → precompute scripts in pipeline order on a real SQLite DB, asserting: +// (a) a total_restated annex drives current_value + served value_after with the corrected total and is +// NOT annex_total_suspect; +// (b) a genuine_increment annex is not flagged and keeps its (larger, correct) value_after; +// (c) an untreated doubled annex still gets the Tier-1 annex_total_suspect flag; +// (d) full-vs-slice parity for the restated contract. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +// #279/ADR-0033: refresh-slice.sql + normalize-raw.sql read interest_link_evidence, so 0009 must be applied too. +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const derivePath = resolve(root, 'scripts/derive-amendments.sql'); +const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); +const promotePath = resolve(root, 'scripts/promote-amendments.sql'); +const precomputePath = resolve(root, 'scripts/precompute.sql'); +const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); + +const etlRuns = [ + ['normalize-raw', [derivePath, normalizePath, promotePath, precomputePath]], + ['refresh-slice', [derivePath, refreshSlicePath, precomputePath]], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-totalrestated-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); + readScript(dbPath, migration9Path); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +const AUTH_EIK = '000695114'; +const BIDDER_EIK = '831646048'; + +interface AmendmentStep { + before: number | null; + after: number; + publishedAt: string; + treatment: string | null; + restatedAfter: number | null; +} + +interface Case { + unp: string; + signing: number; + steps: AmendmentStep[]; +} + +function seedContracts(dbPath: string, cases: Case[]): void { + const tenders = cases + .map( + (c) => + `('eop:tenders:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'public', ${c.signing}, 'BGN')`, + ) + .join(',\n'); + const contracts = cases + .map( + (c) => + `('eop:contracts:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'C-${c.unp}', '2026-06-01', ${c.signing}, 'BGN', '${BIDDER_EIK}', 'Тестов изпълнител')`, + ) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type, estimated_value, currency) + VALUES ${tenders}; + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, contractor_eik, contractor_name) + VALUES ${contracts};`, + ); +} + +// Seed raw_amendments with value_treatment + value_after_restated already populated — simulating the TS +// ingest output (base.ts runs the amendment-total.ts heuristic before staging). +function seedAmendments(dbPath: string, cases: Case[]): void { + const rows = cases + .flatMap((c) => + c.steps.map( + (s, i) => + `('eop:annexes:${c.unp}', '2026-06-01T00:00:00Z', 'A-${c.unp}-${i + 1}', '${c.unp}', 'C-${c.unp}', '${s.publishedAt}', ${s.before ?? 'NULL'}, ${s.after}, 'BGN', ${s.treatment === null ? 'NULL' : `'${s.treatment}'`}, ${s.restatedAfter ?? 'NULL'})`, + ), + ) + .join(',\n'); + if (!rows) return; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, fetched_at, document_number, unp, contract_number, published_at, + value_before, value_after, currency, value_treatment, value_after_restated) + VALUES ${rows};`, + ); +} + +interface ContractRow { + id: string; + value_flag: string; + current_value: number | null; +} + +const contractsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT id, value_flag, current_value FROM contracts`, + ); + const out = new Map(); + for (const r of rows) out.set(r.id.split(':')[2]!, r); + return out; +}; + +interface AmendmentRow { + unp: string; + value_after: number | null; + value_delta: number | null; + value_restated: number | null; +} + +const amendmentsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT unp, value_after, value_delta, value_restated FROM amendments`, + ); + const out = new Map(); + for (const r of rows) out.set(r.unp, r); + return out; +}; + +// (a) total_restated: doubled value_after (981240) but the основание text announced the true total +// (539240). Ingest set value_after_restated=539240, value_treatment='total_restated'. +const RESTATED: Case = { + unp: 'UNP-RESTATED', + signing: 442_000, + steps: [ + { + before: 442_000, + after: 981_240, + publishedAt: '2026-06-10', + treatment: 'total_restated', + restatedAfter: 539_240, + }, + ], +}; + +// (b) genuine_increment: value_after (60226.85) is a real ≥2× increase already applied; the text confirmed +// it, so ingest set value_treatment='genuine_increment' with restatedAfter NULL. Must NOT be flagged. +const GENUINE: Case = { + unp: 'UNP-GENUINE', + signing: 10_226.85, + steps: [ + { + before: 10_226.85, + after: 60_226.85, + publishedAt: '2026-06-10', + treatment: 'genuine_increment', + restatedAfter: null, + }, + ], +}; + +// (c) untreated doubled annex: no text signal, before ≈ signing, same currency → Tier-1 unchanged. +const UNTREATED: Case = { + unp: 'UNP-DOUBLE', + signing: 77_000_000, + steps: [ + { + before: 77_000_000, + after: 154_000_000, + publishedAt: '2026-06-10', + treatment: null, + restatedAfter: null, + }, + ], +}; + +describe('#305 Tier-2 text-based amendment value correction', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: a total_restated annex drives the corrected total and is not flagged`, () => { + withEtlDb(label, (dbPath) => { + seedContracts(dbPath, [RESTATED]); + seedAmendments(dbPath, [RESTATED]); + for (const p of scriptPaths) readScript(dbPath, p); + + const contract = contractsByUnp(dbPath).get('UNP-RESTATED'); + expect(contract?.value_flag, 'restated annex is not arithmetic-flagged').toBe('ok'); + expect( + contract?.current_value, + 'current_value is the corrected total, NOT the doubled value', + ).toBe(539_240); + + const amendment = amendmentsByUnp(dbPath).get('UNP-RESTATED'); + expect(amendment?.value_after, 'served value_after is the corrected total').toBe(539_240); + expect(amendment?.value_delta, 'served delta is self-consistent (after − before)').toBe( + 539_240 - 442_000, + ); + expect(amendment?.value_restated, 'served row is marked restated').toBe(1); + }); + }); + + it(`${label}: a genuine_increment annex is not flagged and keeps its value_after`, () => { + withEtlDb(label, (dbPath) => { + seedContracts(dbPath, [GENUINE]); + seedAmendments(dbPath, [GENUINE]); + for (const p of scriptPaths) readScript(dbPath, p); + + const contract = contractsByUnp(dbPath).get('UNP-GENUINE'); + expect(contract?.value_flag, 'confirmed-genuine increment is not flagged').not.toBe( + 'annex_total_suspect', + ); + expect(contract?.current_value, 'current_value keeps the genuine increase').toBe(60_226.85); + + const amendment = amendmentsByUnp(dbPath).get('UNP-GENUINE'); + expect(amendment?.value_after, 'served value_after unchanged').toBe(60_226.85); + expect(amendment?.value_restated, 'genuine increment is not marked restated').toBe(0); + }); + }); + + it(`${label}: an untreated doubled annex still gets the Tier-1 annex_total_suspect flag`, () => { + withEtlDb(label, (dbPath) => { + seedContracts(dbPath, [UNTREATED]); + seedAmendments(dbPath, [UNTREATED]); + for (const p of scriptPaths) readScript(dbPath, p); + + const contract = contractsByUnp(dbPath).get('UNP-DOUBLE'); + expect(contract?.value_flag, 'untreated double is still flagged').toBe( + 'annex_total_suspect', + ); + + const amendment = amendmentsByUnp(dbPath).get('UNP-DOUBLE'); + expect(amendment?.value_restated, 'untreated double is not marked restated').toBe(0); + }); + }); + } + + it('full-vs-slice parity: the total_restated contract resolves identically on both paths', () => { + let full: ContractRow | undefined; + withEtlDb('parity-full', (dbPath) => { + seedContracts(dbPath, [RESTATED]); + seedAmendments(dbPath, [RESTATED]); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + full = contractsByUnp(dbPath).get('UNP-RESTATED'); + }); + + let slice: ContractRow | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedContracts(dbPath, [RESTATED]); + seedAmendments(dbPath, [RESTATED]); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + slice = contractsByUnp(dbPath).get('UNP-RESTATED'); + }); + + // Pin both paths to the concrete expected values — a cross-equality (full === slice) would also + // pass on dual-undefined, so assert against the literal on each path instead. + expect(full?.value_flag).toBe('ok'); + expect(slice?.value_flag).toBe('ok'); + expect(full?.current_value).toBe(539_240); + expect(slice?.current_value).toBe(539_240); + }); +}); diff --git a/packages/db/src/amendments-total-suspect.test.ts b/packages/db/src/amendments-total-suspect.test.ts new file mode 100644 index 00000000..9b9b5f34 --- /dev/null +++ b/packages/db/src/amendments-total-suspect.test.ts @@ -0,0 +1,714 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +// #305 single-annex value double-count: a driving annex whose value_after is >=2x its value_before is +// a data defect — ЗОП чл.116 caps a single amendment at +50%, so one step cannot legally more than +// double a contract. Such contracts get value_flag = 'annex_total_suspect' and fall back to +// signing_value, exactly like annex_suspect, so the doubled figure is excluded from every EUR +// aggregate. The ABS(value_after - current_value) tie binds the flag to the annex that DRIVES +// current_value: a doubled annex later superseded by a correct one is NOT flagged. +// +// The flag CASE and its value-fallback siblings live in copies across both derive paths (normalize-raw +// and refresh-slice) plus refresh-slice's reconciliation re-flag, so the rule is exercised through the +// REAL scripts in pipeline order on a real SQLite database. A copy left behind fails here. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +// #305 residual: served amendments gained value_suspect (promote + refresh-slice write it). +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +// #279/ADR-0033: refresh-slice.sql + normalize-raw.sql read interest_link_evidence, so 0009 must be applied too. +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const derivePath = resolve(root, 'scripts/derive-amendments.sql'); +const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); +const promotePath = resolve(root, 'scripts/promote-amendments.sql'); +const precomputePath = resolve(root, 'scripts/precompute.sql'); +const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); + +// Real pipeline order (scripts/import.mjs): full derive runs derive-amendments → normalize-raw → +// promote-amendments → precompute; the slice derive runs derive-amendments → refresh-slice (which +// promotes the window's amendments itself) → precompute. precompute populates current_value_eur, which +// these assertions read. +const etlRuns = [ + ['normalize-raw', [derivePath, normalizePath, promotePath, precomputePath]], + ['refresh-slice', [derivePath, refreshSlicePath, precomputePath]], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-totalsuspect-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); + readScript(dbPath, migration9Path); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +// Signing value in BGN; exactly 100_000 EUR at the fixed peg (÷1.95583) so the fallback reads directly. +const SIGNING_BGN = 195_583; +const AUTH_EIK = '000695114'; +const BIDDER_EIK = '831646048'; + +interface AmendmentStep { + before: number | null; + after: number; + publishedAt: string; +} + +interface Case { + unp: string; + signing: number; + steps: AmendmentStep[]; +} + +function seedContracts(dbPath: string, cases: Case[]): void { + const tenders = cases + .map( + (c) => + `('eop:tenders:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'public', ${c.signing}, 'BGN')`, + ) + .join(',\n'); + const contracts = cases + .map( + (c) => + `('eop:contracts:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'C-${c.unp}', '2026-06-01', ${c.signing}, 'BGN', '${BIDDER_EIK}', 'Тестов изпълнител')`, + ) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type, estimated_value, currency) + VALUES ${tenders}; + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, contractor_eik, contractor_name) + VALUES ${contracts};`, + ); +} + +function seedAmendments(dbPath: string, cases: Case[]): void { + const rows = cases + .flatMap((c) => + c.steps.map( + (s, i) => + `('eop:annexes:${c.unp}', '2026-06-01T00:00:00Z', 'A-${c.unp}-${i + 1}', '${c.unp}', 'C-${c.unp}', '${s.publishedAt}', ${s.before ?? 'NULL'}, ${s.after}, 'BGN')`, + ), + ) + .join(',\n'); + if (!rows) return; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, fetched_at, document_number, unp, contract_number, published_at, + value_before, value_after, currency) + VALUES ${rows};`, + ); +} + +interface Row { + id: string; + value_flag: string; + amount_eur: number | null; + current_value_eur: number | null; +} + +const rowsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT id, value_flag, ROUND(amount_eur) AS amount_eur, + ROUND(current_value_eur) AS current_value_eur + FROM contracts`, + ); + const out = new Map(); + for (const r of rows) out.set(r.id.split(':')[2]!, r); + return out; +}; + +// #305 residual: seed raw_amendments including value_treatment + value_after_restated (the TS ingest +// output) so the served value_suspect / value_restated marks can be asserted end-to-end. +interface TreatedStep { + before: number | null; + after: number; + publishedAt: string; + treatment: string | null; + restatedAfter: number | null; +} +interface TreatedCase { + unp: string; + signing: number; + steps: TreatedStep[]; +} + +function seedTreatedContracts(dbPath: string, cases: TreatedCase[]): void { + seedContracts( + dbPath, + cases.map((c) => ({ unp: c.unp, signing: c.signing, steps: [] })), + ); +} + +function seedTreatedAmendments(dbPath: string, cases: TreatedCase[]): void { + const rows = cases + .flatMap((c) => + c.steps.map( + (s, i) => + `('eop:annexes:${c.unp}', '2026-06-01T00:00:00Z', 'A-${c.unp}-${i + 1}', '${c.unp}', 'C-${c.unp}', '${s.publishedAt}', ${s.before ?? 'NULL'}, ${s.after}, 'BGN', ${s.treatment === null ? 'NULL' : `'${s.treatment}'`}, ${s.restatedAfter ?? 'NULL'})`, + ), + ) + .join(',\n'); + if (!rows) return; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, fetched_at, document_number, unp, contract_number, published_at, + value_before, value_after, currency, value_treatment, value_after_restated) + VALUES ${rows};`, + ); +} + +interface ServedAmendment { + unp: string; + value_after: number | null; + value_suspect: number | null; + value_restated: number | null; +} +const servedByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT unp, value_after, value_suspect, value_restated FROM amendments`, + ); + const out = new Map(); + for (const r of rows) out.set(r.unp, r); + return out; +}; + +// (a) flag-only double: no основание signal (value_treatment NULL), before ≈ signing, value_after = 3× +// before. We can't bridge the true total → mark value_suspect = 1, keep value_restated = 0, and the +// served value_after stays the untrusted figure (the UI blanks it; the number is never rewritten). +const FLAG_ONLY: TreatedCase = { + unp: 'UNP-FLAGONLY', + signing: 100, + steps: [ + { before: 100, after: 300, publishedAt: '2026-06-10', treatment: null, restatedAfter: null }, + ], +}; +// (b) total_restated: text-treated → value_restated = 1, value_suspect = 0. +const RESTATED: TreatedCase = { + unp: 'UNP-RESTATED', + signing: 442_000, + steps: [ + { + before: 442_000, + after: 981_240, + publishedAt: '2026-06-10', + treatment: 'total_restated', + restatedAfter: 539_240, + }, + ], +}; +// (c) genuine_increment: text-confirmed real increase → both marks 0. +const GENUINE: TreatedCase = { + unp: 'UNP-GENUINE', + signing: 10_226.85, + steps: [ + { + before: 10_226.85, + after: 60_226.85, + publishedAt: '2026-06-10', + treatment: 'genuine_increment', + restatedAfter: null, + }, + ], +}; + +describe('#305 residual: per-amendment value_suspect marker on the served row', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: a flag-only double is marked value_suspect=1, value_restated=0, value_after untouched`, () => { + withEtlDb(label, (dbPath) => { + seedTreatedContracts(dbPath, [FLAG_ONLY, RESTATED, GENUINE]); + seedTreatedAmendments(dbPath, [FLAG_ONLY, RESTATED, GENUINE]); + for (const p of scriptPaths) readScript(dbPath, p); + + const served = servedByUnp(dbPath); + + const flagOnly = served.get('UNP-FLAGONLY'); + expect(flagOnly?.value_suspect, 'flag-only double is marked suspect').toBe(1); + expect(flagOnly?.value_restated, 'flag-only double is NOT restated').toBe(0); + expect(flagOnly?.value_after, 'served value_after is never rewritten').toBe(300); + + const restated = served.get('UNP-RESTATED'); + expect(restated?.value_restated, 'total_restated row is marked restated').toBe(1); + expect(restated?.value_suspect, 'a restated row is never also suspect').toBe(0); + + const genuine = served.get('UNP-GENUINE'); + expect(genuine?.value_suspect, 'genuine increment is not suspect').toBe(0); + expect(genuine?.value_restated, 'genuine increment is not restated').toBe(0); + }); + }); + } + + it('full-vs-slice parity: the flag-only double gets value_suspect=1 on both paths', () => { + let full: ServedAmendment | undefined; + withEtlDb('parity-full', (dbPath) => { + seedTreatedContracts(dbPath, [FLAG_ONLY]); + seedTreatedAmendments(dbPath, [FLAG_ONLY]); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + full = servedByUnp(dbPath).get('UNP-FLAGONLY'); + }); + + let slice: ServedAmendment | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedTreatedContracts(dbPath, [FLAG_ONLY]); + seedTreatedAmendments(dbPath, [FLAG_ONLY]); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + slice = servedByUnp(dbPath).get('UNP-FLAGONLY'); + }); + + // Pin both paths to the concrete expected value — a cross-equality (full === slice) would also pass + // on dual-undefined, so assert against the literal on each path instead. + expect(full?.value_suspect).toBe(1); + expect(slice?.value_suspect).toBe(1); + }); +}); + +describe('#305 annex_total_suspect single-annex value double-count', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags a doubled driving annex and falls back to the signing value`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // (a) The bug: 77M signed, one annex reports value_after = 2x value_before (154M). ЗОП caps a + // single amendment at +50%, so this is a double-count defect: flag it and drop back to signing. + { + unp: 'UNP-DOUBLE', + signing: 77_000_000, + steps: [{ before: 77_000_000, after: 154_000_000, publishedAt: '2026-06-10' }], + }, + // (b) A genuine small increase (+30%): before 100 → after 130. Must stay 'ok' and keep its + // current value, guarding against false positives. + { + unp: 'UNP-SMALL', + signing: 100, + steps: [{ before: 100, after: 130, publishedAt: '2026-06-10' }], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const rows = rowsByUnp(dbPath); + + // (a) flagged, value base falls back to signing (77M BGN ÷1.95583), NOT the doubled 154M. + const doubled = rows.get('UNP-DOUBLE'); + expect(doubled?.value_flag, 'doubled annex flagged').toBe('annex_total_suspect'); + const signingEur = Math.round(77_000_000 / 1.95583); + expect(doubled?.amount_eur, 'amount_eur falls back to signing').toBe(signingEur); + // Excluded from the current_value aggregate entirely. + expect(doubled?.current_value_eur, 'current_value_eur suppressed').toBeNull(); + + // (b) genuine +30% increase untouched: stays ok, value reflects current_value (130 BGN). + const small = rows.get('UNP-SMALL'); + expect(small?.value_flag, 'small increase stays ok').toBe('ok'); + expect(small?.amount_eur, 'small increase keeps current value').toBe( + Math.round(130 / 1.95583), + ); + expect(small?.current_value_eur).toBe(Math.round(130 / 1.95583)); + }); + }); + + it(`${label}: an old doubled annex superseded by a correct later annex is NOT flagged`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // (d) An early annex doubled (before 100 → after 200), but a LATER annex sets a correct, + // sub-2x current_value (before 200 → after 130). The doubled step no longer DRIVES + // current_value, so the ABS(...-current_value) tie must leave the contract 'ok'. + { + unp: 'UNP-SUPERSEDED', + signing: 100, + steps: [ + { before: 100, after: 200, publishedAt: '2026-06-10' }, + { before: 200, after: 130, publishedAt: '2026-06-20' }, + ], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const row = rowsByUnp(dbPath).get('UNP-SUPERSEDED'); + expect(row?.value_flag, 'superseded double is not flagged').toBe('ok'); + // Served at the corrected current value (130 BGN), not the transient doubled 200. + expect(row?.amount_eur).toBe(Math.round(130 / 1.95583)); + expect(row?.current_value_eur).toBe(Math.round(130 / 1.95583)); + }); + }); + } + + it('full-vs-slice parity: the doubled contract gets the same value_flag on both paths', () => { + const cases: Case[] = [ + { + unp: 'UNP-DOUBLE', + signing: 77_000_000, + steps: [{ before: 77_000_000, after: 154_000_000, publishedAt: '2026-06-10' }], + }, + ]; + + let fullFlag: string | undefined; + withEtlDb('parity-full', (dbPath) => { + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + fullFlag = rowsByUnp(dbPath).get('UNP-DOUBLE')?.value_flag; + }); + + let sliceFlag: string | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + sliceFlag = rowsByUnp(dbPath).get('UNP-DOUBLE')?.value_flag; + }); + + expect(fullFlag).toBe('annex_total_suspect'); + expect(sliceFlag).toBe('annex_total_suspect'); + expect(fullFlag).toBe(sliceFlag); + }); +}); + +// #305 multi-annex residual: the doubled step is NOT always the first annex. When a later annex reports +// a new TOTAL added to an already-grown value, its value_before is the prior CUMULATIVE total (a preceding +// annex's value_after), not signing. The relaxed anchor flags these too; the ≥2× single-step gate (ЗОП +// чл.116) keeps slow legitimate climbs — whose later steps never reach 2× — untouched. +describe('#305 annex_total_suspect multi-annex value double-count', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags a later-in-chain double whose value_before ties to a prior annex total, not signing`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // Chain: 1M signed → annex1 +40% (1.4M, legal, not a double) → annex2 doubles the 1.4M total to + // 2.8M. The driving annex's value_before (1.4M) equals the PRIOR annex's value_after, not signing + // (1M), so the old signing-only anchor missed it. Must now flag and fall back to signing. + { + unp: 'UNP-MULTI-DOUBLE', + signing: 1_000_000, + steps: [ + { before: 1_000_000, after: 1_400_000, publishedAt: '2026-06-10' }, + { before: 1_400_000, after: 2_800_000, publishedAt: '2026-06-20' }, + ], + }, + // Control: same shape but the later step is a legal +36% (1.4M → 1.9M), below 2×. The relaxed + // anchor matches value_before to the prior total, but the ≥2× gate must keep this 'ok'. + { + unp: 'UNP-MULTI-OK', + signing: 1_000_000, + steps: [ + { before: 1_000_000, after: 1_400_000, publishedAt: '2026-06-10' }, + { before: 1_400_000, after: 1_900_000, publishedAt: '2026-06-20' }, + ], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const rows = rowsByUnp(dbPath); + + const multiDouble = rows.get('UNP-MULTI-DOUBLE'); + expect(multiDouble?.value_flag, 'later-in-chain double flagged').toBe( + 'annex_total_suspect', + ); + expect(multiDouble?.amount_eur, 'amount_eur falls back to signing').toBe( + Math.round(1_000_000 / 1.95583), + ); + expect(multiDouble?.current_value_eur, 'current_value_eur suppressed').toBeNull(); + + const multiOk = rows.get('UNP-MULTI-OK'); + expect(multiOk?.value_flag, 'legal <2x later step stays ok').toBe('ok'); + expect(multiOk?.amount_eur, 'ok row keeps current value').toBe( + Math.round(1_900_000 / 1.95583), + ); + expect(multiOk?.current_value_eur).toBe(Math.round(1_900_000 / 1.95583)); + }); + }); + } + + // The per-row value_suspect marker (served amendments) must also catch the later-in-chain double, so the + // UI blanks that specific annex row — not just the contract-level flag. + const MULTI_TREATED: TreatedCase = { + unp: 'UNP-MULTI-SUSPECT', + signing: 1_000_000, + steps: [ + { + before: 1_000_000, + after: 1_400_000, + publishedAt: '2026-06-10', + treatment: null, + restatedAfter: null, + }, + { + before: 1_400_000, + after: 2_800_000, + publishedAt: '2026-06-20', + treatment: null, + restatedAfter: null, + }, + ], + }; + + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: marks value_suspect=1 on the later-in-chain doubled annex row, not the legal earlier one`, () => { + withEtlDb(label, (dbPath) => { + seedTreatedContracts(dbPath, [MULTI_TREATED]); + seedTreatedAmendments(dbPath, [MULTI_TREATED]); + for (const p of scriptPaths) readScript(dbPath, p); + + const served = sqliteJson<{ value_after: number; value_suspect: number }>( + dbPath, + `SELECT value_after, value_suspect FROM amendments + WHERE unp = 'UNP-MULTI-SUSPECT' ORDER BY value_after`, + ); + expect(served.length, 'both annex rows served').toBe(2); + // The legal +40% step (1.4M) is not suspect; the doubled step (2.8M) is. + const legal = served.find((r) => r.value_after === 1_400_000); + const doubled = served.find((r) => r.value_after === 2_800_000); + expect(legal?.value_suspect, 'legal earlier step not suspect').toBe(0); + expect(doubled?.value_suspect, 'later-in-chain double marked suspect').toBe(1); + }); + }); + } +}); + +// #305 NEW-HIGH-1 (multi-annex chain contamination): the double-count correction is per-row and does NOT +// propagate down a chain. A restated prior annex (doubled → corrected down) leaves a LATER annex still +// computed by the feed on the contaminated (raw, doubled) base. The later annex's own step ratio is +// legitimate (<2×) so the arithmetic gate misses it and the prior is text-treated (excluded) — yet +// current_value inherited the doubled total. The new branch flags it → signing fallback, on both paths. +describe('#305 NEW-HIGH-1 multi-annex chain contamination', () => { + // annex1 doubled 1M→2.4M, text-restated to 1.4M; annex2 is a real +15% the feed computed on the RAW 2.4M + // base (2.4M→2.76M). annex2's own ratio is 1.15× so the gate misses it; annex1 is treated so it is + // excluded. Without the fix, current_value serves the contaminated 2.76M. + const CONTAM: TreatedCase = { + unp: 'UNP-CHAIN-CONTAM', + signing: 1_000_000, + steps: [ + { + before: 1_000_000, + after: 2_400_000, + publishedAt: '2026-06-10', + treatment: 'total_restated', + restatedAfter: 1_400_000, + }, + { + before: 2_400_000, + after: 2_760_000, + publishedAt: '2026-06-20', + treatment: null, + restatedAfter: null, + }, + ], + }; + // Control: the same shape on an HONEST base — annex1 is a genuine +40% (no restatement), annex2 +15% on + // the clean 1.4M base. No treated prior, so the contamination branch must NOT fire; stays 'ok'. + const CLEAN: TreatedCase = { + unp: 'UNP-CHAIN-CLEAN', + signing: 1_000_000, + steps: [ + { + before: 1_000_000, + after: 1_400_000, + publishedAt: '2026-06-10', + treatment: null, + restatedAfter: null, + }, + { + before: 1_400_000, + after: 1_610_000, + publishedAt: '2026-06-20', + treatment: null, + restatedAfter: null, + }, + ], + }; + + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags a later annex riding a restated prior's doubled base, and keeps a clean chain ok`, () => { + withEtlDb(label, (dbPath) => { + seedTreatedContracts(dbPath, [CONTAM, CLEAN]); + seedTreatedAmendments(dbPath, [CONTAM, CLEAN]); + for (const p of scriptPaths) readScript(dbPath, p); + + const rows = rowsByUnp(dbPath); + + const contam = rows.get('UNP-CHAIN-CONTAM'); + expect(contam?.value_flag, 'contaminated later annex flagged').toBe('annex_total_suspect'); + expect(contam?.amount_eur, 'falls back to signing, not the contaminated 2.76M').toBe( + Math.round(1_000_000 / 1.95583), + ); + expect(contam?.current_value_eur, 'contaminated current_value suppressed').toBeNull(); + + const clean = rows.get('UNP-CHAIN-CLEAN'); + expect(clean?.value_flag, 'honest two-step growth stays ok').toBe('ok'); + expect(clean?.current_value_eur).toBe(Math.round(1_610_000 / 1.95583)); + }); + }); + } + + it('full-vs-slice parity: the contaminated chain gets the same flag on both paths', () => { + let fullFlag: string | undefined; + withEtlDb('parity-full', (dbPath) => { + seedTreatedContracts(dbPath, [CONTAM]); + seedTreatedAmendments(dbPath, [CONTAM]); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + fullFlag = rowsByUnp(dbPath).get('UNP-CHAIN-CONTAM')?.value_flag; + }); + + let sliceFlag: string | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedTreatedContracts(dbPath, [CONTAM]); + seedTreatedAmendments(dbPath, [CONTAM]); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + sliceFlag = rowsByUnp(dbPath).get('UNP-CHAIN-CONTAM')?.value_flag; + }); + + expect(fullFlag).toBe('annex_total_suspect'); + expect(sliceFlag).toBe('annex_total_suspect'); + }); +}); + +// #305 84818-class: an EXACT single-step 2× (value_after ≈ 2× value_before) is the ЗОП чл.116 defect +// signature even when value_before anchors to NEITHER signing NOR a prior annex total (an orphan base) — +// as in real contract 84818, whose annex reports 76.77M → 153.54M on a base unrelated to the contract's +// signing. The gate flags it → signing fallback (EXCLUDE), without ever rewriting the value. +describe('#305 84818-class orphan exact-double', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags an exact 2× on an orphan base, but leaves a non-exact orphan jump ok`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // Orphan exact 2×: value_before 90 000 ties neither signing (195 583) nor any prior annex, and + // value_after is exactly 2×. Flag → signing fallback (100 000 EUR), never the doubled 180 000. + { + unp: 'UNP-ORPHAN-EXACT', + signing: 195_583, + steps: [{ before: 90_000, after: 180_000, publishedAt: '2026-06-10' }], + }, + // Control: an orphan jump that is NOT an exact 2× (1.5×) is ambiguous — with no anchor and no + // exact-double signature it must stay ok (the relaxed rule is scoped to EXACT 2× only). + { + unp: 'UNP-ORPHAN-SMALL', + signing: 195_583, + steps: [{ before: 90_000, after: 135_000, publishedAt: '2026-06-10' }], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const rows = rowsByUnp(dbPath); + + const orphan = rows.get('UNP-ORPHAN-EXACT'); + expect(orphan?.value_flag, 'orphan exact 2× flagged').toBe('annex_total_suspect'); + expect(orphan?.amount_eur, 'falls back to signing, not the doubled 180k').toBe( + Math.round(195_583 / 1.95583), + ); + expect(orphan?.current_value_eur, 'doubled current suppressed').toBeNull(); + + const small = rows.get('UNP-ORPHAN-SMALL'); + expect(small?.value_flag, 'non-exact orphan jump stays ok').toBe('ok'); + expect(small?.current_value_eur).toBe(Math.round(135_000 / 1.95583)); + }); + }); + } +}); + +// #305 NEW-HIGH-2 (reconciliation parity): the slice reconciliation reads the CUMULATIVE served +// `amendments`, whose value_after is RESTATED, while the full path anchors on RAW values. A restated prior +// annex used to flip the anchor's "prev not itself a double" test (restated 1.4M < 2×1.2M passes; the raw +// 2.4M would fail), flagging on the slice but not on the full rebuild. The `prev.value_restated = 0` guard +// restores parity: both paths reach the same verdict for a restated-prior + doubled-later chain. +describe('#305 NEW-HIGH-2 slice reconciliation parity', () => { + // annex1 doubled 1.2M→2.4M, restated to 1.4M; annex2 grows the RESTATED 1.4M to 2.9M (a ≥2× step, but + // deliberately NOT an exact 2× so the 84818-class rule doesn't fire and mask the guard under test). On the + // full (raw) path annex2's value_before (1.4M) anchors to neither signing (1M) nor the raw prior total + // (2.4M) → 'ok'. Pre-guard the slice reconciliation matched the restated 1.4M and flagged — the flip. + // Post-guard (prev.value_restated = 0): 'ok', matching the full rebuild. + const FLIP: TreatedCase = { + unp: 'UNP-RECON-FLIP', + signing: 1_000_000, + steps: [ + { + before: 1_200_000, + after: 2_400_000, + publishedAt: '2026-06-10', + treatment: 'total_restated', + restatedAfter: 1_400_000, + }, + { + before: 1_400_000, + after: 2_900_000, + publishedAt: '2026-06-20', + treatment: null, + restatedAfter: null, + }, + ], + }; + + it('full and slice agree on a restated-prior + doubled-later chain (no flag flip)', () => { + let fullFlag: string | undefined; + withEtlDb('parity-full', (dbPath) => { + seedTreatedContracts(dbPath, [FLIP]); + seedTreatedAmendments(dbPath, [FLIP]); + for (const p of [derivePath, normalizePath, promotePath, precomputePath]) + readScript(dbPath, p); + fullFlag = rowsByUnp(dbPath).get('UNP-RECON-FLIP')?.value_flag; + }); + + let sliceFlag: string | undefined; + withEtlDb('parity-slice', (dbPath) => { + seedTreatedContracts(dbPath, [FLIP]); + seedTreatedAmendments(dbPath, [FLIP]); + for (const p of [derivePath, refreshSlicePath, precomputePath]) readScript(dbPath, p); + sliceFlag = rowsByUnp(dbPath).get('UNP-RECON-FLIP')?.value_flag; + }); + + // The canonical full-rebuild verdict, matched by the slice (pre-guard the slice flipped to suspect). + expect(sliceFlag).toBe(fullFlag); + }); +}); diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index efd769fe..e0fda021 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -16,6 +16,26 @@ const migration3 = readFileSync( resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'), 'utf8', ); +// …and 0009, which those same blocks now join for the Trade Register evidence gate (#279, ADR-0033). +const migration9 = readFileSync( + resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'), + 'utf8', +); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6 = readFileSync( + resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'), + 'utf8', +); +// #305 residual: served amendments gained value_suspect (promote + refresh-slice write it). +const migration7 = readFileSync( + resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'), + 'utf8', +); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8 = readFileSync( + resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'), + 'utf8', +); const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); const normalize = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); const precompute = readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8'); @@ -66,7 +86,10 @@ function build(path: 'normalize' | 'refresh'): DatabaseSync { const db = new DatabaseSync(':memory:'); db.exec(schema); db.exec(migration2); - db.exec(migration3); + db.exec(migration3 + migration9); + db.exec(migration6); + db.exec(migration7); + db.exec(migration8); db.exec(staging); db.exec(seed); if (path === 'normalize') { diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index df67b27e..574c8cec 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -11,6 +11,13 @@ const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // refresh-slice.sql's officials block reads interest_links (0003); build it so the script doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); const etlPaths = [ ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], @@ -40,6 +47,10 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { readScript(dbPath, schemaPath); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, stagingPath); run(dbPath); } finally { diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index 52acac25..f6d9c7ff 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -13,6 +13,7 @@ import { fileURLToPath } from 'node:url'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { assertIntegrity, + checkAmendmentTwins, checkCurrentAmountParity, checkDateSanity, checkEikValidity, @@ -28,6 +29,8 @@ const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bid const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // precompute.sql's officials block reads interest_links (0003); build it so precompute doesn't fail. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); const precomputePath = resolve(root, 'scripts/precompute.sql'); function sqlite(dbPath: string, sql: string): void { @@ -71,6 +74,7 @@ function freshDb(): string { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); sqlite(dbPath, CLEAN_FIXTURE); return dbPath; } @@ -117,6 +121,7 @@ describe('reconciliation gate — clean corpus', () => { 'eik-validity', 'date-sanity', 'staging-reconciliation', + 'amendment-twin-dedup', ]) expect(results.find((r) => r.name === nm)?.skipped, `${nm} must not skip`).toBe(false); }); @@ -229,6 +234,100 @@ describe('reconciliation gate — injected violations', () => { expect(result.detail).toMatch(/EMPTY corpus/); }); + // #286/#302: the OCDS→EOP bridge lets an OCDS amendment reach the same contract as its EOP twin, and + // the prefer-EOP dedup (a DELETE two scripts earlier) is the SOLE guard, since promotion into + // `amendments` is unconditional. This gate is that dedup's post-condition. The suite already pins the + // dedup behaviourally; these cases pin the GATE — without them an `ok: n === 0` → `ok: true` edit + // leaves every package green while the gate is inert. + // + // Sources are the real staged partition shapes: `eop:annexes:` spanning 2020-05-08..2026-08-11 on + // the live corpus, `ocds:` only 2026 (the OCDS feed is go-forward). The gate matches by source + // PREFIX, so the fixtures below use both ends of the real EOP range — a pattern pinned to one year + // drops an arm and blows the count. The OCDS side cannot be spread the same way without inventing a + // partition that does not exist, so an `ocds:2026%` narrowing stays out of reach of honest fixtures; + // it is latent-in-2027, not a present regression. + it('amendment-twin-dedup catches an EOP and an OCDS amendment on the same (unp, contract_number)', async () => { + const db = track(freshDb()); + sqlite( + db, + `INSERT INTO amendments (id, natural_key, contract_number, unp, published_at, document_number, source) + VALUES + ('am:UNP-1:C-1:E1','am:UNP-1:C-1:E1','C-1','UNP-1','2026-03-05','E1','eop:annexes:2026-03-05'), + ('am:UNP-1:C-1:ocds-1','am:UNP-1:C-1:ocds-1','C-1','UNP-1','2026-03-05','ocds-e82gsb-1','ocds:2026-03-05');`, + ); + const result = await checkAmendmentTwins(runner(db)); + expect(result.ok).toBe(false); + expect(result.skipped).toBe(false); + // Anchor the COUNT (so `11` cannot satisfy `1`) but keep the prose match to the stable half of the + // message, not the whole sentence. + expect(result.detail).toMatch(/^1\b/); + expect(result.detail).toMatch(/prefer-EOP dedup regressed/); + }); + + it('amendment-twin-dedup stays green on the shapes a working dedup actually leaves behind', async () => { + const db = track(freshDb()); + sqlite( + db, + `INSERT INTO amendments (id, natural_key, contract_number, unp, published_at, document_number, source) + VALUES + -- two EOP annexes on one contract: the normal case, not a twin + ('am:UNP-1:C-1:E1','am:UNP-1:C-1:E1','C-1','UNP-1','2022-03-01','E1','eop:annexes:2022-03-01'), + ('am:UNP-1:C-1:E2','am:UNP-1:C-1:E2','C-1','UNP-1','2022-06-01','E2','eop:annexes:2022-06-01'), + -- a genuinely OCDS-only annex on a DIFFERENT contract: what the dedup deliberately keeps + ('am:UNP-2:C-2:ocds-1','am:UNP-2:C-2:ocds-1','C-2','UNP-2','2026-02-03','ocds-e82gsb-1','ocds:2026-02-03'), + -- an OCDS row the bridge refused (still keyed by its OCID) next to an EOP annex on the same + -- contract number: an honest residual, NOT double counting, because every consumer keys on + -- (unp, contract_number) and this row's unp matches no contract + ('am:ocds-3:C-1:ocds-2','am:ocds-3:C-1:ocds-2','C-1','ocds-e82gsb-3','2026-04-07','ocds-e82gsb-2','ocds:2026-04-07'), + -- NULL contract_number on both sides: cannot roll onto a contract, so it cannot double count + ('am:UNP-3::E3','am:UNP-3::E3',NULL,'UNP-3','2026-05-06','E3','eop:annexes:2026-05-06'), + ('am:UNP-3::ocds-3','am:UNP-3::ocds-3',NULL,'UNP-3','2026-05-06','ocds-e82gsb-4','ocds:2026-05-06'), + -- NULL unp on both sides, same contract number: amendments.unp is nullable and both mappers + -- can leave it empty, but such a row joins no contract either, so it must stay green too + ('am:C-9:E4','am:C-9:E4','C-9',NULL,'2026-07-02','E4','eop:annexes:2026-07-02'), + ('am:C-9:ocds-5','am:C-9:ocds-5','C-9',NULL,'2026-07-02','ocds-e82gsb-5','ocds:2026-07-02');`, + ); + const result = await checkAmendmentTwins(runner(db)); + expect(result.ok).toBe(true); + expect(result.skipped).toBe(false); + expect(result.detail).toMatch(/prefer-EOP dedup intact/); + }); + + // THREE offending pairs that deliberately share a value along each axis — (U1,C1), (U1,C2), (U2,C1) — + // so the grouping key itself is pinned: collapsing it to `unp` alone counts 2, to `contract_number` + // alone counts 2, and only the real composite key counts 3. One pair also carries two EOP rows, so + // counting rows instead of pairs overshoots. The EOP sources sit at both ends of the real partition + // range (2020 and 2026), which is what kills a year-pinned pattern. + it('amendment-twin-dedup counts each offending pair once, keyed on BOTH columns', async () => { + const db = track(freshDb()); + sqlite( + db, + `INSERT INTO amendments (id, natural_key, contract_number, unp, published_at, document_number, source) + VALUES + ('am:UNP-1:C-1:E1','am:UNP-1:C-1:E1','C-1','UNP-1','2020-05-08','E1','eop:annexes:2020-05-08'), + ('am:UNP-1:C-1:E2','am:UNP-1:C-1:E2','C-1','UNP-1','2026-08-11','E2','eop:annexes:2026-08-11'), + ('am:UNP-1:C-1:ocds-1','am:UNP-1:C-1:ocds-1','C-1','UNP-1','2026-01-04','ocds-e82gsb-1','ocds:2026-01-04'), + -- ...and TWO OCDS rows, which the dedup would drop together: a gate that demanded exactly one + -- row per side would walk straight past this pair + ('am:UNP-1:C-1:ocds-4','am:UNP-1:C-1:ocds-4','C-1','UNP-1','2026-06-15','ocds-e82gsb-4','ocds:2026-06-15'), + ('am:UNP-1:C-2:E3','am:UNP-1:C-2:E3','C-2','UNP-1','2023-09-12','E3','eop:annexes:2023-09-12'), + ('am:UNP-1:C-2:ocds-2','am:UNP-1:C-2:ocds-2','C-2','UNP-1','2026-05-20','ocds-e82gsb-2','ocds:2026-05-20'), + ('am:UNP-2:C-1:E4','am:UNP-2:C-1:E4','C-1','UNP-2','2026-08-11','E4','eop:annexes:2026-08-11'), + ('am:UNP-2:C-1:ocds-3','am:UNP-2:C-1:ocds-3','C-1','UNP-2','2026-08-11','ocds-e82gsb-3','ocds:2026-08-11');`, + ); + const result = await checkAmendmentTwins(runner(db)); + expect(result.ok).toBe(false); + expect(result.detail).toMatch(/^3\b/); + }); + + it('amendment-twin-dedup self-skips when the amendments table is absent (staging-only DB)', async () => { + const db = track(freshDb()); + sqlite(db, 'DROP TABLE amendments;'); + const result = await checkAmendmentTwins(runner(db)); + expect(result.skipped).toBe(true); + expect(result.ok).toBe(true); + }); + it('eik-validity catches eik_valid=1 with a non-numeric eik_normalized', async () => { const db = track(freshDb()); sqlite(db, "UPDATE bidders SET eik_normalized = 'AB12' WHERE id = 'eik:131071587';"); diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index e5296aaa..a37050b2 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -11,6 +11,8 @@ const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration1 = resolve(root, 'packages/db/migrations/0001_flow_pairs_bidder_index.sql'); const migration2 = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); const migration3 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const migration10 = resolve(root, 'packages/db/migrations/0010_publishing_gate_constraints.sql'); const backfill = resolve(root, 'scripts/backfill-current-value-currency.sql'); const precompute = resolve(root, 'scripts/precompute.sql'); @@ -131,6 +133,7 @@ describe('served migrations', () => { ); readScript(dbPath, migration2); readScript(dbPath, migration3); + readScript(dbPath, migration9); readScript(dbPath, backfill); readScript(dbPath, precompute); @@ -150,4 +153,350 @@ describe('served migrations', () => { rmSync(dir, { recursive: true, force: true }); } }); + + // 0006 attaches the Trade Register evidence seal to a link (#279, ADR-0033). A SIDE TABLE rather than + // columns on interest_links, for two reasons that are easy to forget: SQLite's ADD COLUMN has no + // IF NOT EXISTS and migrations are applied by a bare `d1 execute --file` with no tracking, so a + // re-apply must be a no-op; and load.mjs rebuilds the CACBG tables from 0003 alone, so any column + // added here would have to be duplicated into 0003 and kept in step forever. + it('0006 adds the evidence seal and re-applying it is a no-op', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-0006-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + for (const m of [migration0, migration1, migration2, migration3, migration9]) + readScript(dbPath, m); + + expect( + sqlite( + dbPath, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='interest_link_evidence';", + ).trim(), + ).toBe('1'); + + // A seal for a real link survives a second apply — the migration must not drop and recreate. + sqlite( + dbPath, + `INSERT INTO persons (id, name) VALUES ('person:a', 'Иван Петров Тестов'); + INSERT INTO bidders (id, name, eik_normalized, eik_valid) + VALUES ('eik:201122335', 'АЛФА СТРОЙ ООД', '201122335', 1); + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:k', 'k', 'person:a', 'eik:201122335', '201122335', 'АЛФА СТРОЙ ООД', + 'test', 'document', 'owns', 'private_ownership', 'published'); + INSERT INTO interest_link_evidence + (link_key, evidence_kind, matched_fact, lookup_date, rules_version, live_status) + VALUES ('k', 'document', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live');`, + ); + readScript(dbPath, migration9); // idempotent re-apply + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_link_evidence;').trim()).toBe('1'); + + // The FK is real: a seal for a link that does not exist is rejected. D1 enforces foreign keys, + // so this is what stops the ship path inserting seals before (or after wiping) their links. + expect(() => + sqlite( + dbPath, + `PRAGMA foreign_keys=ON; + INSERT INTO interest_link_evidence + (link_key, evidence_kind, matched_fact, lookup_date, rules_version, live_status) + VALUES ('nope', 'document', 'eik', '2026-08-05', 'tr-rules-1', 'live');`, + ), + ).toThrow(/FOREIGN KEY/i); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + // #279 §2: the publishing gate reads these columns as enums, but the schema let them hold anything. + // A value that merely LOOKS like a gate value — the trailing space in 'published ' §2 names — passes + // every writer and then silently fails `status = 'published'` (or worse, passes a LIKE somewhere). + // A CHECK is the only place this can be enforced for every writer at once, including a hand-run UPDATE. + it('constrains the publishing-gate enums and the declaration identity key', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + for (const m of [migration0, migration3, migration9, migration10]) readScript(dbPath, m); + + const seedLink = (status: string) => + sqlite( + dbPath, + `INSERT INTO persons(id, name) VALUES ('person:a', 'А') ON CONFLICT DO NOTHING; + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:${status}', '${status}', 'person:a', 'eik:201122335', '201122335', + 'АЛФА СТРОЙ ООД', 'test', 'document', 'owns', 'private_ownership', '${status}');`, + ); + + // §2's named case: 'published ' is not 'published'. It must be rejected, not stored. + expect(() => seedLink('published ')).toThrow(/CHECK/i); + expect(() => seedLink('publushed')).toThrow(/CHECK/i); + // POSITIVE CONTROL — every real status still inserts. A CHECK that rejected everything would pass + // the assertions above while breaking the loader. + for (const s of ['published', 'held', 'withdrawn', 'suppressed', 'internal']) + expect(() => seedLink(s)).not.toThrow(); + + // interest_class gates the surface just as hard: a non-surfaced class is what keeps a management + // role off the public page, so a typo'd class is a leak in the same way. + expect(() => + sqlite( + dbPath, + `INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:c', 'c', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership ', 'held');`, + ), + ).toThrow(/CHECK/i); + + // evidence_kind decides whether a link may be read at all (SURFACED_OWNERSHIP). ADR-0035's + // document_uncorroborated must be a legal value — and a misspelling must not be. + const seal = (kind: string) => + sqlite( + dbPath, + `INSERT INTO interest_link_evidence + (link_key, evidence_kind, lookup_date, rules_version, live_status) + VALUES ('${kind}', '${kind}', '2026-08-12', 'tr-rules-1', 'live');`, + ); + expect(() => seal('document_uncorroberated')).toThrow(/CHECK|FOREIGN KEY/i); + for (const k of ['document', 'confirmed', 'document_uncorroborated', 'refuted']) + expect(() => { + sqlite( + dbPath, + `INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:${k}', '${k}', 'person:a', 'eik:1', '1', 'X', 't', '${k}', 'owns', + 'private_ownership', 'held'); + INSERT INTO interest_link_evidence + (link_key, evidence_kind, lookup_date, rules_version, live_status) + VALUES ('${k}', '${k}', '2026-08-12', 'tr-rules-1', 'live');`, + ); + }).not.toThrow(); + + // §2: `UNIQUE (xml_file, control_hash)` did not constrain re-import. SQLite counts NULLs as + // DISTINCT and control_hash is genuinely optional at the source, so two hashless imports of the + // SAME declaration both inserted and double-counted the stakes it carries. + const decl = (id: string, hash: string | null, folder = '2023') => + sqlite( + dbPath, + `INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('${id}', 'person:a', 'A.xml', ${hash === null ? 'NULL' : `'${hash}'`}, + '${folder}', 'assets', 'https://x/A.xml');`, + ); + expect(() => decl('d:n1', null)).not.toThrow(); + expect(() => decl('d:n2', null)).toThrow(/UNIQUE/i); // the case that used to slip through + expect(() => decl('d:1', 'H1')).not.toThrow(); + expect(() => decl('d:2', 'H1')).toThrow(/UNIQUE/i); + // POSITIVE CONTROLS — the key must not over-constrain. The register reuses basenames across + // FOLDERS, so the same xml_file in a different folder is a DIFFERENT declaration (load.mjs + // namespaces its id for this reason), and a genuinely different hash is a corrected re-filing. + expect(() => decl('d:f2', null, '2024')).not.toThrow(); + expect(() => decl('d:3', 'H2')).not.toThrow(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + // THE POINT OF 0007, stated as an executable claim rather than a comment (todorkolev, #309). + // + // 0003 is an ALREADY-APPLIED migration and is therefore never edited: `CREATE TABLE IF NOT EXISTS` is a + // no-op on a live database, so an in-place CHECK would exist only on freshly built ones. That would put + // two different schemas under one name — and the divergence would land precisely on the served database, + // which is where a hand-run `UPDATE status='published '` during an incident actually happens. + // + // So 0007 owns the enforcement for BOTH shapes, and this test is what keeps them one shape: a fresh + // 0000..0007 build and a legacy database retrofitted by 0007 must reject the SAME values and accept the + // SAME values. If either drifts, the parity assertions below fail rather than a reviewer noticing. + it('a fresh build and a retrofitted legacy database enforce the SAME gate', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-parity-')); + const fresh = resolve(dir, 'fresh.sqlite'); + const legacy = resolve(dir, 'legacy.sqlite'); + try { + // (1) fresh: the migration chain exactly as a new environment applies it. + for (const m of [migration0, migration3, migration9, migration10]) readScript(fresh, m); + + // (2) legacy: 0003 + 0006 as they were BEFORE this PR, then 0007 retrofits. `interest_links` is + // recreated without constraints because that is the shape every already-deployed database holds. + for (const m of [migration0, migration3, migration9]) readScript(legacy, m); + sqlite( + legacy, + `DROP TABLE interest_link_evidence; + DROP TABLE interest_links; + CREATE TABLE interest_links ( + id TEXT PRIMARY KEY, link_key TEXT NOT NULL UNIQUE, + person_id TEXT NOT NULL REFERENCES persons(id), bidder_id TEXT NOT NULL, eik TEXT NOT NULL, + entity_key TEXT NOT NULL, match_method TEXT, matcher_version TEXT NOT NULL, + publish_tier TEXT NOT NULL, relation TEXT NOT NULL, + interest_class TEXT NOT NULL DEFAULT 'management_role', + contemporaneous INTEGER NOT NULL DEFAULT 0, own_institution TEXT NOT NULL DEFAULT 'none', + evidence_count INTEGER NOT NULL DEFAULT 1, first_declared_year TEXT, last_declared_year TEXT, + contract_count INTEGER NOT NULL DEFAULT 0, contract_value_eur REAL, first_contract_year TEXT, + last_contract_year TEXT, status TEXT NOT NULL DEFAULT 'held', verified_by TEXT, + verified_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now'))); + CREATE TABLE interest_link_evidence ( + link_key TEXT PRIMARY KEY REFERENCES interest_links(link_key), evidence_kind TEXT NOT NULL, + registry_role TEXT, matched_fact TEXT, entry_number TEXT, entry_date TEXT, + lookup_date TEXT NOT NULL, rules_version TEXT NOT NULL, live_status TEXT NOT NULL, + sealed_at TEXT NOT NULL DEFAULT (datetime('now')));`, + ); + readScript(legacy, migration10); + + const link = (db: string, id: string, status: string, cls = 'private_ownership') => + sqlite( + db, + `INSERT INTO persons(id, name) VALUES ('person:a', 'А') ON CONFLICT DO NOTHING; + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:${id}', '${id}', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + '${cls}', '${status}');`, + ); + const seal = (db: string, key: string, kind: string) => + sqlite( + db, + `INSERT INTO interest_link_evidence + (link_key, evidence_kind, lookup_date, rules_version, live_status) + VALUES ('${key}', '${kind}', '2026-08-14', 'tr-rules-1', 'live');`, + ); + + for (const [label, db] of [ + ['fresh', fresh], + ['legacy', legacy], + ] as const) { + // REJECTED identically — 'published ' is #279 §2's named case, and it is the one a human types. + expect(() => link(db, `bad1-${label}`, 'published ')).toThrow(/CHECK/i); + expect(() => link(db, `bad2-${label}`, 'publushed')).toThrow(/CHECK/i); + expect(() => link(db, `bad3-${label}`, 'held', 'private_ownership ')).toThrow(/CHECK/i); + // ACCEPTED identically — the gate is a bound, not a blanket, on both shapes. + expect(() => link(db, `ok-${label}`, 'published')).not.toThrow(); + expect(() => seal(db, `ok-${label}`, 'document')).not.toThrow(); + expect(() => seal(db, `ok-${label}`, 'document_uncorroborated')).toThrow(/UNIQUE|CHECK/i); + // …and the evidence enum is enforced on both too. + expect(() => link(db, `k2-${label}`, 'held')).not.toThrow(); + expect(() => seal(db, `k2-${label}`, 'document_uncorroberated')).toThrow(/CHECK/i); + // The UPDATE path is the one an incident actually uses, and no CHECK would ever cover it on a + // legacy table — only the trigger does. + expect(() => + sqlite(db, `UPDATE interest_links SET status='published ' WHERE link_key='ok-${label}';`), + ).toThrow(/CHECK/i); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + // The retrofit has to be safe on a LIVE D1: ship-related-persons wipes rows, never table definitions, + // so a deployed database keeps its unconstrained 0003 shape until this migration rebuilds it. + it('0007 retrofits a pre-existing database, keeps its rows, and re-applies as a no-op', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-migrations-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + for (const m of [migration0, migration3, migration9]) readScript(dbPath, m); + // Simulate a database provisioned BEFORE today: 0003 now declares the constraints, so the legacy + // shape has to be recreated explicitly. This is the state every already-deployed environment is in + // — `CREATE TABLE IF NOT EXISTS` never revisited it and ship-related-persons only wipes rows. + sqlite( + dbPath, + `DROP TABLE declarations; + CREATE TABLE declarations ( + id TEXT PRIMARY KEY, person_id TEXT NOT NULL REFERENCES persons(id), xml_file TEXT NOT NULL, + control_hash TEXT, folder_year TEXT NOT NULL, declared_year TEXT, template TEXT NOT NULL, + category TEXT, institution TEXT, position TEXT, source_url TEXT NOT NULL, + UNIQUE (xml_file, control_hash)); + DROP TABLE interest_links; + CREATE TABLE interest_links ( + id TEXT PRIMARY KEY, link_key TEXT NOT NULL UNIQUE, + person_id TEXT NOT NULL REFERENCES persons(id), bidder_id TEXT NOT NULL, eik TEXT NOT NULL, + entity_key TEXT NOT NULL, match_method TEXT, matcher_version TEXT NOT NULL, + publish_tier TEXT NOT NULL, relation TEXT NOT NULL, + interest_class TEXT NOT NULL DEFAULT 'management_role', contemporaneous INTEGER NOT NULL DEFAULT 0, + own_institution TEXT NOT NULL DEFAULT 'none', evidence_count INTEGER NOT NULL DEFAULT 1, + first_declared_year TEXT, last_declared_year TEXT, contract_count INTEGER NOT NULL DEFAULT 0, + contract_value_eur REAL, first_contract_year TEXT, last_contract_year TEXT, + status TEXT NOT NULL DEFAULT 'held', verified_by TEXT, verified_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')));`, + ); + // The rows the retrofit must preserve — plus the two shapes it must not. + sqlite( + dbPath, + `INSERT INTO persons(id, name) VALUES ('person:a', 'А'); + INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('d:keep', 'person:a', 'K.xml', 'HK', '2023', 'assets', 'https://x/K.xml'); + INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('d:drop', 'person:a', 'D.xml', NULL, '2023', 'assets', 'https://x/D.xml'); + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:k', 'k', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership', 'published'); + -- The uninterpretable status §2 is about, already stored because nothing rejected it. + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:bad', 'bad', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership', 'published ');`, + ); + + readScript(dbPath, migration10); + // The valid rows survive — nothing is rebuilt, so nothing can be lost… + expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( + 'd:keep', + ); + expect(sqlite(dbPath, "SELECT link_key FROM interest_links WHERE link_key='k';").trim()).toBe( + 'k', + ); + // …including the pre-existing trailing-space row. A trigger constrains FUTURE writes; it cannot + // retroactively reject a row already stored, and deleting one would mean dropping a real link on a + // guess. The loader rewrites the table wholesale on the next run, which is what corrects it — and + // the read gate never showed it anyway, since 'published ' is not 'published'. + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('2'); + // The hashless row SURVIVES — it is a real declaration and NOT NULL was rejected for that reason — + // but it is now covered by the natural key, so re-importing it is refused rather than duplicated. + expect(sqlite(dbPath, "SELECT COUNT(*) FROM declarations WHERE id='d:drop';").trim()).toBe( + '1', + ); + expect(() => + sqlite( + dbPath, + `INSERT INTO declarations + (id, person_id, xml_file, control_hash, folder_year, template, source_url) + VALUES ('d:dup', 'person:a', 'D.xml', NULL, '2023', 'assets', 'https://x/D.xml');`, + ), + ).toThrow(/UNIQUE/i); + // …and the constraint is in force for every writer afterwards, on INSERT and on UPDATE alike. + // UPDATE is the one that matters most: a hand-run status change during an incident is exactly when + // a stray character gets typed, and it is the path no application-level validation covers. + expect(() => + sqlite(dbPath, "UPDATE interest_links SET status='published ' WHERE link_key='k';"), + ).toThrow(/CHECK failed/i); + expect(() => + sqlite( + dbPath, + `INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, matcher_version, publish_tier, + relation, interest_class, status) + VALUES ('il:x', 'x', 'person:a', 'eik:1', '1', 'X', 't', 'document', 'owns', + 'private_ownership', 'publushed');`, + ), + ).toThrow(/CHECK failed/i); + // POSITIVE CONTROL: a legitimate status change still goes through — the trigger is a bound. + expect(() => + sqlite(dbPath, "UPDATE interest_links SET status='withdrawn' WHERE link_key='k';"), + ).not.toThrow(); + + // Migrations here are applied by a bare `d1 execute --file` with no applied-migrations tracking, + // so a second application MUST be a no-op rather than an error or a data loss. + readScript(dbPath, migration10); + expect(sqlite(dbPath, 'SELECT COUNT(*) FROM interest_links;').trim()).toBe('2'); + expect(sqlite(dbPath, "SELECT id FROM declarations WHERE id='d:keep';").trim()).toBe( + 'd:keep', + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/packages/db/src/queries/details.test.ts b/packages/db/src/queries/details.test.ts index 9371762f..014ffa8e 100644 --- a/packages/db/src/queries/details.test.ts +++ b/packages/db/src/queries/details.test.ts @@ -195,9 +195,35 @@ describe('getContract', () => { expect(detail?.value.suspect).toBe(true); expect(detail?.value.signingEur).toBe(256.49); expect(detail?.value.currentEur).toBe(flag === 'annex_suspect' ? 1025.96 : 256.49); + expect(detail?.value.currentValueDoubled).toBe(false); } }); + it('#307 blanks the current value for a KNOWN 2× double-count (annex_total_suspect)', async () => { + const detail = await getContract( + fakeDb( + { + ...baseContractRow, + signing_value: 256.49, + current_value: 512.98, // the doubled native figure — must NOT resurface + signing_value_eur: 256.49, + current_value_eur: null, // excluded from aggregates upstream + value_flag: 'annex_total_suspect', + }, + [], + ), + 'c:1', + ); + + expect(detail?.value.suspect).toBe(true); + expect(detail?.value.currentValueDoubled).toBe(true); + // Blanked (—), never the doubled 512.98 nor a fabricated fallback. + expect(detail?.value.currentEur).toBeNull(); + expect(detail?.value.deltaPct).toBeNull(); + // The trustworthy signing value is still shown. + expect(detail?.value.signingEur).toBe(256.49); + }); + // Exercises the real cohort path end-to-end (baseContractRow is clean-value, CPV '72', amount 5000). // Guards the argument order into contractCohort: swapping value_flag ↔ division would make it return // null and this would fail. @@ -351,6 +377,38 @@ describe('getContract', () => { }); }); + it('#305 residual: suppresses value_after and delta for a suspect (uncorrectable double-count) annex', async () => { + const detail = await getContract( + fakeDb( + { ...baseContractRow, contract_number: 'C-6' }, + [], + [ + { + value_before: 1000, + value_after: 3000, // the source's untrusted doubled/tripled total + value_delta: 2000, + currency: 'EUR', + published_at: '2024-03-01', + document_number: 'A1', + description: 'Изменение на стойността', + value_restated: 0, + value_suspect: 1, + fx_rate: null, + }, + ], + ), + 'c:1', + ); + + expect(detail?.amendments[0]).toMatchObject({ + valueAfterEur: null, // suppressed — we don't stand behind the doubled figure + deltaEur: null, + suspect: true, + restated: false, + description: 'Изменение на стойността', // description still shown + }); + }); + it('converts foreign-currency amendments to EUR via the annex fx rate', async () => { const detail = await getContract( fakeDb( diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index 7a211d8d..9786347e 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -431,6 +431,8 @@ interface AmendmentRow { published_at: string | null; document_number: string | null; description: string | null; + value_restated: number | null; + value_suspect: number | null; fx_rate: number | null; } @@ -454,7 +456,8 @@ export const AMENDMENTS_SQL = `SELECT am.value_before, am.value_after, am.value_ WHERE f.base_currency = am.currency AND f.rate_date <= am.published_at AND f.rate_date >= date(am.published_at, '-10 days') - ORDER BY f.rate_date DESC LIMIT 1) AS fx_rate + ORDER BY f.rate_date DESC LIMIT 1) AS fx_rate, + am.value_restated, am.value_suspect FROM amendments am WHERE am.unp = ? AND am.contract_number = ? ORDER BY am.published_at, am.id`; @@ -542,14 +545,20 @@ export async function getContract( const suspect = r.value_flag === 'value_suspect' || r.value_flag === 'annex_suspect' || + r.value_flag === 'annex_total_suspect' || r.value_flag === 'review' || r.value_flag === 'value_low'; const dateSuspect = r.date_flag === 'signed_after_publication'; + // #307 — annex_total_suspect is a KNOWN exact 2× double-count in current_value. Its current_value_eur is + // already NULL (excluded from aggregates), so the native fallback below would resurface the doubled figure + // under an "unverified" label. Blank it instead: a known-wrong number is worse than an honest gap. + const currentValueDoubled = r.value_flag === 'annex_total_suspect'; const signingEur = r.signing_value_eur ?? eurFromNative(r.signing_value, r.contract_currency, r.fx_rate); - const currentRaw = - r.current_value_eur ?? - eurFromNative(r.current_value, r.current_value_currency || r.contract_currency, r.fx_rate); + const currentRaw = currentValueDoubled + ? null + : (r.current_value_eur ?? + eurFromNative(r.current_value, r.current_value_currency || r.contract_currency, r.fx_rate)); const procedureEstimatedEur = eurFromNative( r.estimated_value, r.tender_currency, @@ -606,12 +615,13 @@ export async function getContract( estimatedEur: currentLotEstimatedEur ?? procedureEstimatedEur, procedureEstimatedEur, signingEur, - currentEur: currentRaw ?? signingEur, + currentEur: currentValueDoubled ? null : (currentRaw ?? signingEur), deltaPct: !suspect && currentRaw != null && signingEur != null && signingEur !== 0 ? (currentRaw - signingEur) / signingEur : null, suspect, + currentValueDoubled, }; const authority: ContractParty = { @@ -670,16 +680,28 @@ export async function getContract( const amendments: ContractDetail['amendments'] = amendmentRows.results.map((am) => { const beforeEur = eurFromNative(am.value_before, am.currency, am.fx_rate); const afterEur = eurFromNative(am.value_after, am.currency, am.fx_rate); + // #305 residual: a suspected double-count we could NOT correct from the основание text. Its + // value_after is the untrusted doubled figure, so suppress it (and the derived delta) rather than + // show a number we can't stand behind — the UI marks the row „непотвърден тотал". + const suspect = am.value_suspect === 1; return { date: am.published_at, documentNumber: am.document_number, description: am.description?.trim() || null, - valueAfterEur: afterEur, + valueAfterEur: suspect ? null : afterEur, // Compute delta from the SAME before/after we display, so the row is self-consistent (after − // before == delta) even when the source's recorded value_delta disagrees with them. When only // one of before/after is known, the recorded delta can't be reconciled against valueAfterEur — // show „—" rather than a figure that might not add up. - deltaEur: beforeEur != null && afterEur != null ? afterEur - beforeEur : null, + deltaEur: suspect + ? null + : beforeEur != null && afterEur != null + ? afterEur - beforeEur + : null, + // #305 Tier-2: the served value_after was rewritten from the основание text (a double-count total + // restated to the true value) — let the UI mark the corrected row. + restated: am.value_restated === 1, + suspect, }; }); diff --git a/packages/db/src/queries/related-persons.test.ts b/packages/db/src/queries/related-persons.test.ts index 51985651..8ba09404 100644 --- a/packages/db/src/queries/related-persons.test.ts +++ b/packages/db/src/queries/related-persons.test.ts @@ -33,6 +33,13 @@ function row(over: Record = {}) { first_contract_year: '2021', last_contract_year: '2024', source_url: 'https://register.cacbg.bg/2024/i.xml', + // SURFACED_OWNERSHIP only returns rows carrying a publishing seal, so the fixture carries one too — + // a fixture without it would exercise a row shape the SQL cannot produce. + evidence_kind: 'document', + registry_role: 'owner', + entry_number: '20110502101007', + entry_date: '2011-05-02', + lookup_date: '2026-08-05', ...over, }; } @@ -197,3 +204,47 @@ describe('conflict reads soft-fail on an un-migrated env (no 500)', () => { } }); }); + +// The evidence seal is what licenses the card's registry sentence, and the two rungs assert very +// different things: 'document' renders „лицето е вписано като съдружник/собственик" — that the register +// names THIS person in THIS company — while 'confirmed' claims only that the declared data matched. +// The mapper used to read `kind === 'confirmed' ? 'confirmed' : 'document'`, so NULL, 'refuted', a rung +// added later, or a typo all became the STRONGEST claim we can make about a named human being. The SQL +// gate makes that unreachable today; the failure DIRECTION is still wrong, and this is the one place in +// the codebase where being wrong by default is libel rather than a rendering glitch. +describe('an unrecognised evidence seal withholds the link instead of upgrading it', () => { + for (const kind of ['refuted', 'unknown', 'bar_joint_stock', 'outside_tr', 'future_rung_v9']) + it(`'${kind}' never renders as a registry claim`, async () => { + const db = fakeDb({ '10': [row({ evidence_kind: kind })] }); + expect(await getConflictLeaderboard(db, 10)).toEqual([]); + }); + + it('a NULL seal — the half-migrated read the old comment invited — withholds too', async () => { + const db = fakeDb({ '10': [row({ evidence_kind: null })] }); + expect(await getConflictLeaderboard(db, 10)).toEqual([]); + }); + + it('both publishing rungs still map, and to DIFFERENT kinds — the guard bounds, it does not flatten', async () => { + for (const kind of ['document', 'confirmed']) { + const db = fakeDb({ '10': [row({ evidence_kind: kind })] }); + const links = await getConflictLeaderboard(db, 10); + expect(links).toHaveLength(1); + expect(links[0]!.evidenceKind).toBe(kind); + } + }); + + it('the official page 404s rather than render a page under a name with nothing left to show', async () => { + const db = fakeDb({ 'person:ivan': [row({ evidence_kind: 'refuted' })] }); + expect(await getOfficialConflicts(db, 'person:ivan')).toBeNull(); + expect( + await getCompanyConflicts(fakeDb({ '111': [row({ evidence_kind: null })] }), '111'), + ).toBeNull(); + }); + + it('one withheld row does not take its sealed siblings down with it', async () => { + const db = fakeDb({ + '10': [row({ link_key: 'ok|111' }), row({ link_key: 'bad|111', evidence_kind: 'refuted' })], + }); + expect((await getConflictLeaderboard(db, 10)).map((l) => l.linkKey)).toEqual(['ok|111']); + }); +}); diff --git a/packages/db/src/queries/related-persons.ts b/packages/db/src/queries/related-persons.ts index 05c3f228..25f2a752 100644 --- a/packages/db/src/queries/related-persons.ts +++ b/packages/db/src/queries/related-persons.ts @@ -19,6 +19,9 @@ const CONFLICT_TABLES = [ 'declared_interests', 'interest_link_authorities', 'related_persons_internal', + // 0006 (#279, ADR-0033). Listed here for the same reason as the rest: on an environment where 0006 + // has not been applied yet, the evidence join must degrade to an empty surface rather than a 500. + 'interest_link_evidence', ]; // „D1_ERROR: no such table: interest_links: SQLITE_ERROR" → capture the table name and test membership. const MISSING_TABLE = /no such table:\s*(?:main\.)?"?([a-z_]+)"?/i; @@ -79,11 +82,28 @@ interface LinkRow { first_contract_year: string | null; last_contract_year: string | null; source_url: string | null; + evidence_kind: string | null; + registry_role: string | null; + entry_number: string | null; + entry_date: string | null; + lookup_date: string | null; } // The winner's contracts, joined exactly as the ETL aggregate does (contracts→tenders→authorities→bidders, // matched by eik_normalized) so any read-time subset is a true subset of the stored contract_count/value. // Alias-distinct (cc/tt/aa/bb) so it composes as a correlated subquery under the LINK_SELECT `il`/`b` scope. +// +// `tt` and `aa` are NOT projected here, which makes both joins look dead and invites deleting them. They +// are not dead — they are the SHAPE, and its counterpart is the WRITER at scripts/cacbg/load.mjs (the +// per-winner contract query that fills contract_count / contract_value_eur). The two must stay identical. +// +// Removing them on the read side ALONE would widen the read to contracts the stored aggregate never +// counted: `contemporaneous_contract_count` could then exceed `contract_count`, and the EXISTS gate below +// (the one that decides whether a link surfaces at all) would publish links the I5 zero-contract gate had +// excluded. Removing them on BOTH sides is defensible — an unresolvable authority currently drops a +// contract from the money everywhere, consistently — but it changes published figures and so must be +// re-baselined against ADR-0033 §10's control totals, not slipped in as a cleanup. Tracked as #226 §1.6. +// `related-persons-sql.test.ts` pins this string to the writer's so neither can be "optimised" alone. const CONTRACT_JOIN = `FROM contracts cc JOIN tenders tt ON tt.id = cc.tender_id JOIN authorities aa ON aa.id = tt.authority_id @@ -116,8 +136,15 @@ export const NEXUS_ORDER = `(il.own_institution = 'exact') DESC, (contemporaneou // The two surfaced ownership classes (ADR-0032): the official's own stake (private_ownership) and a close // relative's (family_ownership). Anchored once here + in LINK_CONTRACTS_SQL so the read gate and the // drill-down never drift. +// …and the identity must rest on a Trade Register fact (#279, ADR-0033). `status='published'` already +// encodes the loader's decision, so this EXISTS is belt-and-braces: it makes the read path refuse a link +// whose seal is missing or whose evidence is a withholding rung, even if a future writer sets status +// wrongly. 'document' and 'confirmed' are the only two rungs that publish; bar_joint_stock, unknown, +// refuted and outside_tr never reach a reader. export const SURFACED_OWNERSHIP = `il.status = 'published' - AND il.interest_class IN ('private_ownership', 'family_ownership')`; + AND il.interest_class IN ('private_ownership', 'family_ownership') + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed'))`; // Redundant-family collapse (ADR-0032, per todorkolev review). A family link is DROPPED when the SAME official // already has a published OWN stake in the SAME winner. Rendering both a self row and a family row for one // (official, ЕИК) is a de-anonymisation vector: the office-holder is himself in that company's Търговски @@ -156,8 +183,13 @@ export const LINK_SELECT = `SELECT il.link_key, il.person_id, p.name AS official -- (name, institution), ADR-0026; same subquery the search projection uses). Correlated per row, but the -- leaderboard is ≤1000 rows and hourly-cached, so the extra scan is immaterial. (SELECT d.institution FROM declarations d WHERE d.person_id = il.person_id - ORDER BY d.declared_year DESC LIMIT 1) AS institution + ORDER BY d.declared_year DESC LIMIT 1) AS institution, + -- The evidence the link rests on, so the card can explain itself (ADR-0033 decision 7). LEFT JOIN + -- rather than an inner one: SURFACED_OWNERSHIP already requires a publishing seal, and an inner join + -- here would silently re-filter rather than surface a contradiction. + ev.evidence_kind, ev.registry_role, ev.entry_number, ev.entry_date, ev.lookup_date FROM interest_links il + LEFT JOIN interest_link_evidence ev ON ev.link_key = il.link_key JOIN persons p ON p.id = il.person_id JOIN bidders b ON b.id = il.bidder_id WHERE ${SURFACED_OWNERSHIP} @@ -170,6 +202,25 @@ export const LINK_SELECT = `SELECT il.link_key, il.person_id, p.name AS official -- …and drop a family link redundant with the official's own stake in the same winner (de-anon guard). AND ${NOT_REDUNDANT_FAMILY}`; +// The two rungs that license a public claim (ADR-0033 decision 1), and the ONLY two the card knows how +// to render. Anything else — 'refuted', 'unknown', 'bar_joint_stock', 'outside_tr', a rung added by a +// later rules_version, a typo, or NULL from a row with no seal — is withheld, never mapped. +// +// The mapper used to read `kind === 'confirmed' ? 'confirmed' : 'document'`, which turned every one of +// those into 'document' — the STRONGEST claim on the surface, rendering „лицето е вписано като +// съдружник/собственик": that the register names this specific person in this specific company. The SQL +// gate makes it unreachable today, but the direction was wrong, and this is the one mapping in the +// codebase where a default is a defamatory statement about a named human being rather than a glitch. +// The LEFT JOIN in LINK_SELECT is deliberately not an inner one so a contradiction SURFACES here; the +// old fallback then converted exactly that contradiction into the strongest possible label. +const PUBLISHING_EVIDENCE = new Set(['document', 'confirmed']); + +/** Rows whose seal licenses a public claim. Withholding is silent by design — the SQL already filters + * these out, so anything reaching here is a contradiction to drop, not a condition to report per row. */ +function sealed(rows: LinkRow[]): LinkRow[] { + return rows.filter((r) => PUBLISHING_EVIDENCE.has(String(r.evidence_kind))); +} + // own_institution is a 4-value verdict; only the deterministic 'exact' surfaces as true (the // name_contains/locality heuristics are disclosed elsewhere, never asserted as fact). function toLink(r: LinkRow): ConflictLink { @@ -193,6 +244,18 @@ function toLink(r: LinkRow): ConflictLink { firstContractYear: r.first_contract_year, lastContractYear: r.last_contract_year, sourceUrl: r.source_url, + // Narrowed, not defaulted — `sealed()` above has already dropped every other value, so this asserts + // what the filter guarantees instead of inventing a rung the row never carried. + evidenceKind: r.evidence_kind as 'document' | 'confirmed', + registryRole: + r.registry_role === 'owner' || r.registry_role === 'manager' ? r.registry_role : null, + registryEntryNumber: r.entry_number, + registryEntryDate: r.entry_date, + // Narrowed for the same reason as evidenceKind, not defaulted. `lookup_date` is NOT NULL in + // migration 0006 and the row only reaches here through the seal filter, so `?? ''` was a branch + // that could not run — and if the invariant ever broke it would have shipped an empty string as a + // date, which reads as a valid-but-blank provenance rather than as the contradiction it is. + registryLookupDate: r.lookup_date as string, }; } @@ -203,7 +266,7 @@ export const LEADERBOARD_SQL = `${LINK_SELECT} * relative's) in a procurement winner, ranked NEXUS-first (own-institution → contemporaneous → value). */ export async function getConflictLeaderboard(db: D1Database, limit = 100): Promise { try { - const rows = (await db.prepare(LEADERBOARD_SQL).bind(limit).all()).results; + const rows = sealed((await db.prepare(LEADERBOARD_SQL).bind(limit).all()).results); return rows.map(toLink); } catch (e) { if (conflictSchemaAbsent(e, 'leaderboard')) return []; // un-migrated env → empty surface, not a 500 @@ -221,7 +284,9 @@ export async function getOfficialConflicts( personId: string, ): Promise { try { - const rows = (await db.prepare(OFFICIAL_SQL).bind(personId).all()).results; + // Filtered BEFORE the emptiness check, so a person whose every link is withheld 404s rather than + // rendering an empty page under their name. + const rows = sealed((await db.prepare(OFFICIAL_SQL).bind(personId).all()).results); if (rows.length === 0) return null; const links = rows.map(toLink); return { official: links[0]!.official, links }; @@ -240,7 +305,7 @@ export async function getCompanyConflicts( eik: string, ): Promise { try { - const rows = (await db.prepare(COMPANY_SQL).bind(eik).all()).results; + const rows = sealed((await db.prepare(COMPANY_SQL).bind(eik).all()).results); if (rows.length === 0) return null; return { company: rows[0]!.company, eik, links: rows.map(toLink) }; } catch (e) { diff --git a/packages/db/src/queries/search.ts b/packages/db/src/queries/search.ts index 7a19f01e..c3f3f02d 100644 --- a/packages/db/src/queries/search.ts +++ b/packages/db/src/queries/search.ts @@ -112,15 +112,23 @@ interface HitRow { // One group's ranked hits. Company rows additionally carry a свързани-лица flag: a LEFT JOIN against the // published conflict links keyed on the winner's ЕИК (= the company row's `ident`). Published, self OR family -// stake, AND the winner must have LIVE contracts (the same read-time N9 gate LINK_SELECT applies), so search -// flags exactly the companies the /conflicts surface shows and never one whose page would 404. A family-only -// winner badges — the /conflicts page already publishes that link by name, so the badge discloses nothing the -// surface doesn't. Binds: kind, match, limit. +// stake, backed by a Trade Register evidence SEAL, AND the winner must have LIVE contracts (the same +// read-time N9 gate LINK_SELECT applies), so search flags exactly the companies the /conflicts surface shows +// and never one whose page would 404. A family-only winner badges — the /conflicts page already publishes +// that link by name, so the badge discloses nothing the surface doesn't. Binds: kind, match, limit. +// +// The seal EXISTS clause is the same one SURFACED_OWNERSHIP applies (related-persons.ts), and it belongs +// here for a sharper reason than symmetry: search is the WIDER surface. A published row whose seal is +// missing or withholding — a legacy row, a partial run, a loader bug — would badge a свързани-лица claim +// about a named official to every searcher, while the page behind the badge correctly showed nothing. +// Four copies of this predicate now exist (here, SURFACED_OWNERSHIP, precompute.sql, refresh-slice.sql); +// related-persons-sql.test.ts pins them to each other. // Exported so search-sql.test runs the EXACT SQL (not a copy). -// Built with or without the interest_links conflict join. The join is on the свързани-лица migration (0003); -// on an env where it has not been applied yet the join would make the ENTIRE search 500 with „no such table: -// interest_links". search() detects the table once per request and picks the no-conflict variant when absent, -// so search degrades to has_conflict=0 rather than breaking (ADR-0031 robustness ask). +// Built with or without the conflict join. The join reads BOTH свързани-лица migrations — interest_links +// (0003) and interest_link_evidence (0006) — and on an env where either is unapplied it would make the +// ENTIRE search 500 with „no such table". search() detects both tables once per request and picks the +// no-conflict variant when either is absent, so search degrades to has_conflict=0 rather than breaking +// (ADR-0031 robustness ask). const hitsSql = (withConflict: boolean): string => `SELECT search_index.ref, search_index.title, search_index.ident, search_index.subtitle, search_index.amount, rank, ct.kind AS entity_kind, ct.ownership_kind, ct.eik_valid, @@ -133,6 +141,8 @@ const hitsSql = (withConflict: boolean): string => `SELECT search_index.ref, sea ? `LEFT JOIN ( SELECT DISTINCT il.eik FROM interest_links il WHERE il.status = 'published' AND il.interest_class IN ('private_ownership', 'family_ownership') + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed')) AND EXISTS (SELECT 1 FROM contracts cc JOIN bidders bb ON bb.id = cc.bidder_id WHERE bb.eik_normalized = il.eik) ) cf ON search_index.kind = 'company' AND cf.eik = search_index.ident` @@ -161,12 +171,21 @@ export async function search(db: D1Database, rawQuery: string): Promise(); const counts = new Map(countRows.results.map((r) => [r.kind, r.n])); - // Pick the hits SQL once: on an env where the свързани-лица migration (0003) is not applied, the conflict - // join would 500 the whole search — fall back to the no-conflict variant (has_conflict=0) instead. + // Pick the hits SQL once: on an env where the свързани-лица migrations are not applied, the conflict join + // would 500 the whole search — fall back to the no-conflict variant (has_conflict=0) instead. + // + // BOTH tables are required, not just interest_links: the join now also reads interest_link_evidence + // (0006), so an env with 0003 but not 0006 would break every search on every kind. Requiring both also + // gives the right answer on such an env — with no seal table nothing is provably sealed, so nothing + // should badge. const hasConflictTable = - (await db - .prepare("SELECT 1 AS n FROM sqlite_master WHERE type='table' AND name='interest_links'") - .first<{ n: number }>()) != null; + ( + await db + .prepare( + "SELECT COUNT(*) AS n FROM sqlite_master WHERE type='table' AND name IN ('interest_links','interest_link_evidence')", + ) + .first<{ n: number }>() + )?.n === 2; const hitsSql = hasConflictTable ? SEARCH_HITS_SQL : SEARCH_HITS_SQL_NO_CONFLICT; const built = await Promise.all( diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 261a2874..3415a3b6 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -13,6 +13,14 @@ const migration1Path = resolve(root, 'packages/db/migrations/0001_flow_pairs_bid const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); // refresh-slice.sql / precompute.sql officials block reads interest_links (0003) — build it in every chain. const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006, joined by the officials block for the Trade Register evidence gate (#279, ADR-0033). +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (refresh-slice promotes them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +// #305 residual: served amendments gained value_suspect (refresh-slice promotes it). +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); const deriveAmendmentsPath = resolve(root, 'scripts/derive-amendments.sql'); @@ -185,6 +193,10 @@ function initWorkDb(dbPath: string): void { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); } @@ -576,6 +588,10 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -650,6 +666,195 @@ describe('refresh-slice EOP base derivation', () => { } }); + it('bridges an OCDS-only annex to its УНП on the slice path (issue #286)', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-ocds-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration1Path); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + // 0006 too: refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279), so the + // script cannot parse against a DB that stops at 0003 — every site here applies both. + readScript(dbPath, migration9Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); + readScript(dbPath, workStagingSchemaPath); + + // An EOP procedure (tender + base contract) with УНП UNP-SLICE / tender.id TENDER-SLICE. An + // OCDS-only annex arrives keyed by the OCID, carrying tender_ext_id = TENDER-SLICE and NO EOP + // twin. The slice path's bridge must recover UNP-SLICE so the annex links to the served contract + // instead of staying a dead OCID row — the #286 fix, exercised through refresh-slice.sql. + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject, + cpv_code, cpv_description, contract_kind, estimated_value, currency, authority_name, + authority_eik, authority_type, published_at) + VALUES + ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-SLICE', 'TENDER-SLICE', + 'open', 'Slice tender', '45000000', 'Construction', 'works', 5000, 'BGN', + 'Authority Slice', '733456781', 'public', '2026-06-01'); + + INSERT INTO raw_contracts + (source, dataset_year, dataset_variant, fetched_at, needs_enrichment, document_number, + published_at, unp, tender_ext_id, procedure_type, procurement_subject, cpv_code, + cpv_description, contract_kind, estimated_value, procurement_currency, legal_basis, + award_criteria, authority_name, authority_eik, authority_type, main_activity, notice_type, + lot_id, contract_number, contract_date, signing_value, currency, contract_subject, + awarded_to_group, contractor_eik, contractor_name, contractor_country, winner_size, + eu_funded, bids_received, bids_sme, bids_rejected, bids_non_eea, duration_days) + VALUES + ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-SLICE', + '2026-06-01', 'UNP-SLICE', 'TENDER-SLICE', 'open', 'Slice tender', '45000000', + 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Slice', '733456781', + 'public', 'activity', 'notice', NULL, 'CONTRACT-SLICE', '2026-06-02', 1000, 'BGN', + 'Slice contract', 0, '787777778', 'Bidder Slice', 'BG', 'small', 0, 1, 1, 0, 0, 30); + + INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, + contract_number, contract_date, published_at, unp, tender_ext_id, authority_eik, + authority_name, procurement_subject, contract_kind, value_before, value_after, value_delta, + currency, description) + VALUES + ('ocds:2026-06-02', 2026, 'ocds', '2026-06-08T00:00:00Z', '1', 'AMD-SLICE-O', + 'CONTRACT-SLICE', '2026-06-02', '2026-06-03', 'ocds-e82gsb-555', 'TENDER-SLICE', + '733456781', 'Authority Slice', 'Slice tender', 'works', 1000, NULL, NULL, 'BGN', + 'OCDS-only annex');`, + ); + + readScript(dbPath, refreshSlicePath); + + // No served amendment keeps an OCID — the OCDS-only annex bridged to the real УНП. + expect( + sqliteJson<{ n: number }>( + dbPath, + "SELECT COUNT(*) AS n FROM amendments WHERE unp LIKE 'ocds-%'", + )[0]?.n, + ).toBe(0); + + // The annex is served against CONTRACT-SLICE, keyed by the recovered УНП (not the OCID). + expect( + sqliteJson<{ unp: string; source: string }>( + dbPath, + `SELECT unp, CASE WHEN source LIKE 'ocds:%' THEN 'ocds' ELSE 'eop' END AS source + FROM amendments WHERE contract_number = 'CONTRACT-SLICE'`, + ), + ).toEqual([{ unp: 'UNP-SLICE', source: 'ocds' }]); + + // …and it shows on the contract (annex_count = 1); current_value stays NULL — OCDS never sets + // an after-value, so nothing is fabricated. + expect( + sqliteJson<{ annex_count: number; current_value: number | null }>( + dbPath, + "SELECT annex_count, current_value FROM contracts WHERE contract_number = 'CONTRACT-SLICE'", + )[0], + ).toEqual({ annex_count: 1, current_value: null }); + + expect(sqlite(dbPath, 'PRAGMA foreign_key_check;').trim()).toBe(''); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('reconciles OCDS twins against the cumulative served amendments across windows (#286, HIGH 1)', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-xwindow-')); + try { + // A base EOP procedure (tender + contract) with УНП UNP-XW / tender.id TXW. `source` varies per + // window so a later slice re-reads the same contract (same derived id → INSERT OR REPLACE, no dup). + const base = (source: string): string => + `INSERT INTO raw_tenders + (source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject, + cpv_code, cpv_description, contract_kind, estimated_value, currency, authority_name, + authority_eik, authority_type, published_at) + VALUES + ('eop:tenders:${source}', 2026, '2026-06-09T00:00:00Z', 'UNP-XW', 'TXW', 'open', + 'Cross-window tender', '45000000', 'Construction', 'works', 5000, 'BGN', + 'Authority XW', '833456781', 'public', '2026-06-01'); + INSERT INTO raw_contracts + (source, dataset_year, dataset_variant, fetched_at, needs_enrichment, document_number, + published_at, unp, tender_ext_id, procedure_type, procurement_subject, cpv_code, + cpv_description, contract_kind, estimated_value, procurement_currency, legal_basis, + award_criteria, authority_name, authority_eik, authority_type, main_activity, notice_type, + lot_id, contract_number, contract_date, signing_value, currency, contract_subject, + awarded_to_group, contractor_eik, contractor_name, contractor_country, winner_size, + eu_funded, bids_received, bids_sme, bids_rejected, bids_non_eea, duration_days) + VALUES + ('eop:contracts:${source}', 2026, 'eop', '2026-06-09T00:00:00Z', 0, 'DOC-XW', + '2026-06-01', 'UNP-XW', 'TXW', 'open', 'Cross-window tender', '45000000', + 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority XW', '833456781', + 'public', 'activity', 'notice', NULL, 'CONTRACT-XW', '2026-06-02', 1000, 'BGN', + 'Cross-window contract', 0, '887777778', 'Bidder XW', 'BG', 'small', 0, 1, 1, 0, 0, 30);`; + + const eopAnnex = `INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, + contract_number, contract_date, published_at, unp, authority_eik, authority_name, + procurement_subject, contract_kind, value_before, value_after, value_delta, currency, description) + VALUES + ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-09T00:00:00Z', '1', 'AMD-XW-E', + 'CONTRACT-XW', '2026-06-02', '2026-06-03', 'UNP-XW', '833456781', 'Authority XW', + 'Cross-window tender', 'works', 1000, 1500, 500, 'BGN', 'EOP annex');`; + + const ocdsTwin = (source: string): string => + `INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, + contract_number, contract_date, published_at, unp, tender_ext_id, authority_eik, + authority_name, procurement_subject, contract_kind, value_before, value_after, value_delta, + currency, description) + VALUES + ('ocds:${source}', 2026, 'ocds', '2026-06-09T00:00:00Z', '1', 'AMD-XW-O', + 'CONTRACT-XW', '2026-06-02', '2026-06-03', 'ocds-e82gsb-321', 'TXW', '833456781', + 'Authority XW', 'Cross-window tender', 'works', 1000, NULL, NULL, 'BGN', 'OCDS twin');`; + + const servedRows = (dbPath: string) => + sqliteJson<{ unp: string; source: string }>( + dbPath, + `SELECT unp, CASE WHEN source LIKE 'ocds:%' THEN 'ocds' ELSE 'eop' END AS source + FROM amendments WHERE contract_number = 'CONTRACT-XW'`, + ); + const rollup = (dbPath: string) => + sqliteJson<{ annex_count: number; current_value: number | null }>( + dbPath, + "SELECT annex_count, current_value FROM contracts WHERE contract_number = 'CONTRACT-XW'", + )[0]; + + // Direction A — EOP annex served first, OCDS twin arrives in a LATER window (the EOP annex is not + // in that window's raw_amendments). Pre-fix, the twin bridged, survived the raw-only DELETE, and + // promoted into the cumulative served table → annex_count = 2. The served-table check must drop it. + const dbA = resolve(dir, 'a.sqlite'); + initWorkDb(dbA); + sqlite(dbA, `${base('2026-06-01')}\n${eopAnnex}`); + readScript(dbA, refreshSlicePath); + expect(servedRows(dbA)).toEqual([{ unp: 'UNP-XW', source: 'eop' }]); + + resetRawStaging(dbA); + sqlite(dbA, `${base('2026-06-05')}\n${ocdsTwin('2026-06-05')}`); + readScript(dbA, refreshSlicePath); + expect(servedRows(dbA)).toEqual([{ unp: 'UNP-XW', source: 'eop' }]); // NOT doubled + expect(rollup(dbA)).toEqual({ annex_count: 1, current_value: 1500 }); + expect(sqlite(dbA, 'PRAGMA foreign_key_check;').trim()).toBe(''); + + // Direction B — OCDS-only annex served first (net-new, no EOP twin yet), then the EOP annex arrives + // in a later window. The convergence DELETE must drop the stale served OCDS row so promotion of the + // EOP annex leaves exactly one served row (not two). + const dbB = resolve(dir, 'b.sqlite'); + initWorkDb(dbB); + sqlite(dbB, `${base('2026-06-01')}\n${ocdsTwin('2026-06-01')}`); + readScript(dbB, refreshSlicePath); + expect(servedRows(dbB)).toEqual([{ unp: 'UNP-XW', source: 'ocds' }]); + + resetRawStaging(dbB); + sqlite(dbB, `${base('2026-06-05')}\n${eopAnnex}`); + readScript(dbB, refreshSlicePath); + expect(servedRows(dbB)).toEqual([{ unp: 'UNP-XW', source: 'eop' }]); // OCDS twin converged away + expect(rollup(dbB)).toEqual({ annex_count: 1, current_value: 1500 }); + expect(sqlite(dbB, 'PRAGMA foreign_key_check;').trim()).toBe(''); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it('does not insert an OCDS duplicate after an existing EOP contract', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const dbPath = resolve(dir, 'test.sqlite'); @@ -658,6 +863,10 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -709,6 +918,10 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -760,6 +973,10 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -897,6 +1114,10 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, migration1Path); readScript(dbPath, migration2Path); readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, diff --git a/packages/db/src/related-persons-sql.test.ts b/packages/db/src/related-persons-sql.test.ts index af338041..ddcf4287 100644 --- a/packages/db/src/related-persons-sql.test.ts +++ b/packages/db/src/related-persons-sql.test.ts @@ -1,6 +1,6 @@ /// import { execFileSync } from 'node:child_process'; -import { mkdtempSync, rmSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -10,8 +10,11 @@ import { LEADERBOARD_SQL, LINK_CONTRACTS_LIMIT, LINK_CONTRACTS_SQL, + LINK_SELECT, OFFICIAL_SQL, + SURFACED_OWNERSHIP, } from './queries/related-persons'; +import { SEARCH_HITS_SQL } from './queries/search'; // Integration test for the свързани-лица SQL. The query layer's unit tests (queries/related-persons.test) // use a fake D1 and never run the aggregation; this runs the EXACT exported SQL against a real SQLite @@ -22,6 +25,8 @@ import { const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// …and 0006: SURFACED_OWNERSHIP now requires a Trade Register evidence seal (#279, ADR-0033). +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function sqlite(dbPath: string, sql: string): string { return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }).trim(); @@ -122,6 +127,13 @@ INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, co -- Rollup row for the awarding body — the per-authority capture-share denominator the read query LEFT JOINs. INSERT INTO authority_totals (authority_id, name, spent_eur, contracts, suppliers, avg_eur) VALUES ('a:1','ОБЩИНА ТЕСТ',50000000,10,4,5000000); + +-- Every link carries a Trade Register evidence seal (#279, ADR-0033). SURFACED_OWNERSHIP now requires +-- one, so a fixture without seals renders an EMPTY surface and every assertion below passes vacuously. +-- Derived from interest_links itself, so a row added to the fixture later is sealed automatically and +-- cannot silently drop off the surface. +INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links; `; describe('свързани-лица SQL (real SQLite)', () => { @@ -131,6 +143,7 @@ describe('свързани-лица SQL (real SQLite)', () => { try { readScript(dbPath, migration0); readScript(dbPath, migration2); + readScript(dbPath, migration9); sqlite(dbPath, FIXTURE); return fn(dbPath); } finally { @@ -193,7 +206,9 @@ describe('свързани-лица SQL (real SQLite)', () => { INSERT INTO persons (id, name) VALUES ('person:zero','Нула Тестов'); INSERT INTO interest_links (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES - ('il:zerofam','person:zero|666|family','person:zero','eik:666','666','НУЛА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2021',0,0,NULL,NULL,'published');`, + ('il:zerofam','person:zero|666|family','person:zero','eik:666','666','НУЛА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2021',0,0,NULL,NULL,'published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence);`, ); const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); expect(board.some((r) => r.official === 'Нула Тестов')).toBe(false); // no live contracts → gated out @@ -217,6 +232,8 @@ describe('свързани-лица SQL (real SQLite)', () => { (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES ('il:alen','person:alen|701','person:alen','eik:701','701','АЛЕН КО ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'none',1,'2020','2021',1,1000000,'2021','2021','published'), ('il:boyan','person:boyan|702','person:boyan','eik:702','702','БОЯН КО ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'none',1,'2020','2021',1,10000000,'2021','2021','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES ('t:71','unp71','Обект А','a:1','открита процедура'),('t:72','unp72','Обект Б','a:1','открита процедура'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES @@ -322,6 +339,8 @@ describe('свързани-лица SQL (real SQLite)', () => { (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES ('il:dubl-self','person:dubl|800','person:dubl','eik:800','800','ДУБЪЛ ЕООД','exact_name_key','v1','B_distinctive','owns','private_ownership',0,'none',1,'2020','2022',2,60000,'2021','2022','published'), ('il:dubl-fam','person:dubl|800|family','person:dubl','eik:800b','800','ДУБЪЛ ЕООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2022',2,60000,'2021','2022','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES ('t:80','unp80','Обект Дубъл','a:1','открита процедура'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES ('c:80','t:80','eik:800',60000,'EUR','2021-05-01','Д-80',60000);`, ); @@ -350,6 +369,8 @@ describe('свързани-лица SQL (real SQLite)', () => { (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES ('il:asim-self-held','person:asim|850','person:asim','eik:850','850','АСИМ ЕООД','exact_name_key','v1','C_hold','owns','private_ownership',0,'none',1,'2020','2022',2,40000,'2021','2022','held'), ('il:asim-fam','person:asim|850|family','person:asim','eik:850','850','АСИМ ЕООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2022',2,40000,'2021','2022','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES ('t:85','unp85','Обект Асим','a:1','открита процедура'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES ('c:85','t:85','eik:850',40000,'EUR','2021-05-01','Д-85',40000);`, ); @@ -475,7 +496,9 @@ describe('свързани-лица SQL (real SQLite)', () => { INSERT INTO persons (id, name) VALUES ('person:praz','Празен Тестов'); INSERT INTO interest_links (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES - ('il:praz','person:praz|900','person:praz','eik:900','900','ПРАЗЕН ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'exact',1,'2020','2022',7,7000000,'2021','2022','published');`, + ('il:praz','person:praz|900','person:praz','eik:900','900','ПРАЗЕН ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'exact',1,'2020','2022',7,7000000,'2021','2022','published'); + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links WHERE link_key NOT IN (SELECT link_key FROM interest_link_evidence);`, ); const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); expect(board.some((r) => r.official === 'Празен Тестов')).toBe(false); // gated out — no live contracts @@ -504,4 +527,173 @@ describe('свързани-лица SQL (real SQLite)', () => { ); }); }); + + // ── the evidence seal gate (#279, ADR-0033 decision 1) ────────────────────── + // + // This block exists because the gate had NO test pressure at all: deleting the entire + // `EXISTS (… interest_link_evidence …)` clause out of SURFACED_OWNERSHIP left the whole db suite + // green (361/361), because every fixture above seals every link 'document'. That clause is the only + // SQL between a WITHHELD link and a named public claim that a specific official owns a specific + // company, so an untested one is the most expensive kind of dead rail. + // + // Each rung below is a REAL withholding outcome of evidence.mjs, not an invented value: + // refuted — the register contradicts the declared stake + // bar_joint_stock — an АД/ЕАД, where a declared parcel of shares is not a material conflict + // unknown — no rung reached; the deed proves nothing either way + // outside_tr — ДЗЗД/BULSTAT, not in the Trade Register at all + // …plus a link with NO seal row whatsoever, which is what a half-loaded run produces. + describe('a link withheld by its evidence seal never reaches any public query', () => { + // All six share the SAME winner (eik 111, which has live contracts) and the same shape, so the + // ONLY thing that differs is the seal. Without that, an absent row could be absent for an unrelated + // reason and every assertion here would pass vacuously — which is exactly the bug being fixed. + const WITHHELD = [ + ['refuted', 'person:ref', 'Оборен Тестов'], + ['bar_joint_stock', 'person:bar', 'Акционер Тестов'], + ['unknown', 'person:unk', 'Неясен Тестов'], + ['outside_tr', 'person:out', 'Извън Тестов'], + ] as const; + + function seedRungs(dbPath: string): void { + const people: [string, string][] = [ + ...WITHHELD.map(([, id, name]) => [id, name] as [string, string]), + ['person:none', 'Безпечатен Тестов'], + ['person:ok', 'Потвърден Тестов'], + ]; + const links: string[] = [...WITHHELD.map(([, id]) => id), 'person:none', 'person:ok']; + sqlite( + dbPath, + `INSERT INTO persons (id, name) VALUES ${people.map(([id, n]) => `('${id}','${n}')`).join(',')}; + INSERT INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, relation, interest_class, contemporaneous, own_institution, evidence_count, first_declared_year, last_declared_year, contract_count, contract_value_eur, first_contract_year, last_contract_year, status) VALUES + ${links + .map( + (p) => + `('il:${p.split(':')[1]}','${p}|111','${p}','eik:111','111','ТРЕЙС ГРУП ХОЛД АД','exact_name_key','v1','B_distinctive','owns','private_ownership',1,'none',1,'2019','2023',3,1000,'2020','2021','published')`, + ) + .join(',')}; + INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) VALUES + ${WITHHELD.map(([kind, id]) => `('${id}|111','${kind}',NULL,NULL,'2026-08-05','tr-rules-1','live')`).join(',')}, + ('person:ok|111','confirmed',NULL,'seat:СОФИЯ','2026-08-05','tr-rules-1','live');`, + ); + // person:none deliberately gets NO evidence row at all. + } + + it('the POSITIVE CONTROL surfaces — so every absence below is caused by the seal, not the fixture', () => { + withDb((dbPath) => { + seedRungs(dbPath); + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => r.official === 'Потвърден Тестов')).toBe(true); + expect(rows(dbPath, lit(OFFICIAL_SQL, 'person:ok'))).toHaveLength(1); + }); + }); + + for (const [kind, personId, name] of WITHHELD) { + it(`'${kind}' is withheld from the leaderboard, the official page, the company page and the drill-down`, () => { + withDb((dbPath) => { + seedRungs(dbPath); + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => r.official === name)).toBe(false); + expect(board.some((r) => r.link_key === `${personId}|111`)).toBe(false); + // The official's own page must 404 rather than render a withheld claim under their name… + expect(rows(dbPath, lit(OFFICIAL_SQL, personId))).toHaveLength(0); + // …the company page must not list them among that winner's office-holders… + expect(rows(dbPath, lit(COMPANY_SQL, '111')).some((r) => r.official === name)).toBe( + false, + ); + // …and the drill-down must not enumerate the contracts of a link it may not name. + expect(rows(dbPath, lit(LINK_CONTRACTS_SQL, `${personId}|111`))).toHaveLength(0); + }); + }); + } + + it('a link with NO seal row at all is withheld too — a half-loaded run must not publish', () => { + // The LEFT JOIN in LINK_SELECT would happily return this row with NULL evidence columns; only the + // EXISTS gate keeps it off the surface. This is the case a partial load actually produces. + withDb((dbPath) => { + seedRungs(dbPath); + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => r.official === 'Безпечатен Тестов')).toBe(false); + expect(rows(dbPath, lit(OFFICIAL_SQL, 'person:none'))).toHaveLength(0); + expect( + rows(dbPath, lit(COMPANY_SQL, '111')).some((r) => r.official === 'Безпечатен Тестов'), + ).toBe(false); + expect(rows(dbPath, lit(LINK_CONTRACTS_SQL, 'person:none|111'))).toHaveLength(0); + }); + }); + + it('the company page shows ONLY the sealed office-holders of that winner', () => { + // Four withheld links and one confirmed link all point at eik 111, alongside the base fixture's + // Иван. A gate that let any withheld rung through would show up here as an extra name. + withDb((dbPath) => { + seedRungs(dbPath); + const names = rows(dbPath, lit(COMPANY_SQL, '111')).map((r) => r.official); + expect(names.sort()).toEqual(['Иван Минев', 'Потвърден Тестов']); + }); + }); + }); +}); + +// The read-time contract join and the WRITER that fills contract_count/contract_value_eur must use the +// same join shape. On the read side `tenders`/`authorities` are never projected, so both joins read as +// dead and a reviewer will eventually propose deleting them (one did — cefothe on #309). Deleting them +// there alone widens the read past the stored aggregate: contemporaneous counts could exceed +// contract_count, and the EXISTS gate would surface links the zero-contract gate excluded. Nothing but a +// test can hold a TypeScript template string and a .mjs prepared statement to the same shape. +describe('the read-time contract join matches the writer that stored the aggregate', () => { + const writer = readFileSync(resolve(root, 'scripts/cacbg/load.mjs'), 'utf8'); + // The writer's per-winner contract query, located by its projection rather than by line number. + const writerJoin = /FROM contracts c JOIN tenders t ON[^"]*?WHERE b\.eik_normalized/.exec( + writer, + )?.[0]; + + it('the writer still joins contracts→tenders→authorities→bidders', () => { + expect(writerJoin, 'load.mjs per-winner contract query not found').toBeTruthy(); + for (const rel of ['tenders', 'authorities', 'bidders']) expect(writerJoin).toContain(rel); + }); + + it('CONTRACT_JOIN joins the same four relations, in the same direction', () => { + // Compared by RELATION and join condition, not by text: the two are written in different languages + // and alias differently (cc/tt/aa/bb vs c/t/a/b), so only the shape is comparable. + const flat = LINK_SELECT.replace(/\s+/g, ' '); + expect(flat).toContain('FROM contracts cc'); + expect(flat).toContain('JOIN tenders tt ON tt.id = cc.tender_id'); + expect(flat).toContain('JOIN authorities aa ON aa.id = tt.authority_id'); + expect(flat).toContain('JOIN bidders bb ON bb.id = cc.bidder_id'); + // …and the writer's equivalents, so a change to either side fails here. + expect(writerJoin).toContain('JOIN tenders t ON t.id=c.tender_id'); + expect(writerJoin).toContain('JOIN authorities a ON a.id=t.authority_id'); + expect(writerJoin).toContain('JOIN bidders b ON b.id=c.bidder_id'); + }); +}); + +// The evidence-seal gate has FOUR copies, and they must say the same thing. Reviewers found the fourth +// (the company-search badge) checking `status='published'` alone, so search advertised a свързани-лица +// claim about a named official that the detail page correctly withheld. `refresh-slice.sql` matters for a +// different reason: it runs on the 6-hourly cron, so drift there silently keeps officials in the search +// index whose links no longer surface. Nothing but a test binds SQL in three languages and two file types. +describe('the evidence-seal gate is identical in all four places it is written', () => { + const sources: [string, string][] = [ + ['related-persons.ts (SURFACED_OWNERSHIP)', SURFACED_OWNERSHIP], + ['search.ts (company badge)', SEARCH_HITS_SQL], + ['precompute.sql', readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8')], + ['refresh-slice.sql', readFileSync(resolve(root, 'scripts/refresh-slice.sql'), 'utf8')], + ]; + + it.each(sources)('%s requires a publishing evidence seal', (_label, sql) => { + // Whitespace-insensitive: the four are formatted for their own file, and a line break is not drift. + const flat = sql.replace(/\s+/g, ' '); + expect(flat).toMatch( + /EXISTS \( ?SELECT 1 FROM interest_link_evidence e WHERE e\.link_key = il\.link_key AND e\.evidence_kind IN \('document','confirmed'\) ?\)/, + ); + }); + + it.each(sources)('%s admits exactly the two publishing rungs', (_label, sql) => { + // The rung list is the gate. A future rung added to one copy and not the others either leaks an + // unproven claim or silently drops a proven one, depending on which copy gained it. + const kinds = [...sql.matchAll(/evidence_kind IN \(([^)]*)\)/g)].map((m) => + m[1]!.replace(/\s|'/g, ''), + ); + expect(kinds.length).toBeGreaterThan(0); + for (const k of kinds) expect(k).toBe('document,confirmed'); + }); }); diff --git a/packages/db/src/search-sql.test.ts b/packages/db/src/search-sql.test.ts index 58c25205..59f9a95c 100644 --- a/packages/db/src/search-sql.test.ts +++ b/packages/db/src/search-sql.test.ts @@ -16,6 +16,7 @@ import { SEARCH_HITS_SQL, SEARCH_HITS_SQL_NO_CONFLICT } from './queries/search'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const migration0 = resolve(root, 'packages/db/migrations/0000_init.sql'); const migration2 = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +const migration9 = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); function readScript(dbPath: string, path: string): void { execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${path}\n`, stdio: 'pipe' }); @@ -44,17 +45,26 @@ INSERT INTO bidders (id, name, eik_normalized, eik_valid, kind) VALUES ('eik:333','ГАМА ООД','333333333',1,'company'), ('eik:444','ДЕЛТА ООД','444444444',1,'company'), ('eik:555','ЕПСИЛОН ООД','555555555',1,'company'), - ('eik:666','ЗЕТА ООД','666666666',1,'company'); + ('eik:666','ЗЕТА ООД','666666666',1,'company'), + -- #279: two winners with a PUBLISHED link that must not badge. ЙОТА's link carries NO evidence seal at + -- all (a legacy row, a partial run, or a loader bug); КАПА's seal is a WITHHOLDING rung. The detail page + -- refuses both via SURFACED_OWNERSHIP; the badge has to agree, or search advertises an unproven claim + -- about a named official on a page that then shows nothing. + ('eik:777','ЙОТА ООД','777777777',1,'company'), + ('eik:888','КАПА ООД','888888888',1,'company'); INSERT INTO company_totals (bidder_id, name, kind, eik, eik_valid, won_eur, contracts, authorities) VALUES ('eik:111','АЛФА ООД','company','111111111',1,1000000,1,1), ('eik:222','БЕТА ООД','company','222222222',1,500000,1,1), ('eik:333','ГАМА ООД','company','333333333',1,200000,1,1), ('eik:444','ДЕЛТА ООД','company','444444444',1,300000,1,1), ('eik:555','ЕПСИЛОН ООД','company','555555555',1,700000,1,1), - ('eik:666','ЗЕТА ООД','company','666666666',1,900000,1,1); + ('eik:666','ЗЕТА ООД','company','666666666',1,900000,1,1), + ('eik:777','ЙОТА ООД','company','777777777',1,400000,1,1), + ('eik:888','КАПА ООД','company','888888888',1,600000,1,1); INSERT INTO persons (id, name) VALUES ('person:ИВАН МИНЕВ','Иван Минев'),('person:ГЕОРГИ ПЕТРОВ','Георги Петров'), - ('person:ДАНА ФАМ','Дана Фам'),('person:БОРИС БОРД','Борис Борд'),('person:ДВОЕН ТЕСТ','Двоен Тест'); + ('person:ДАНА ФАМ','Дана Фам'),('person:БОРИС БОРД','Борис Борд'),('person:ДВОЕН ТЕСТ','Двоен Тест'), + ('person:БЕЗ ПЕЧАТ','Без Печат'),('person:ЗАДЪРЖАН ПЕЧАТ','Задържан Печат'); INSERT INTO declarations (id, person_id, xml_file, control_hash, folder_year, declared_year, template, category, institution, position, source_url) VALUES ('decl:i','person:ИВАН МИНЕВ','i.xml','H1','2024','2023','assets','','ОБЩИНА РУСЕ','', 'https://register.cacbg.bg/2024/i.xml'), ('decl:g','person:ГЕОРГИ ПЕТРОВ','g.xml','H2','2024','2023','assets','','МИНИСТЕРСТВО Х','', 'https://register.cacbg.bg/2024/g.xml'), @@ -75,15 +85,30 @@ INSERT INTO interest_links -- so the redundant-family collapse (ADR-0032) DROPS the family link from the index; the winner's €50k counts -- ONCE, not €100k — no de-anonymization vector, no double-count. ('il:ds','person:ДВОЕН ТЕСТ|666','person:ДВОЕН ТЕСТ','eik:666','666666666','ЗЕТА ООД','exact_name_key','v1','B_distinctive','owns','private_ownership',0,'none',1,'2020','2023',1,50000,'2021','2021','published'), - ('il:dfam','person:ДВОЕН ТЕСТ|666|family','person:ДВОЕН ТЕСТ','eik:666','666666666','ЗЕТА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2023',1,50000,'2021','2021','published'); + ('il:dfam','person:ДВОЕН ТЕСТ|666|family','person:ДВОЕН ТЕСТ','eik:666','666666666','ЗЕТА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',0,'none',1,'2020','2023',1,50000,'2021','2021','published'), + -- published, ownership class, live contracts — and NO seal. Everything the badge used to check, passed. + ('il:no','person:БЕЗ ПЕЧАТ|777','person:БЕЗ ПЕЧАТ','eik:777','777777777','ЙОТА ООД','exact_name_key','v1','document','owns','private_ownership',0,'none',1,'2020','2023',1,400000,'2021','2021','published'), + -- published, and sealed with a rung that WITHHOLDS (ADR-0035). A seal existing is not a seal permitting. + ('il:un','person:ЗАДЪРЖАН ПЕЧАТ|888','person:ЗАДЪРЖАН ПЕЧАТ','eik:888','888888888','КАПА ООД','exact_name_key','v1','document_uncorroborated','owns','private_ownership',0,'none',1,'2020','2023',1,600000,'2021','2021','published'); +INSERT INTO interest_link_evidence (link_key, evidence_kind, lookup_date, rules_version, live_status) VALUES + ('person:ИВАН МИНЕВ|111','document','2026-08-12','tr-rules-1','live'), + ('person:ИВАН МИНЕВ|333','confirmed','2026-08-12','tr-rules-1','live'), + ('person:ДАНА ФАМ|444','confirmed','2026-08-12','tr-rules-1','live'), + ('person:БОРИС БОРД|555','document','2026-08-12','tr-rules-1','live'), + ('person:ДВОЕН ТЕСТ|666','document','2026-08-12','tr-rules-1','live'), + ('person:ДВОЕН ТЕСТ|666|family','document','2026-08-12','tr-rules-1','live'), + ('person:ЗАДЪРЖАН ПЕЧАТ|888','document_uncorroborated','2026-08-12','tr-rules-1','live'); INSERT INTO authorities (id, name) VALUES ('a:1','ВЕДОМСТВО ТЕСТ'); INSERT INTO tenders (id, source_id, title, authority_id, procedure_type) VALUES - ('t:1','s1','Т1','a:1','open'),('t:3','s3','Т3','a:1','open'),('t:4','s4','Т4','a:1','open'),('t:6','s6','Т6','a:1','open'); + ('t:1','s1','Т1','a:1','open'),('t:3','s3','Т3','a:1','open'),('t:4','s4','Т4','a:1','open'), + ('t:6','s6','Т6','a:1','open'),('t:7','s7','Т7','a:1','open'),('t:8','s8','Т8','a:1','open'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES ('c:1','t:1','eik:111',1000000,'EUR','2021-05-01','N1',1000000), ('c:3','t:3','eik:333',200000,'EUR','2021-05-01','N3',200000), ('c:4','t:4','eik:444',300000,'EUR','2021-05-01','N4',300000), - ('c:6','t:6','eik:666',50000,'EUR','2021-05-01','N6',50000); + ('c:6','t:6','eik:666',50000,'EUR','2021-05-01','N6',50000), + ('c:7','t:7','eik:777',400000,'EUR','2021-05-01','N7',400000), + ('c:8','t:8','eik:888',600000,'EUR','2021-05-01','N8',600000); `; // Search-index population — a STRUCTURAL proxy for scripts/precompute.sql's officials block: it exercises the @@ -116,6 +141,7 @@ function withDb(fn: (dbPath: string) => void): void { try { readScript(dbPath, migration0); readScript(dbPath, migration2); + readScript(dbPath, migration9); exec(dbPath, FIXTURE); exec(dbPath, POPULATE_INDEX); fn(dbPath); @@ -151,6 +177,27 @@ describe('search свързани-лица SQL', () => { }); }); + it('the badge requires an evidence SEAL, not merely status=published (#279)', () => { + // The detail page gate is belt-and-braces — status AND a publishing seal (SURFACED_OWNERSHIP). The + // badge checked status alone, so an evidence-less published row advertised a свързани-лица claim on a + // named official in search while the page it links to correctly withheld it. Search is the wider + // surface of the two: it is what a reader sees before deciding to look. + withDb((dbPath) => { + // NO seal at all — the legacy/partial-run/loader-bug shape. + const iota = rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'йота*', 10)); + expect(iota).toHaveLength(1); + expect(iota[0]!.has_conflict).toBe(0); + // Sealed, but with a rung that WITHHOLDS. A seal existing is not a seal permitting. + const kapa = rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'капа*', 10)); + expect(kapa).toHaveLength(1); + expect(kapa[0]!.has_conflict).toBe(0); + // POSITIVE CONTROL — a properly sealed link still badges, on both publishing rungs. Without this a + // gate that rejected everything would satisfy the two assertions above. + expect(rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'алфа*', 10))[0]!.has_conflict).toBe(1); + expect(rows(dbPath, lit(SEARCH_HITS_SQL, 'company', 'гама*', 10))[0]!.has_conflict).toBe(1); + }); + }); + it('indexes family_ownership (ADR-0032) but NEVER management_role/ex-officio (interest_class gate)', () => { withDb((dbPath) => { // family_ownership now reaches the index identically to self (ADR-0032): Дана (a relative's stake) IS diff --git a/packages/db/src/seed-surface.test.ts b/packages/db/src/seed-surface.test.ts new file mode 100644 index 00000000..78a70aaa --- /dev/null +++ b/packages/db/src/seed-surface.test.ts @@ -0,0 +1,95 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { COMPANY_SQL, LEADERBOARD_SQL, OFFICIAL_SQL } from './queries/related-persons'; + +// The dev seed exists for ONE reason: `/conflicts` renders an empty surface without it, so the whole +// feature is unverifiable in a browser. That makes the seed a promise about the read gate, and a promise +// nothing checked — it was written against SURFACED_OWNERSHIP as it stood, then #279 added the evidence +// seal requirement to that same predicate and the fixture silently stopped surfacing. A fresh dev DB +// rendered empty, which is exactly the state the seed was created to prevent. +// +// So: run the REAL migrations, the REAL seed.sql, and the REAL exported queries. Any future change to +// the publishing gate that the seed does not keep up with now fails here rather than in someone's +// browser, days later, as „the local app looks broken". + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const migrations = [ + 'packages/db/migrations/0000_init.sql', + 'packages/db/migrations/0003_related_persons_foundation.sql', + 'packages/db/migrations/0009_interest_link_evidence.sql', +].map((m) => resolve(root, m)); +const seed = resolve(root, 'scripts/seed.sql'); + +function seededDb(fn: (dbPath: string) => T): T { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-seed-')); + const dbPath = resolve(dir, 'seed.sqlite'); + try { + for (const file of [...migrations, seed]) + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${file}\n`, stdio: 'pipe' }); + return fn(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} +function rows(dbPath: string, sql: string): Record[] { + const out = execFileSync('sqlite3', ['-json', dbPath], { input: sql, encoding: 'utf8' }).trim(); + return out ? JSON.parse(out) : []; +} +function lit(sql: string, ...vals: (string | number)[]): string { + let i = 0; + return sql.replace(/\?/g, () => { + const v = vals[i++]; + return typeof v === 'number' ? String(v) : `'${String(v).replace(/'/g, "''")}'`; + }); +} + +describe('scripts/seed.sql produces a surface a developer can actually see', () => { + it('the leaderboard is NOT empty — the seed survives every gate in SURFACED_OWNERSHIP', () => { + seededDb((dbPath) => { + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.length).toBeGreaterThan(0); + // Both published outcomes the seed is built around: the official's own stake and a relative's. + expect(board.map((r) => r.relation).sort()).toEqual(['owns', 'related']); + }); + }); + + it('every surfaced seed link carries a real publishing seal, not a coincidence', () => { + seededDb((dbPath) => { + for (const r of rows(dbPath, lit(LEADERBOARD_SQL, 100))) + expect(['document', 'confirmed']).toContain(r.evidence_kind); + }); + }); + + it('the held joint-stock link stays off the surface — the seed keeps its negative case', () => { + // ГАМА ИНВЕСТ АД is seeded as a withheld link precisely so a change to the publishing rule shows up + // as a before/after rather than as an empty page either way. If it ever surfaces, the materiality + // bar (ADR-0022) or the seal gate has quietly stopped working. + seededDb((dbPath) => { + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + expect(board.some((r) => String(r.company).includes('ГАМА ИНВЕСТ'))).toBe(false); + expect(rows(dbPath, lit(COMPANY_SQL, '204556676'))).toHaveLength(0); + }); + }); + + it('the official and company pages render too — not just the leaderboard', () => { + seededDb((dbPath) => { + const board = rows(dbPath, lit(LEADERBOARD_SQL, 100)); + const own = board.find((r) => r.relation === 'owns')!; + expect(rows(dbPath, lit(OFFICIAL_SQL, String(own.person_id))).length).toBeGreaterThan(0); + expect(rows(dbPath, lit(COMPANY_SQL, String(own.eik))).length).toBeGreaterThan(0); + }); + }); + + it('re-running the seed is idempotent — INSERT OR IGNORE, no duplicated links', () => { + seededDb((dbPath) => { + const before = rows(dbPath, lit(LEADERBOARD_SQL, 100)).length; + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${seed}\n`, stdio: 'pipe' }); + expect(rows(dbPath, lit(LEADERBOARD_SQL, 100))).toHaveLength(before); + }); + }); +}); diff --git a/packages/db/src/value-flag-annex-step-sql.test.ts b/packages/db/src/value-flag-annex-step-sql.test.ts new file mode 100644 index 00000000..8c52bd89 --- /dev/null +++ b/packages/db/src/value-flag-annex-step-sql.test.ts @@ -0,0 +1,338 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +// The mis-keyed annex rule (#248): annex_suspect used to need the aggregate current/signing ratio to +// reach 100x, so a single annex with a mis-typed value that pushed a contract to, say, 30x its signed +// value was served at face value. The new rule is a CONJUNCTION: one annex step jumped >=10x AND the +// aggregate ended >=5x over signing. Both halves are load-bearing - the corpus has chains with a +// 36,058x single step whose aggregate ends BELOW signing (a later annex corrects the typo), where +// flagging would RAISE the shown value, and slow legitimate chains that double a few times without any +// single suspicious step. +// +// The flag CASE lives in five copies across the two derive paths, plus a re-flag guard in the +// refresh-slice reconciliation pass, so the rule is exercised through the REAL scripts (in their real +// pipeline order) on a real SQLite database. A copy left behind in normalize-raw, or in the +// reconciliation pass, fails here rather than in production. The two refresh-slice INSERT-time copies +// are the one thing these tests CANNOT pin in isolation: refresh-slice promotes the window's +// amendments itself and its reconciliation pass then re-flags every amended contract, so a stale +// INSERT copy is always superseded in-script and never observable in the end state. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script +// only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it +// ever reaches the value_flag CASE these tests are about. +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const derivePath = resolve(root, 'scripts/derive-amendments.sql'); +const promotePath = resolve(root, 'scripts/promote-amendments.sql'); +// Real pipeline order (scripts/import.mjs): the full derive runs derive-amendments before +// normalize-raw; the slice derive runs derive-amendments, and promote-amendments has populated the +// served amendments table that refresh-slice's reconciliation pass re-rolls current_value from. +const etlRuns = [ + ['normalize-raw', [derivePath, resolve(root, 'scripts/normalize-raw.sql')]], + ['refresh-slice', [derivePath, promotePath, resolve(root, 'scripts/refresh-slice.sql')]], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-annexstep-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** Signing value in BGN; exactly 100_000 EUR so the multiples below read directly. The procedure + * estimate is the same, which keeps every case away from the estimate-driven flags (the стотинки + * band sits at 95x-105x the estimate; the largest aggregate used here is 30x). */ +const SIGNING_BGN = 195_583; +const AUTH_EIK = '000695114'; +const BIDDER_EIK = '831646048'; + +interface AmendmentStep { + before: number | null; + after: number; + publishedAt: string; +} + +interface Case { + unp: string; + steps: AmendmentStep[]; +} + +function seedContracts(dbPath: string, cases: { unp: string }[]): void { + const tenders = cases + .map( + (c) => + `('eop:tenders:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'public', ${SIGNING_BGN}, 'BGN')`, + ) + .join(',\n'); + const contracts = cases + .map( + (c) => + `('eop:contracts:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'C-${c.unp}', '2026-06-01', ${SIGNING_BGN}, 'BGN', '${BIDDER_EIK}', 'Тестов изпълнител')`, + ) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type, estimated_value, currency) + VALUES ${tenders}; + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, contractor_eik, contractor_name) + VALUES ${contracts};`, + ); +} + +function seedAmendments(dbPath: string, cases: Case[]): void { + const rows = cases + .flatMap((c) => + c.steps.map( + (s, i) => + `('eop:annexes:${c.unp}', '2026-06-01T00:00:00Z', 'A-${c.unp}-${i + 1}', '${c.unp}', 'C-${c.unp}', '${s.publishedAt}', ${s.before ?? 'NULL'}, ${s.after}, 'BGN')`, + ), + ) + .join(',\n'); + if (!rows) return; + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, fetched_at, document_number, unp, contract_number, published_at, + value_before, value_after, currency) + VALUES ${rows};`, + ); +} + +interface Row { + id: string; + value_flag: string; + amount_eur: number | null; +} + +const flagsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT id, value_flag, ROUND(amount_eur) AS amount_eur FROM contracts`, + ); + const out = new Map(); + for (const r of rows) { + const unp = r.id.split(':')[2]!; + out.set(unp, r); + } + return out; +}; + +describe('mis-keyed annex step in the value_flag CASE', () => { + for (const [label, scriptPaths] of etlRuns) { + it(`${label}: flags one big annex step whose aggregate stays inflated, and repairs to signing`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // A single mis-typed annex: 30x in one step, nothing corrects it. Far below the 100x the + // old rule needed. + { + unp: 'UNP-STEP30', + steps: [{ before: SIGNING_BGN, after: SIGNING_BGN * 30, publishedAt: '2026-06-10' }], + }, + // Both thresholds exactly at the boundary: step exactly 10x, aggregate exactly 5x. + { + unp: 'UNP-EDGE', + steps: [{ before: SIGNING_BGN / 2, after: SIGNING_BGN * 5, publishedAt: '2026-06-10' }], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const flags = flagsByUnp(dbPath); + for (const unp of ['UNP-STEP30', 'UNP-EDGE']) { + expect(flags.get(unp)?.value_flag, `${unp} should be annex_suspect`).toBe( + 'annex_suspect', + ); + // Fell back to the signed value (100 000 EUR), not served at the inflated current value. + expect(flags.get(unp)?.amount_eur, `${unp} repaired amount`).toBe(100_000); + } + }); + }); + + it(`${label}: never flags a chain whose later annex corrects the typo back down`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // The corpus counter-example class: a huge step UP, then a correction that lands the + // aggregate BELOW signing. Flagging would replace 50 000 EUR with 100 000 EUR - the + // repair itself would inflate the contract. + { + unp: 'UNP-CORRECTED', + steps: [ + { before: 5, after: SIGNING_BGN, publishedAt: '2026-06-10' }, + { before: SIGNING_BGN, after: SIGNING_BGN / 2, publishedAt: '2026-06-20' }, + ], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const flags = flagsByUnp(dbPath); + expect(flags.get('UNP-CORRECTED')?.value_flag).toBe('ok'); + // Served at the corrected current value - half the signing value. + expect(flags.get('UNP-CORRECTED')?.amount_eur).toBe(50_000); + }); + }); + + it(`${label}: needs BOTH halves - a big step under 5x aggregate, and a slow climb without one`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // Step 49x, but the aggregate stops at 4.9x signing - under the 5x floor. + { + unp: 'UNP-AGG49', + steps: [ + { before: SIGNING_BGN / 10, after: SIGNING_BGN * 4.9, publishedAt: '2026-06-10' }, + ], + }, + // Aggregate 8x through three doublings - no single step comes near 10x. + { + unp: 'UNP-SLOW8', + steps: [ + { before: SIGNING_BGN, after: SIGNING_BGN * 2, publishedAt: '2026-06-10' }, + { before: SIGNING_BGN * 2, after: SIGNING_BGN * 4, publishedAt: '2026-06-20' }, + { before: SIGNING_BGN * 4, after: SIGNING_BGN * 8, publishedAt: '2026-06-30' }, + ], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const flags = flagsByUnp(dbPath); + for (const unp of ['UNP-AGG49', 'UNP-SLOW8']) { + expect(flags.get(unp)?.value_flag, `${unp} must not be annex_suspect`).toBe('ok'); + } + // Both keep their as-recorded current value. + expect(flags.get('UNP-AGG49')?.amount_eur).toBe(490_000); + expect(flags.get('UNP-SLOW8')?.amount_eur).toBe(800_000); + }); + }); + + it(`${label}: the plain >=100x aggregate still flags even with no usable step`, () => { + withEtlDb(label, (dbPath) => { + const cases: Case[] = [ + // value_before missing on the only annex, so the step half can't fire - the original + // aggregate rule must still catch the 150x blow-up on its own. + { + unp: 'UNP-AGG150', + steps: [{ before: null, after: SIGNING_BGN * 150, publishedAt: '2026-06-10' }], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of scriptPaths) readScript(dbPath, p); + + const flags = flagsByUnp(dbPath); + expect(flags.get('UNP-AGG150')?.value_flag).toBe('annex_suspect'); + expect(flags.get('UNP-AGG150')?.amount_eur).toBe(100_000); + }); + }); + } + + it('refresh-slice: the CLI slice path (no external promote) lands the same flags', () => { + // scripts/import.mjs runSliceDerive runs derive-amendments + refresh-slice WITHOUT + // promote-amendments. That still works because refresh-slice promotes the window's amendments + // itself (INSERT OR REPLACE INTO amendments) before its reconciliation pass, which is therefore + // authoritative for annex flags on this path too - this run pins exactly that pipeline shape. + withEtlDb('no-promote', (dbPath) => { + const cases: Case[] = [ + { + unp: 'UNP-STEP30', + steps: [{ before: SIGNING_BGN, after: SIGNING_BGN * 30, publishedAt: '2026-06-10' }], + }, + { + unp: 'UNP-CORRECTED', + steps: [ + { before: 5, after: SIGNING_BGN, publishedAt: '2026-06-10' }, + { before: SIGNING_BGN, after: SIGNING_BGN / 2, publishedAt: '2026-06-20' }, + ], + }, + ]; + seedContracts(dbPath, cases); + seedAmendments(dbPath, cases); + for (const p of [derivePath, resolve(root, 'scripts/refresh-slice.sql')]) + readScript(dbPath, p); + + const flags = flagsByUnp(dbPath); + expect(flags.get('UNP-STEP30')?.value_flag).toBe('annex_suspect'); + expect(flags.get('UNP-STEP30')?.amount_eur).toBe(100_000); + expect(flags.get('UNP-CORRECTED')?.value_flag).toBe('ok'); + expect(flags.get('UNP-CORRECTED')?.amount_eur).toBe(50_000); + }); + }); + + it('refresh-slice: re-flags a served contract when the annex arrives in a LATER window', () => { + // The production shape of this bug class: the contract clears one refresh window as 'ok', the + // mis-keyed annex lands in the next. The reconciliation pass matches the contract through + // raw_amendments alone, re-rolls current_value from the served amendments table, and its + // re-flag guard must let the contract through to be re-classified - a guard still keyed to the + // old >=100x rule would keep it 'ok' forever. + const [, refreshScripts] = etlRuns[1]!; + withEtlDb('two-window', (dbPath) => { + const cases: Case[] = [ + { + unp: 'UNP-LATE', + steps: [{ before: SIGNING_BGN, after: SIGNING_BGN * 30, publishedAt: '2026-07-10' }], + }, + ]; + // Window 1: the contract alone. + seedContracts(dbPath, cases); + for (const p of refreshScripts) readScript(dbPath, p); + expect(flagsByUnp(dbPath).get('UNP-LATE')?.value_flag).toBe('ok'); + + // Window 2: only the annex is in the transient staging. + sqlite(dbPath, 'DELETE FROM raw_contracts; DELETE FROM raw_tenders;'); + seedAmendments(dbPath, cases); + for (const p of refreshScripts) readScript(dbPath, p); + + const row = flagsByUnp(dbPath).get('UNP-LATE'); + expect(row?.value_flag).toBe('annex_suspect'); + expect(row?.amount_eur).toBe(100_000); + }); + }); +}); diff --git a/packages/db/src/value-flag-lot-stotinki-sql.test.ts b/packages/db/src/value-flag-lot-stotinki-sql.test.ts new file mode 100644 index 00000000..67fea352 --- /dev/null +++ b/packages/db/src/value-flag-lot-stotinki-sql.test.ts @@ -0,0 +1,327 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +// Issue #247, second reported contract. The стотинки band added in #298 measures against the PROCEDURE +// estimate, so it only fires when the whole procedure is one lot. On a multi-lot procedure the dropped +// decimal point still lands at exactly 100x the LOT's own estimate, while the ratio to the procedure +// total lands wherever that lot's share puts it - 00621-2020-0008 sits at 89.8x and is served at +// 14,212,416 EUR instead of ~142,000 EUR. +// +// The own-row estimate CANNOT be used on its own: docs/etl.md is explicit that for framework and +// unit-price procedures (medicines, fuel) it is a UNIT price, and a whole call-off legitimately dwarfs +// it. So the arm is a conjunction - 95x..105x of the own estimate AND at least 10x the procedure +// estimate, the threshold that already means "implausibly large for this procedure". A unit-price +// call-off sits at ~1x the procedure estimate and is spared. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// The амендмент columns refresh-slice.sql/promote-amendments.sql now write into served `amendments` +// (#305 value_restated/value_treatment/value_suspect, #306 contract_number_raw/link_method). Without +// them sqlite3 aborts on the amendment promotion long before it reaches the value_flag CASE under test. +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +// #279/ADR-0033: refresh-slice.sql's свързани-лица block reads interest_link_evidence, so 0009 too. +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const etlPaths = [ + ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], + ['refresh-slice', resolve(root, 'scripts/refresh-slice.sql')], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-lotband-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); + readScript(dbPath, migration9Path); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** The lot's own estimate: 195_583 BGN = exactly 100_000 EUR, so the multiples read directly. */ +const OWN_BGN = 195_583; +const AUTH_EIK = '000695114'; +const BIDDER_EIK = '831646048'; + +interface Case { + unp: string; + /** Contract value, in BGN. */ + valueBgn: number; + /** The lot's own estimate on the contract row, in BGN. `null` = the row carries none. */ + ownEstBgn: number | null; + /** The whole procedure's estimate, in BGN. */ + procEstBgn: number; + /** Currency the row's OWN estimate is denominated in (`procurement_currency`). Defaults to BGN. */ + ownEstCurrency?: string; +} + +function seed(dbPath: string, cases: Case[]): void { + const tenders = cases + .map( + (c) => + `('eop:tenders:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'public', ${c.procEstBgn}, 'BGN')`, + ) + .join(',\n'); + const contracts = cases + .map( + (c) => + `('eop:contracts:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'C-${c.unp}', '2026-06-01', ${c.valueBgn}, 'BGN', ${c.ownEstBgn ?? 'NULL'}, '${c.ownEstCurrency ?? 'BGN'}', '${BIDDER_EIK}', 'Тестов изпълнител')`, + ) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type, estimated_value, currency) + VALUES ${tenders}; + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, estimated_value, procurement_currency, + contractor_eik, contractor_name) + VALUES ${contracts};`, + ); +} + +/** + * An annex on the contract. With `afterBgn` omitted it is a no-op (before = after), which leaves + * `current_value` on the signing value and `eff_eur` unchanged; its only job then is to make the row + * visible to refresh-slice.sql's reconciliation pass, whose `contract_base` requires the contract to + * HAVE an amendment. With a raised `afterBgn` it also trips the annex condition that makes that pass + * actually recompute the flag rather than keep the one the INSERT path assigned. + */ +function seedAnnex(dbPath: string, unp: string, beforeBgn: number, afterBgn = beforeBgn): void { + sqlite( + dbPath, + `INSERT INTO raw_amendments + (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, contract_number, + contract_date, published_at, unp, authority_eik, authority_name, value_before, value_after, + value_delta, currency, contractor_eik, description) + VALUES + ('eop:annexes:${unp}', 2026, 'eop', '2026-06-08T00:00:00Z', '1', 'AMD-${unp}', 'C-${unp}', + '2026-06-01', '2026-06-09', '${unp}', '${AUTH_EIK}', 'Тестов възложител', ${beforeBgn}, + ${afterBgn}, ${afterBgn - beforeBgn}, 'BGN', '${BIDDER_EIK}', 'Изменение');`, + ); +} + +interface Row { + id: string; + value_flag: string; + amount_eur: number | null; +} + +const flagsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT id, value_flag, ROUND(amount_eur) AS amount_eur FROM contracts`, + ); + const out = new Map(); + for (const r of rows) out.set(r.id.split(':')[2]!, r); + return out; +}; + +describe('стотинки band measured against the lot estimate (issue #247)', () => { + for (const [label, scriptPath] of etlPaths) { + it(`${label}: catches the reported multi-lot case the procedure band misses`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // The shape of 00621-2020-0008: exactly 100x the lot estimate, but only 89.8x the procedure + // estimate, so the #298 band (which measures against the procedure) never sees it. + { + unp: 'UNP-LOT100', + valueBgn: OWN_BGN * 100, + ownEstBgn: OWN_BGN, + procEstBgn: Math.round(OWN_BGN * (100 / 89.8)), + }, + ]); + readScript(dbPath, scriptPath); + const row = flagsByUnp(dbPath).get('UNP-LOT100'); + expect(row, 'the seeded contract must reach the served table').toBeDefined(); + expect(row?.value_flag).toBe('value_suspect'); + // Repaired to the procedure estimate, the established anchor for value_suspect. + expect(row?.amount_eur).toBe(111_358); + }); + }); + + it(`${label}: spares a unit-price call-off at 100x its own per-unit estimate`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // A framework/unit-price procedure: the row estimate is a UNIT price, the call-off is 100x it + // in absolute terms but only a fifth of the procedure ceiling. Repairing this would DESTROY a + // real contract - it must stay untouched. + { + unp: 'UNP-UNIT', + valueBgn: OWN_BGN * 100, + ownEstBgn: OWN_BGN, + procEstBgn: OWN_BGN * 500, + }, + ]); + readScript(dbPath, scriptPath); + const row = flagsByUnp(dbPath).get('UNP-UNIT'); + // Without this the `.not.toBe` below passes vacuously on an unseeded row (review cefothe #1). + expect(row, 'the seeded contract must reach the served table').toBeDefined(); + expect(row?.value_flag).not.toBe('value_suspect'); + // Served at face value: 100 x 100_000 EUR. + expect(row?.amount_eur).toBe(10_000_000); + }); + }); + + it(`${label}: pins the 95x/105x edges of the band`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // Exactly on each edge — both inside, because the band is inclusive on purpose. + { unp: 'UNP-AT95', valueBgn: OWN_BGN * 95, ownEstBgn: OWN_BGN, procEstBgn: OWN_BGN * 9 }, + { + unp: 'UNP-AT105', + valueBgn: OWN_BGN * 105, + ownEstBgn: OWN_BGN, + procEstBgn: OWN_BGN * 10, + }, + // One step outside each edge — a dropped decimal lands AT ~100x, never at 94x or 106x, so + // these are ordinary large contracts and must keep their money. + { unp: 'UNP-AT94', valueBgn: OWN_BGN * 94, ownEstBgn: OWN_BGN, procEstBgn: OWN_BGN * 9 }, + { + unp: 'UNP-AT106', + valueBgn: OWN_BGN * 106, + ownEstBgn: OWN_BGN, + procEstBgn: OWN_BGN * 10, + }, + ]); + readScript(dbPath, scriptPath); + const flags = flagsByUnp(dbPath); + for (const unp of ['UNP-AT95', 'UNP-AT105', 'UNP-AT94', 'UNP-AT106']) + expect(flags.get(unp), `seeded contract ${unp} missing`).toBeDefined(); + expect(flags.get('UNP-AT95')?.value_flag).toBe('value_suspect'); + expect(flags.get('UNP-AT105')?.value_flag).toBe('value_suspect'); + expect(flags.get('UNP-AT94')?.value_flag).not.toBe('value_suspect'); + expect(flags.get('UNP-AT106')?.value_flag).not.toBe('value_suspect'); + expect(flags.get('UNP-AT94')?.amount_eur).toBe(9_400_000); + expect(flags.get('UNP-AT106')?.amount_eur).toBe(10_600_000); + }); + }); + + it(`${label}: pins the 1000 EUR floors on both estimates`, () => { + withEtlDb(label, (dbPath) => { + const TINY = 1_000; // ≈ 511 EUR — under the floor + seed(dbPath, [ + // Own estimate under 1000 EUR: at these sizes a 100x ratio is noise, not a dropped decimal. + { unp: 'UNP-TINYOWN', valueBgn: TINY * 100, ownEstBgn: TINY, procEstBgn: TINY * 10 }, + // Procedure estimate under 1000 EUR: same reasoning on the other half of the conjunction. + { unp: 'UNP-TINYPROC', valueBgn: OWN_BGN * 100, ownEstBgn: OWN_BGN, procEstBgn: TINY }, + ]); + readScript(dbPath, scriptPath); + const flags = flagsByUnp(dbPath); + expect(flags.get('UNP-TINYOWN'), 'seeded contract missing').toBeDefined(); + expect(flags.get('UNP-TINYPROC'), 'seeded contract missing').toBeDefined(); + expect(flags.get('UNP-TINYOWN')?.value_flag).not.toBe('value_suspect'); + expect(flags.get('UNP-TINYPROC')?.value_flag).not.toBe('value_suspect'); + }); + }); + + it(`${label}: a row with NO own estimate is not caught by the own-row arm`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // No own estimate at all. The own-row arm has nothing to measure against and must not fire; + // the procedure-level band (#298) is the only thing that could still speak here, and at 10x + // the procedure estimate it does not. This is the one place the INSERT and the UPDATE path + // could disagree, because the reconciliation pass reads the estimate from a different column. + { unp: 'UNP-NOOWN', valueBgn: OWN_BGN * 100, ownEstBgn: null, procEstBgn: OWN_BGN * 10 }, + ]); + seedAnnex(dbPath, 'UNP-NOOWN', OWN_BGN * 100); + readScript(dbPath, scriptPath); + const row = flagsByUnp(dbPath).get('UNP-NOOWN'); + expect(row, 'the seeded contract must reach the served table').toBeDefined(); + expect(row?.value_flag).not.toBe('value_suspect'); + expect(row?.amount_eur).toBe(10_000_000); + }); + }); + + it(`${label}: pins the procedure-level floor of the conjunction at 10x`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // Exactly at the floor: 100x the lot estimate and exactly 10x the procedure estimate. + { + unp: 'UNP-AT10', + valueBgn: OWN_BGN * 100, + ownEstBgn: OWN_BGN, + procEstBgn: OWN_BGN * 10, + }, + // Just under it: 100x the lot estimate but only ~8.9x the procedure estimate. + { + unp: 'UNP-UNDER10', + valueBgn: OWN_BGN * 100, + ownEstBgn: OWN_BGN, + procEstBgn: Math.round(OWN_BGN * 11.2), + }, + ]); + readScript(dbPath, scriptPath); + const flags = flagsByUnp(dbPath); + expect(flags.get('UNP-AT10'), 'seeded contract missing').toBeDefined(); + expect(flags.get('UNP-UNDER10'), 'seeded contract missing').toBeDefined(); + expect(flags.get('UNP-AT10')?.value_flag).toBe('value_suspect'); + expect(flags.get('UNP-UNDER10')?.value_flag).not.toBe('value_suspect'); + // Served at face value — the spared row keeps its money, which is the thing that matters. + expect(flags.get('UNP-UNDER10')?.amount_eur).toBe(10_000_000); + }); + }); + } + + // refresh-slice only. The reconciliation pass keeps whatever flag the INSERT path computed unless the + // contract ALSO trips an annex condition — so that is the only shape in which its own copy of the band + // decides anything, and the only way to pin it. (review cefothe #4/#5) + it(`refresh-slice reconciliation: converts the own estimate through ITS currency, not the contract's`, () => { + withEtlDb('reconcile-currency', (dbPath) => { + const AFTER_BGN = OWN_BGN * 100; // 10 000 000 EUR + const SIGNING_BGN = 100_000; // an annex ratio ≥ 100x, which forces the recompute + seed(dbPath, [ + { + unp: 'UNP-RECCUR', + valueBgn: SIGNING_BGN, + ownEstBgn: 100_000, // …denominated in EUR: the contract value is exactly 100x it + ownEstCurrency: 'EUR', + procEstBgn: 977_915, // 500 000 EUR, so the 10x procedure floor is cleared + }, + ]); + seedAnnex(dbPath, 'UNP-RECCUR', SIGNING_BGN, AFTER_BGN); + readScript(dbPath, resolve(root, 'scripts/refresh-slice.sql')); + const row = flagsByUnp(dbPath).get('UNP-RECCUR'); + expect(row, 'the seeded contract must reach the served table').toBeDefined(); + // Converting the estimate with the CONTRACT's currency divides an already-EUR figure by 1.95583, + // lands at ~196x instead of 100x, drops out of the band — and the row falls through to + // annex_suspect, which repairs to the signing value: 51 129 EUR shown instead of 500 000. + expect(row?.value_flag).toBe('value_suspect'); + expect(row?.amount_eur).toBe(500_000); + }); + }); +}); diff --git a/packages/db/src/value-flag-stotinki-sql.test.ts b/packages/db/src/value-flag-stotinki-sql.test.ts new file mode 100644 index 00000000..49ab5e6f --- /dev/null +++ b/packages/db/src/value-flag-stotinki-sql.test.ts @@ -0,0 +1,179 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +// The стотинки band (#247): a value entered in cents rather than leva lands at almost exactly 100x the +// procedure estimate, well under the 200x threshold that value_suspect used to need, so it was served +// at face value. Measured on the real corpus the ratios form an isolated cluster - 13 contracts between +// 95x and 105x, two between 85x and 95x, and NOTHING between 105x and 200x - which is why the band is +// narrow rather than a lowered multiplier. +// +// Both derive paths carry their own copy of the flag CASE (two in normalize-raw, three in +// refresh-slice), so the rule is exercised through the REAL scripts on a real SQLite database, once per +// path. A copy left behind fails here rather than in production. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const migration2Path = resolve(root, 'packages/db/migrations/0002_current_value_currency.sql'); +const migration3Path = resolve(root, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// refresh-slice.sql's свързани-лица block reads interest_link_evidence (#279, ADR-0033), so the script +// only parses against a DB that also has 0006 — without it sqlite3 aborts on „no such table" before it +// ever reaches the value_flag CASE these tests are about. +const migration9Path = resolve(root, 'packages/db/migrations/0009_interest_link_evidence.sql'); +// #305 Tier-2: served amendments gained value_restated/value_treatment (promote + refresh-slice write them). +const migration6Path = resolve(root, 'packages/db/migrations/0006_amendment_restated.sql'); +const migration7Path = resolve(root, 'packages/db/migrations/0007_amendment_value_suspect.sql'); +// #306 provenance columns on served `amendments` — promote/refresh-slice write contract_number_raw + link_method. +const migration8Path = resolve(root, 'packages/db/migrations/0008_amendment_provenance.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const etlPaths = [ + ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], + ['refresh-slice', resolve(root, 'scripts/refresh-slice.sql')], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-stotinki-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, migration2Path); + readScript(dbPath, migration3Path); + readScript(dbPath, migration9Path); + readScript(dbPath, migration6Path); + readScript(dbPath, migration7Path); + readScript(dbPath, migration8Path); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** Estimate in BGN; the EUR estimate is this ÷ 1.95583. Contract values below are in BGN too. */ +const ESTIMATE_BGN = 195_583; // exactly 100_000 EUR, so the multiples below are easy to read +const AUTH_EIK = '000695114'; +const BIDDER_EIK = '831646048'; + +/** One tender + one contract per case, so each case gets its own procedure estimate. */ +function seed(dbPath: string, cases: { unp: string; valueBgn: number }[]): void { + const tenders = cases + .map( + (c) => + `('eop:tenders:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'public', ${ESTIMATE_BGN}, 'BGN')`, + ) + .join(',\n'); + const contracts = cases + .map( + (c) => + `('eop:contracts:${c.unp}', '2026-06-01T00:00:00Z', '${c.unp}', '${AUTH_EIK}', 'Тестов възложител', 'C-${c.unp}', '2026-06-01', ${c.valueBgn}, 'BGN', '${BIDDER_EIK}', 'Тестов изпълнител')`, + ) + .join(',\n'); + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type, estimated_value, currency) + VALUES ${tenders}; + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, contractor_eik, contractor_name) + VALUES ${contracts};`, + ); +} + +interface Row { + id: string; + value_flag: string; + amount_eur: number | null; +} + +const flagsByUnp = (dbPath: string): Map => { + const rows = sqliteJson( + dbPath, + `SELECT id, value_flag, ROUND(amount_eur) AS amount_eur FROM contracts`, + ); + const out = new Map(); + for (const r of rows) { + const unp = r.id.split(':')[2]!; + out.set(unp, r); + } + return out; +}; + +describe('стотинки band in the value_flag CASE', () => { + for (const [label, scriptPath] of etlPaths) { + it(`${label}: flags a value at ~100x the estimate and repairs it to the estimate`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // Exactly 100x: the signature of a value typed in стотинки. + { unp: 'UNP-X100', valueBgn: ESTIMATE_BGN * 100 }, + // Inside the band but not exactly 100x: the стотинки value of a contract whose own price came + // in slightly under the procedure estimate, so the ratio lands a little below the multiple. + { unp: 'UNP-X97', valueBgn: Math.round(ESTIMATE_BGN * 97) }, + ]); + readScript(dbPath, scriptPath); + + const flags = flagsByUnp(dbPath); + for (const unp of ['UNP-X100', 'UNP-X97']) { + expect(flags.get(unp)?.value_flag, `${unp} should be value_suspect`).toBe( + 'value_suspect', + ); + // Repaired to the procedure estimate (100 000 EUR), not served at 100x. + expect(flags.get(unp)?.amount_eur, `${unp} repaired amount`).toBe(100_000); + } + }); + }); + + it(`${label}: leaves ordinary overruns and the gap above the band alone`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [ + // A real overrun: flagged 'review' since 2020, but the value is kept. + { unp: 'UNP-X20', valueBgn: ESTIMATE_BGN * 20 }, + // Just above the band. The corpus has nothing between 105x and 200x; if a case ever appears + // there it must NOT be silently repaired to the estimate. + { unp: 'UNP-X120', valueBgn: ESTIMATE_BGN * 120 }, + // Just below the band, where the two real 85x-95x contracts live. + { unp: 'UNP-X90', valueBgn: ESTIMATE_BGN * 90 }, + ]); + readScript(dbPath, scriptPath); + + const flags = flagsByUnp(dbPath); + for (const unp of ['UNP-X20', 'UNP-X120', 'UNP-X90']) { + expect(flags.get(unp)?.value_flag, `${unp} must not be value_suspect`).toBe('review'); + // Kept at face value, which is the whole point of 'review' as opposed to 'value_suspect'. + expect(flags.get(unp)?.amount_eur, `${unp} keeps its value`).toBeGreaterThan(1_000_000); + } + }); + }); + + it(`${label}: still catches the plain over-200x case it always did`, () => { + withEtlDb(label, (dbPath) => { + seed(dbPath, [{ unp: 'UNP-X500', valueBgn: ESTIMATE_BGN * 500 }]); + readScript(dbPath, scriptPath); + const flags = flagsByUnp(dbPath); + expect(flags.get('UNP-X500')?.value_flag).toBe('value_suspect'); + expect(flags.get('UNP-X500')?.amount_eur).toBe(100_000); + }); + }); + } +}); diff --git a/packages/ingest/src/amendment-total.test.ts b/packages/ingest/src/amendment-total.test.ts new file mode 100644 index 00000000..ff1f9cef --- /dev/null +++ b/packages/ingest/src/amendment-total.test.ts @@ -0,0 +1,348 @@ +// #305 — the value-double-count text heuristic, tested against REAL основание text pulled from the live +// corpus (contracts 145652, 189325, 84818, 108677, 79382, 113291, 103903). The hazard is false positives, +// so the controls (genuine increment, uncorrectable, normal increase) matter as much as the hits. +import { describe, expect, it } from 'vitest'; +import { + classifyAmendmentValue, + restatedValueAfter, + isGenuineIncrement, + type AmendmentValueInput, +} from './amendment-total'; + +const mk = ( + valueBefore: number, + valueAfter: number, + valueDelta: number, + currency: string, + ...texts: string[] +): AmendmentValueInput => ({ valueBefore, valueAfter, valueDelta, currency, texts }); + +describe('#305 amendment value double-count heuristic', () => { + it('restates a total announced as "…на " (145652)', () => { + const t = classifyAmendmentValue( + mk( + 442000, + 981240, + 539240, + 'BGN', + 'относно актуализиране стойността на договора … общата стойност на договора ще възлезе на 539 240.00 лв. без ДДС', + ), + ); + expect(t).toEqual({ kind: 'total_restated', correctedAfter: 539240 }); + }); + + it('restates a total announced as "…от X на " (79382, 113291 family)', () => { + expect( + restatedValueAfter( + mk( + 197720, + 484414, + 286694, + 'BGN', + 'Общата стойност на договор № 447 се променя от 197 720.00 лева без ДДС на 286 694,00 (двеста осемдесет и шест хиляди) лева без ДДС', + ), + ), + ).toBe(286694); + expect( + restatedValueAfter( + mk( + 13662405.12, + 28356190.64, + 14693785.52, + 'BGN', + 'в чл. 7 (1) от Договора общата цена за изпълнение предмета на договора се променя от 13 662 405,12 лв. без ДДС на 14 693 785,52 лв. без ДДС', + ), + ), + ).toBe(14693785.52); + }); + + it('does NOT restate "в размер на / ресурс / increment" phrasings — they name the change, not the total', () => { + // Real corpus 271148→650754: "…максималния ресурс за изменението в размер на 379 606.50" names the + // INCREMENT; restating value_after to it would understate a genuine >100% increase. Must be `none`. + expect( + classifyAmendmentValue( + mk( + 271147.5, + 650754, + 379606.5, + 'BGN', + 'Срокът се удължава до изчерпване на максималния ресурс за изменението в размер на 379 606.50 лв. без ДДС', + ), + ).kind, + ).toBe('none'); + // "…или сума в размер на 86 363.08" — the added-work amount, not the new contract total. + expect( + restatedValueAfter( + mk( + 71969.23, + 151662.82, + 79693.59, + 'BGN', + 'Общата стойност на договорените СМР се променя от 71 969.23 без ДДС или сума в размер на 79 693.59 лв.', + ), + ), + ).toBeNull(); + }); + + it('restates a currency re-denomination that doubled an unchanged total (189325)', () => { + const t = classifyAmendmentValue( + mk( + 77000000, + 154000000, + 77000000, + 'BGN', + 'Считано от 01.10.2025 г., отпечатваната върху акцизните бандероли продажна цена се променя от лева в евро.', + ), + ); + expect(t).toEqual({ kind: 'unchanged_restated', correctedAfter: 77000000 }); + }); + + it('does NOT touch a genuine increment announced as "…с " (108677)', () => { + const input = mk( + 10226.85, + 60226.85, + 50000, + 'EUR', + 'Увеличава се финансовият ресурс на договор № АО – 05 – 168 с 50 000 /петдесет хиляди/ евро без ДДС.', + ); + expect(isGenuineIncrement(input)).toBe(true); + expect(restatedValueAfter(input)).toBeNull(); + }); + + it('does NOT restate an exact-2× when the text announces a DIFFERENT total ("до 18 900") — flag, not rewrite (103903)', () => { + // #307: exact 2× (delta 15 120 = value_before) BUT the text says the value rose "до 18 900" — it is NOT + // unchanged. Neither the doubled 30 240 nor the halved 15 120 is the true total, and 18 900 is not + // recoverable here, so return `none` and let the arithmetic annex_total_suspect flag exclude the row. + const t = classifyAmendmentValue( + mk( + 15120, + 30240, + 15120, + 'BGN', + 'Прогнозната стойност по договора се увеличава от 15 120 лв. без ДДС до 18 900 лв. без ДДС', + ), + ); + expect(t).toEqual({ kind: 'none' }); + }); + + it('does NOT text-freely restate an exact-2× when the основание carries no restatement signal (84818)', () => { + // #307: restructuring note with no value/unchanged signal. A text-free halving could erase a legitimate + // ЗОП чл.116 ал.1 т.1 in-scope +100% (pre-announced option clause), so it must fall to the arithmetic + // annex_total_suspect flag (exclude), not be rewritten to the before-value. + const t = classifyAmendmentValue( + mk( + 76769540.87, + 153539081.74, + 76769540.87, + 'EUR', + 'Следните курсове за 22 пилота се преструктурират и се изпълняват в рамките на гаранционния период', + ), + ); + expect(t).toEqual({ kind: 'none' }); + }); + + it('restates an exact-2× administrative annex (non-value change) to the before-value', () => { + const t = classifyAmendmentValue( + mk( + 2685, + 5370, + 2685, + 'BGN', + 'Променя се упълномощеното лице по договора. Несъществени промени.', + ), + ); + expect(t).toEqual({ kind: 'unchanged_restated', correctedAfter: 2685 }); + }); + + it('does NOT text-freely restate an exact-2× on an outside-ЗОП exception contract', () => { + // ЗОП чл.116's +50% cap does not bind exception contracts, so an exact +100% there can be a genuine + // increase — the text-free rule 3 must stand down and let the arithmetic flag exclude (not rewrite) it. + const base = mk( + 2685, + 5370, + 2685, + 'BGN', + 'Променя се упълномощеното лице по договора. Несъществени промени.', + ); + expect(classifyAmendmentValue({ ...base, outsideZop: true }).kind).toBe('none'); + // …but the same row in-scope of ЗОП is still restated (guard is scoped to rule 3 only). + expect(classifyAmendmentValue({ ...base, outsideZop: false })).toEqual({ + kind: 'unchanged_restated', + correctedAfter: 2685, + }); + }); + + it('still applies the text-confirmed rules on an outside-ЗОП contract', () => { + // The double-count is a feed defect independent of ЗОП scope, so a text-confirmed total is corrected + // even for an exception contract — only the text-free exact-2× fallback is gated by outsideZop. + const total = { + ...mk( + 442000, + 981240, + 539240, + 'BGN', + 'общата стойност на договора ще възлезе на 539 240.00 лв.', + ), + outsideZop: true, + }; + expect(classifyAmendmentValue(total)).toEqual({ + kind: 'total_restated', + correctedAfter: 539240, + }); + const incr = { + ...mk(10226.85, 60226.85, 50000, 'EUR', 'Увеличава се ресурсът с 50 000 евро без ДДС.'), + outsideZop: true, + }; + expect(isGenuineIncrement(incr)).toBe(true); + }); + + it('ignores normal increases (< 2×) and non-self-consistent rows', () => { + expect(classifyAmendmentValue(mk(100, 130, 30, 'BGN', 'обща стойност на 130 лв.')).kind).toBe( + 'none', + ); + // a ≠ b + d ⇒ the double-count model does not apply + expect(classifyAmendmentValue(mk(100, 250, 100, 'BGN', 'обща стойност на 100')).kind).toBe( + 'none', + ); + }); + + it('requires the text figure to actually equal the delta (no coincidental match)', () => { + // delta 500000 appears nowhere as a total; a different figure 12345 does — must not restate. + expect( + restatedValueAfter(mk(400000, 900000, 500000, 'BGN', 'обща стойност на 12 345 лв.')), + ).toBeNull(); + }); + + it('does NOT restate a bare "…на " over a NON-monetary number (#307 HIGH-1 — days / article nos.)', () => { + // "…удължава на 200 дни": 200 is a day count that coincidentally == value_delta. Without a currency + // anchor around the figure it must stay `none`, never overwrite the published 300 with 200. + expect( + classifyAmendmentValue(mk(100, 300, 200, 'BGN', 'Срокът на договора се удължава на 200 дни.')) + .kind, + ).toBe('none'); + // An article number after "на" — non-monetary, must not restate. + expect( + classifyAmendmentValue( + mk(100, 300, 200, 'BGN', 'Договорът се изменя на 200 съгласно чл. 116 на ЗОП.'), + ).kind, + ).toBe('none'); + }); + + it('does NOT anchor a day-count on a currency token elsewhere in the sentence (#307 MONEY_AFTER window)', () => { + // "…удължава на 200 дни, стойността остава 100 лв.": 200 is a DAY count; the "лв." belongs to a + // different figure downstream. A non-monetary unit right after 200 must veto it, not restate 300→200. + expect( + classifyAmendmentValue( + mk(100, 300, 200, 'BGN', 'Срокът се удължава на 200 дни, стойността остава 100 лв.'), + ).kind, + ).toBe('none'); + expect( + classifyAmendmentValue( + mk( + 100, + 300, + 200, + 'BGN', + 'Срокът за изпълнение на договора се удължава на 200 дни, без промяна в договорената сума в лв.', + ), + ).kind, + ).toBe('none'); + }); + + it('vetoes the QUALIFIED day/quantity unit, not just the bare word (#307 review — работни/календарни дни class)', () => { + // The unit almost never comes bare in real annexes ("работни дни", "календарни дни", "200 (двеста) + // дни", "кв.м"). Each of these is a duration/quantity that coincidentally == value_delta; none may + // overwrite the published 300 with 200. Tests the error CLASS, not one literal sentence. + const dayCounts = [ + 'Срокът се удължава на 200 работни дни, стойността остава 100 лв.', + 'Срокът се удължава на 200 календарни дни, стойността остава 100 лв.', + 'Срокът се удължава на 200 работни дни, без промяна в договорената сума в лв.', + 'Срокът се удължава на 200 к.д., стойността остава 100 лв.', + 'Срокът се удължава на 200 (двеста) дни, стойността остава 100 лв.', + 'Срокът се удължава на 200 р.д., стойността остава 100 лв.', + 'Площта се увеличава на 200 кв.м, стойността остава 100 лв.', + 'Обемът се увеличава на 200 куб.м, стойността остава 100 лв.', + ]; + for (const text of dayCounts) { + expect(classifyAmendmentValue(mk(100, 300, 200, 'BGN', text)).kind).toBe('none'); + } + }); + + it('the wider unit veto does NOT swallow a real monetary total (#307 review — reverse direction)', () => { + // A qualified/adjacent-word unit veto must not fire on genuine money phrasings: the figure still + // restates to the announced total. Guards against the veto over-reaching. + const realTotals = [ + 'Общата стойност на договора възлиза на 200 лв. без ДДС.', + 'Общата стойност на договора възлиза на 200 лева.', + 'Новата обща стойност възлиза на 200 лв. за срок от 12 месеца.', + 'Общата стойност се увеличава на 200 лева месечно.', + 'Общата стойност възлиза на 200 лв. и срокът се удължава с 30 работни дни.', + ]; + for (const text of realTotals) { + expect(restatedValueAfter(mk(100, 300, 200, 'BGN', text))).toBe(200); + } + }); + + it('does NOT restate an exact-2× on a bare payment-in-euro clause (#307 в-евро narrowing)', () => { + // "Плащанията…се извършват в евро…" is a payment-currency clause, NOT an unchanged-value signal — it + // must not halve a real +100%. Only "X в евро" re-denomination phrasing may restate (see 189325). + const t = classifyAmendmentValue( + mk( + 250000, + 500000, + 250000, + 'BGN', + 'Плащанията по договора се извършват в евро по сметка на изпълнителя.', + ), + ); + expect(t).toEqual({ kind: 'none' }); + }); + + it('restates a bare "…на " only WHEN a currency unit follows the figure (#307 HIGH-1 anchor)', () => { + // Same "…на " shape as the days case, but a currency unit anchors it as money ⇒ genuine total. + expect( + restatedValueAfter( + mk(100, 300, 200, 'BGN', 'Общата стойност на договора се променя на 200 лв. без ДДС.'), + ), + ).toBe(200); + }); + + it('does NOT rewrite an exact-2× with empty / whitespace-only texts (#307 HIGH-2 repro)', () => { + const t = classifyAmendmentValue({ + valueBefore: 539240, + valueAfter: 1078480, + valueDelta: 539240, + currency: 'BGN', + texts: [null, '', ' '], + outsideZop: null, + }); + expect(t).toEqual({ kind: 'none' }); + }); + + it('does NOT rewrite an exact-2× when the text is unrelated to value (#307 HIGH-2 repro)', () => { + const t = classifyAmendmentValue({ + valueBefore: 250000, + valueAfter: 500000, + valueDelta: 250000, + currency: 'BGN', + outsideZop: false, + texts: ['Смяна на адреса за кореспонденция на изпълнителя.'], + }); + expect(t).toEqual({ kind: 'none' }); + }); + + it('parses a dot-thousands + comma-decimal figure "1.234,56" (#305 number-format recall)', () => { + // Mixed-separator total announced as "…на 1.234,56 лв." — the resolver must read 1234.56, not 1.23. + expect( + restatedValueAfter( + mk(700, 1934.56, 1234.56, 'BGN', 'Общата стойност на договора се променя на 1.234,56 лв.'), + ), + ).toBe(1234.56); + // …and the US ordering "1,234.56" resolves to the same value. + expect( + restatedValueAfter( + mk(700, 1934.56, 1234.56, 'BGN', 'Общата стойност на договора се променя на 1,234.56 лв.'), + ), + ).toBe(1234.56); + }); +}); diff --git a/packages/ingest/src/amendment-total.ts b/packages/ingest/src/amendment-total.ts new file mode 100644 index 00000000..fb5e4495 --- /dev/null +++ b/packages/ingest/src/amendment-total.ts @@ -0,0 +1,227 @@ +// #305 — amendment value double-count. ЦАИС ЕОП sometimes puts the announced NEW TOTAL contract value +// into the "change" field (`contractValueDifference` → `value_delta`), so the feed's +// `currentContractValue` (→ `value_after`) = `lastContractValue` + newTotal — the value is doubled. The +// source is internally consistent (`value_after = value_before + value_delta` at ~100% of rows), so the +// signal is semantic, not arithmetic: is `value_delta` an increment or a total? +// +// This module answers that from the основание free text, which the raw feed carries in three fields +// (changeDescription/changeReason/changeReasonDescription). The discriminator is the Bulgarian preposition +// in front of the figure: "на " (to N) / "възлиза/става/обща стойност" ⇒ N is a TOTAL; "с " (by N) / +// "увеличава се … с" ⇒ N is an INCREMENT. See docs/implementation-plans/305-amendment-value-double-count.md. +// +// Conservative by design: it only classifies when the text unambiguously confirms; otherwise it returns +// `none` and leaves the row to the arithmetic `annex_total_suspect` flag (Tier 1). It NEVER rewrites a +// value it cannot corroborate from text. Note: JS `\b`/`\w` are ASCII-only, so all boundaries/letters use +// Unicode (`\p{L}`, explicit non-letter boundary) with the `u` flag. + +export type AmendmentValueTreatment = + // The delta is an announced total; the true value_after is value_delta (double-count corrected). + | { kind: 'total_restated'; correctedAfter: number } + // An exact 2× (value_delta ≈ value_before): the "difference" field echoed the OLD value, so the value + // is unchanged and value_after was doubled onto itself; the true value_after is value_before. Covers + // currency re-denominations and non-value administrative annexes alike. + | { kind: 'unchanged_restated'; correctedAfter: number } + // The delta is a genuine increment already correctly applied — value_after is right; do NOT flag it. + | { kind: 'genuine_increment' } + // No text signal — leave to the arithmetic flag. + | { kind: 'none' }; + +export interface AmendmentValueInput { + valueBefore: number | null; + valueAfter: number | null; + valueDelta: number | null; + currency: string | null; + texts: Array; + // #305 — the text-free exact-2× rule leans on ЗОП чл.116 (a single amendment caps at +50%, so +100% is a + // defect not a real increase). чл.116 does NOT bind contracts procured outside ЗОП (exception contracts), + // where a genuine +100% is legal — so for those, only the text-confirmed rules may restate. NULL/false = + // in-scope of ЗОП (the safe default: apply the rule). + outsideZop?: boolean | null; +} + +const REL_TOL = 0.005; // 0.5% — the text figure must be the SAME number as value_delta, allowing rounding. +// #305 — capture a full number token that may group thousands with space/nbsp/narrow-nbsp OR with '.'/',' +// (BG "1.234,56", US "1,234.56"); normalizeBgNumber disambiguates the decimal mark below. The token must +// END on a digit so a trailing sentence period ("…100. Нов срок") is not swallowed into the number. +const NUMBER_RE = /\d[\d\u0020\u00a0\u202f.,]*\d|\d/g; +const WS = /[\s  ]/g; + +// A left boundary: start-of-window or a non-letter, non-digit character (Unicode-aware — Cyrillic is a +// letter). Keywords that end the "before the figure" window signal how the figure should be read. +const B = '(?:^|[^\\p{L}\\d])'; +const TOTAL_CTX = new RegExp( + `${B}(?:възлиз\\p{L}*|възлез\\p{L}*|става|обща\\p{L}*\\s+(?:стойност|цена)|крайн\\p{L}*\\s+(?:стойност|цена)|нов\\p{L}*\\s+(?:обща\\s+)?(?:стойност|цена)|на)\\s*$`, + 'iu', +); +// The figure sits right after "от " (the OLD value) or "с/със " (an increment) — not a total. +const NOT_TOTAL_CTX = new RegExp(`${B}(?:от|с|със)\\s*$`, 'iu'); +// "…с " / "…със " — N is an increment already applied. +const INCREMENT_CTX = new RegExp(`${B}(?:с|със)\\s*$`, 'iu'); +// A wider veto on the "…на " total match: Bulgarian "в размер на " ("in the amount of N"), +// "ресурс … в размер на N", "допълнителни … на обща стойност N" name the CHANGE/added-work amount, not +// the new contract total — restating value_after := N there would be wrong (verified on the real corpus). +// Checked over a wider window than NOT_TOTAL_CTX because these markers sit a few words before the figure. +const TOTAL_VETO = /(?:в\s+размер|ресурс\p{L}*|допълнителн\p{L}*|увеличени\p{L}*|намалени\p{L}*)/iu; + +// #307 — a total restatement needs a MONETARY anchor bracketing the figure. Bare "на " is not a money +// signal ("на" also precedes days, article numbers, quantities), so "…удължава на 200 дни" would otherwise +// rewrite the value with a day count. Accept the figure only when a value keyword sits immediately before +// it (MONEY_BEFORE) OR a currency unit follows it (MONEY_AFTER). On the real corpus the value keyword is +// usually far from the figure ("…ще възлезе на 539 240.00 лв."), so the currency unit after the number is +// the load-bearing anchor. No ASCII \b (Cyrillic). +const MONEY_BEFORE = /(?:стойност|цена)\p{L}*\s*$/iu; +const MONEY_AFTER = /(?:^|[^\p{L}])(?:лв\.?|лева|лев|bgn|eur|евро|euro|€|usd|\$)(?![\p{L}])/iu; +// #307 — MONEY_AFTER scans the whole ~60-char window, so a sentence that names both a term and a value +// ("…удължава на 200 дни, стойността остава 100 лв.") lets a downstream currency token anchor a figure +// that is actually a day count. A non-monetary unit sitting IMMEDIATELY after the figure (days, months, +// years, count, percent) overrides any currency further along: the figure is a duration/quantity, never +// the contract value. Anchored at ^ against the post-figure slice so only the immediate suffix counts. +// Real BG annexes almost never write the unit bare — the term is qualified ("работни дни", "календарни +// дни") — so allow one optional adjective word (and an optional spelled-out number in brackets, "200 +// (двеста) дни") between the figure and the unit, and cover area/volume/weight units too. Errs safe: a +// false veto only downgrades a row to `none`, dropping it to the arithmetic annex_total_suspect flag +// rather than publishing a substituted value. +const NON_MONEY_UNIT_AFTER = + /^\s*(?:\([^)]*\)\s*)?(?:\p{L}+\s+)?(?:дни|дн\.|к\.\s?д\.|р\.\s?д\.|месец\p{L}*|години|год\.|броя|бр\.|кв\.?\s?м|куб\.?\s?м|тона|литра|%|процент\p{L}*)/iu; + +// #307 — the exact-2× "unchanged" restatement (rule 3) may only fire WITH a positive textual signal that +// the value did not really change: a currency re-denomination that mechanically doubled the figure, or an +// explicit "unchanged / non-material" phrasing. Absent any signal the row returns `none` and falls to the +// arithmetic annex_total_suspect flag (exclude), rather than silently halving a possibly-legitimate +// ЗОП чл.116 ал.1 т.1 in-scope +100% (a pre-announced option clause `outsideZop` cannot model). +// #307 — the anchor is the "X в евро" re-denomination phrasing (`лев… в евро`), NOT a bare "в евро": a +// payment-currency clause ("Плащанията…се извършват в евро…") says nothing about an unchanged total and +// would silently halve a real doubling. The bare form was also redundant — the 189325 fixture +// ("…се променя от лева в евро") is already caught by the `лев… в евро` alternative. +const RESTATE_UNCHANGED_CTX = + /(?:лев\p{L}*\s+в\s+евро|деноминаци\p{L}*|не\s*се\s+промен\p{L}*|остава\p{L}*\s+непромен\p{L}*|без\s+промяна|несъществен\p{L}*)/iu; + +function normalizeBgNumber(raw: string): number | null { + let t = raw.replace(WS, ''); + // #305 — when BOTH '.' and ',' appear the number uses one as a thousands separator and the other as the + // decimal mark (BG "1.234,56" or US "1,234.56"). The LAST-occurring separator is the decimal; strip the + // other (thousands) and normalise the decimal to '.'. Single-separator numbers keep the existing + // ≤2-fraction-digit convention (the space-thousands corpus: "539 240.00", "286 694,00"). + if (t.includes('.') && t.includes(',')) { + const decimalChar = t.lastIndexOf('.') > t.lastIndexOf(',') ? '.' : ','; + const thousandsChar = decimalChar === '.' ? ',' : '.'; + t = t.split(thousandsChar).join(''); + if (decimalChar === ',') t = t.replace(',', '.'); + } + const m = t.match(/^(\d+)(?:[.,](\d{1,2}))?$/); + if (!m) return null; + const value = Number(m[2] ? `${m[1]}.${m[2]}` : m[1]); + return Number.isFinite(value) && value > 0 ? value : null; +} + +function approxEq(a: number, b: number): boolean { + return Math.abs(a - b) <= REL_TOL * Math.max(Math.abs(a), Math.abs(b)); +} + +// Does a figure ≈ `target` occur in `text` with the ~40 preceding chars matching `contextRe` and (when +// given) NOT matching `excludeRe`? Returns true on the first qualifying occurrence. +function figureInContext( + text: string, + target: number, + contextRe: RegExp, + excludeRe: RegExp | null, + wideVetoRe: RegExp | null = null, + requireMoneyAnchor = false, +): boolean { + NUMBER_RE.lastIndex = 0; + let m: RegExpExecArray | null; + while ((m = NUMBER_RE.exec(text)) !== null) { + const n = normalizeBgNumber(m[0]); + if (n === null || !approxEq(n, target)) continue; + const before = text.slice(Math.max(0, m.index - 40), m.index); + if (excludeRe && excludeRe.test(before)) continue; + // A wider veto looks further back (~55 chars) for "в размер"/"ресурс"/increment markers that make an + // "…на " an amount, not a total. + if (wideVetoRe && wideVetoRe.test(text.slice(Math.max(0, m.index - 55), m.index))) continue; + if (!contextRe.test(before)) continue; + // #307 — a total needs a monetary marker bracketing the figure, else a bare "…на " matches a + // non-monetary number (days, article nos.) that coincidentally ≈ the target. A value keyword right + // before, OR a currency unit within the ~60 chars after, qualifies. + if (requireMoneyAnchor) { + const after = text.slice(m.index + m[0].length, m.index + m[0].length + 60); + // A non-monetary unit immediately after the figure (days/months/years/count/%) vetoes it before + // a downstream currency token can wrongly anchor it as money (#307). + if (NON_MONEY_UNIT_AFTER.test(after)) continue; + if (!MONEY_BEFORE.test(before) && !MONEY_AFTER.test(after)) continue; + } + return true; + } + return false; +} + +export function classifyAmendmentValue(input: AmendmentValueInput): AmendmentValueTreatment { + const b = input.valueBefore; + const a = input.valueAfter; + const d = input.valueDelta; + if (b === null || a === null || d === null || b <= 0 || a <= 0 || d <= 0) return { kind: 'none' }; + // Source self-consistency (a = b + d) is the precondition of the defect model. + if (!approxEq(a, b + d)) return { kind: 'none' }; + // A single annex whose "increment" is at least the whole prior value (2b ≤ a < 10b). Below 2b is a + // normal increase; ≥10b is a mis-key handled by #299's annex_suspect. + if (a < 2 * b || a >= 10 * b) return { kind: 'none' }; + + const text = input.texts.filter((t): t is string => !!t && t.trim() !== '').join(' '); + + // 1) The delta figure appears as an INCREMENT ("с ") — the value is genuinely correct, don't + // touch it. Checked FIRST so an exact 2× that the text calls a real increase is not mis-restated. + if (text && figureInContext(text, d, INCREMENT_CTX, null)) return { kind: 'genuine_increment' }; + + // 2) The delta figure appears as a TOTAL ("на ", "възлиза на …", "обща стойност … "). + // The true value_after is the delta (the announced new total). TOTAL_VETO rejects "в размер на"/ + // "ресурс"/increment phrasings that name the change amount, not the contract total. A monetary anchor + // is REQUIRED (#307) so a bare "…на " over a non-monetary number (days, article nos.) is rejected. + if (text && figureInContext(text, d, TOTAL_CTX, NOT_TOTAL_CTX, TOTAL_VETO, true)) { + return { kind: 'total_restated', correctedAfter: d }; + } + + // 3) Exact 2× (value_delta ≈ value_before): the "difference" field just echoed the OLD value, so + // value_after = before + before double-counts an UNCHANGED value (currency re-denomination or a + // non-value administrative annex). This restatement to value_before is only SAFE with a positive text + // signal (#307): ЗОП чл.116 ал.1 т.1 permits a genuine in-scope +100% via a pre-announced option/review + // clause that `outsideZop` cannot see, so a text-free rewrite could silently HALVE a legitimate value. + // Require either a "value unchanged / re-denomination" phrasing (RESTATE_UNCHANGED_CTX) or the + // before-value itself announced as the new total. Absent any signal, return `none` and let the + // arithmetic annex_total_suspect flag EXCLUDE the row (an honest gap beats a silent corruption). + // Still skipped for outside-ЗОП exception contracts, where a real +100% is legal. + if (approxEq(a, 2 * b) && !input.outsideZop && text) { + if ( + RESTATE_UNCHANGED_CTX.test(text) || + figureInContext(text, b, TOTAL_CTX, NOT_TOTAL_CTX, TOTAL_VETO, true) + ) { + return { kind: 'unchanged_restated', correctedAfter: b }; + } + } + + return { kind: 'none' }; +} + +// The single value the ETL needs: the corrected value_after when the text confirms a double-count, else +// null (leave value_after as the source gave it). +export function restatedValueAfter(input: AmendmentValueInput): number | null { + const t = classifyAmendmentValue(input); + return t.kind === 'total_restated' || t.kind === 'unchanged_restated' ? t.correctedAfter : null; +} + +export function isGenuineIncrement(input: AmendmentValueInput): boolean { + return classifyAmendmentValue(input).kind === 'genuine_increment'; +} + +// Convenience for the ETL staging: the treatment label to store on the raw amendment row (NULL when no +// signal), and the corrected value_after (NULL unless a double-count was confirmed). A non-null treatment +// tells derive/normalize NOT to arithmetic-flag the row (it is either corrected or confirmed-genuine). +export function amendmentValueTreatment(input: AmendmentValueInput): { + treatment: 'total_restated' | 'unchanged_restated' | 'genuine_increment' | null; + restatedAfter: number | null; +} { + const t = classifyAmendmentValue(input); + return { + treatment: t.kind === 'none' ? null : t.kind, + restatedAfter: + t.kind === 'total_restated' || t.kind === 'unchanged_restated' ? t.correctedAfter : null, + }; +} diff --git a/packages/ingest/src/base.test.ts b/packages/ingest/src/base.test.ts index 48f5f3e5..cb577d20 100644 --- a/packages/ingest/src/base.test.ts +++ b/packages/ingest/src/base.test.ts @@ -12,6 +12,7 @@ import { toInt, toPeriodDate, toReal, + toSignedReal, } from './base'; const FIXED_NOW = new Date('2026-06-11T12:00:00Z'); @@ -111,6 +112,92 @@ describe('base EOP mapper', () => { expect(toPeriodDate('2025-06-01', FIXED_NOW)).toBe('2025-06-01'); expect(baseSqlLiteral('tenders', 'end_date', '2043-12-31')).toBe("'2043-12-31'"); }); + + // An annex can REDUCE a contract, and ЦАИС ЕОП publishes that as a negative + // contractValueDifference. Coercing it with toReal dropped the sign - and with it the whole row's + // delta - so every value-reducing annex silently lost its recorded change. + it('keeps a value-reducing annex delta negative instead of nulling it', () => { + const row = mapBaseRecord( + 'annexes', + { + uniqueProcurementNumber: '00224-2025-0005', + contractNumber: '212221', + publicationDate: '05.03.2026', + lastContractValue: '24837,96', + currentContractValue: '24492,80', + contractValueDifference: '-345,16', + }, + { day: '2026-03-05', fetchedAt: '2026-03-05T00:00:00Z' }, + ); + + expect(row?.value_before).toBe(24837.96); + expect(row?.value_after).toBe(24492.8); + expect(row?.value_delta).toBe(-345.16); + // The literal must stay a bare number, not a quoted string, or the staging INSERT changes type. + expect(baseSqlLiteral('annexes', 'value_delta', row?.value_delta)).toBe('-345.16'); + }); + + // #305 Tier-2: base.ts runs the validated основание-text heuristic (amendment-total.ts) for annexes and + // persists value_treatment + value_after_restated onto the raw row. A doubled value_after whose text + // announces the NEW TOTAL ("…на ") is restated to that true total; an untreated annex stays NULL. + it('populates value_treatment/value_after_restated for an annex whose text announces a new total', () => { + const row = mapBaseRecord( + 'annexes', + { + uniqueProcurementNumber: '00224-2025-0009', + contractNumber: '990001', + publicationDate: '05.03.2026', + lastContractValue: '442000', + currentContractValue: '981240', // doubled: source put the new TOTAL in the change field + contractValueDifference: '539240', + contractCurrency: 'BGN', + changeReason: 'Общата стойност на договора се променя на 539 240 лв.', + }, + { day: '2026-03-05', fetchedAt: '2026-03-05T00:00:00Z' }, + ); + + expect(row?.value_after).toBe(981240); // raw after left as the source gave it + expect(row?.value_treatment).toBe('total_restated'); + expect(row?.value_after_restated).toBe(539240); // the corrected true total + // The restated total must serialise as a bare number for the staging INSERT, not a quoted string. + expect(baseSqlLiteral('annexes', 'value_after_restated', row?.value_after_restated)).toBe( + '539240', + ); + }); + + it('leaves value_treatment/value_after_restated NULL for a >2× annex with no text total (not exact-2×)', () => { + // 2.5× (not exact 2×) and no announced total in the text ⇒ no confident signal ⇒ left to the flag. + const row = mapBaseRecord( + 'annexes', + { + uniqueProcurementNumber: '00224-2025-0010', + contractNumber: '990002', + publicationDate: '05.03.2026', + lastContractValue: '1000000', + currentContractValue: '2500000', + contractValueDifference: '1500000', + contractCurrency: 'BGN', + }, + { day: '2026-03-05', fetchedAt: '2026-03-05T00:00:00Z' }, + ); + + expect(row?.value_treatment).toBeNull(); + expect(row?.value_after_restated).toBeNull(); + }); + + it('coerces signed reals without loosening the magnitude-only fields', () => { + expect(toSignedReal('-345,16')).toBe(-345.16); + expect(toSignedReal('-1 234,56')).toBe(-1234.56); + expect(toSignedReal('5833333,33')).toBe(5833333.33); + expect(toSignedReal('0')).toBe(0); + expect(toSignedReal(null)).toBeNull(); + expect(toSignedReal('')).toBeNull(); + expect(toSignedReal('--5')).toBeNull(); + expect(toSignedReal('-abc')).toBeNull(); + expect(toSignedReal(-(MAX_PLAUSIBLE_VALUE + 1))).toBeNull(); + // Magnitude fields keep rejecting a minus sign: there a negative means corrupt input. + expect(toReal('-345,16')).toBeNull(); + }); }); describe('offline SQL literal hardening', () => { diff --git a/packages/ingest/src/base.ts b/packages/ingest/src/base.ts index a2c43c01..4020ee3b 100644 --- a/packages/ingest/src/base.ts +++ b/packages/ingest/src/base.ts @@ -1,5 +1,7 @@ // Base EOP plain-JSON adapter helpers. Pure and Worker-safe: no Node APIs. +import { amendmentValueTreatment } from './amendment-total.ts'; + export type BaseCategory = 'contracts' | 'tenders' | 'annexes'; export type BaseCoercionKind = | 'text' @@ -7,6 +9,7 @@ export type BaseCoercionKind = | 'real' | 'bool' | 'date' + | 'real_signed' | 'secured_inverse' | 'variants_enum'; export type BaseStagingValue = string | number | null; @@ -94,6 +97,19 @@ export function toReal(v: unknown): number | null { return Number.isFinite(n) && n >= 0 && n <= MAX_PLAUSIBLE_VALUE ? n : null; } +// contractValueDifference is the one published figure that is legitimately negative - an annex can +// reduce a contract, and ЦАИС ЕОП publishes that as e.g. "-345,16". Every other REAL column here is a +// magnitude (values, estimates, subcontracting), where a minus sign means corrupt input, so toReal +// keeps rejecting negatives rather than being loosened for all of them. +export function toSignedReal(v: unknown): number | null { + const s = clean(v); + if (s === null) return null; + const compact = s.replace(/\s/g, ''); + if (!compact.startsWith('-')) return toReal(compact); + const magnitude = toReal(compact.slice(1)); + return magnitude === null ? null : -magnitude; +} + export function toBool(v: unknown): number | null { const s = clean(v); if (s === null) return null; @@ -131,6 +147,7 @@ function toVariants(v: unknown): number | null { export function coerce(kind: BaseCoercionKind, v: unknown): BaseStagingValue { if (kind === 'int') return toInt(v); if (kind === 'real') return toReal(v); + if (kind === 'real_signed') return toSignedReal(v); if (kind === 'bool') return toBool(v); if (kind === 'date') return toISODate(v); if (kind === 'secured_inverse') return toSecuredFinancing(v); @@ -297,7 +314,7 @@ export const BASE_CATEGORIES: Record = { field('contract_date', 'contractDate', 'date'), field('value_before', 'lastContractValue', 'real'), field('value_after', 'currentContractValue', 'real'), - field('value_delta', 'contractValueDifference', 'real'), + field('value_delta', 'contractValueDifference', 'real_signed'), field('currency', 'contractCurrency', 'text'), field('contract_subject', 'contractSubject', 'text'), field('awarded_to_group', 'awardedToGroup', 'bool'), @@ -311,6 +328,10 @@ export const BASE_CATEGORIES: Record = { field('description', 'changeDescription', 'text'), field('reason', 'changeReason', 'text'), field('circumstances', 'changeReasonDescription', 'text'), + // #305 Tier-2 — computed from the основание text after the generic mapping (see mapBaseRecord), not + // read from a source key. key=null keeps the generic loop from touching them; mapBaseRecord sets them. + field('value_treatment', null, 'text'), + field('value_after_restated', null, 'real'), field('outside_zop', 'isExceptionContract', 'bool'), field('exemption_legal_basis', 'directAwardJustification', 'text'), field('correction_number', null, 'text'), @@ -364,9 +385,34 @@ export function mapBaseRecord( if (!cfg.keep(record)) return null; const row: BaseStagingRow = fixedValues(cat, meta); for (const f of cfg.fields) row[f.column] = f.key === null ? null : coerce(f.kind, record[f.key]); + // #305 Tier-2 — EOP annexes only: classify value_delta from the основание free text (the validated + // heuristic in amendment-total.ts) and persist the treatment label + corrected total onto the raw row. + // OCDS annexes never reach here (ocds.ts stages them, and they carry value_after = null anyway). + if (cat === 'annexes') { + const treatment = amendmentValueTreatment({ + valueBefore: numOrNull(row.value_before), + valueAfter: numOrNull(row.value_after), + valueDelta: numOrNull(row.value_delta), + currency: strOrNull(row.currency), + texts: [strOrNull(row.description), strOrNull(row.reason), strOrNull(row.circumstances)], + // #305 — outside-ЗОП exception contracts (isExceptionContract) are not bound by чл.116's +50% cap, + // so the text-free exact-2× restatement must not fire on them (see amendment-total.ts rule 3). + outsideZop: numOrNull(row.outside_zop) === 1, + }); + row.value_treatment = treatment.treatment; + row.value_after_restated = treatment.restatedAfter; + } return row; } +function numOrNull(v: BaseStagingValue | undefined): number | null { + return typeof v === 'number' ? v : null; +} + +function strOrNull(v: BaseStagingValue | undefined): string | null { + return typeof v === 'string' ? v : null; +} + // Hard ceiling on a single text literal's character length. EOP/registry text fields // (subjects, descriptions, names) are well under this; anything larger is corrupt or // hostile and is truncated rather than passed to sqlite (avoids SQLITE_TOOBIG / abuse of @@ -432,7 +478,7 @@ export function baseSqlLiteral( ): string { if (value === null || value === undefined) return 'NULL'; const kind = baseColumnKind(cat, column); - if (['int', 'real', 'bool', 'secured_inverse', 'variants_enum'].includes(kind)) { + if (['int', 'real', 'real_signed', 'bool', 'secured_inverse', 'variants_enum'].includes(kind)) { return String(value); } return escapeSqlText(String(value)); diff --git a/packages/ingest/src/fx.test.ts b/packages/ingest/src/fx.test.ts index b6470b36..7f8fdc2c 100644 --- a/packages/ingest/src/fx.test.ts +++ b/packages/ingest/src/fx.test.ts @@ -209,6 +209,65 @@ describe('loadFxRates', () => { await expect(loadFxRates(d1, { fetchedAt: FETCHED_AT, fetchFn })).rejects.toThrow(/HTTP 500/); }); + // Same defect class as the EOP reads: a body that is never read holds its stream open for the rest + // of the invocation. The 404 path is the most-travelled one here — Frankfurter answers 404 for any + // base currency it does not serve, and the loader deliberately continues past it. + it('releases the body of every response it walks away from', async () => { + const openBody = () => { + let cancelled = false; + const body = new ReadableStream({ + start(c) { + c.enqueue(new TextEncoder().encode('x')); + }, + cancel() { + cancelled = true; + }, + }); + return { body, cancelled: () => cancelled }; + }; + + // 404 — unsupported currency, the loader warns and moves on. + { + const { db, d1 } = fxDb(); + stageContract(db, 'USD', '2026-07-08'); + const b = openBody(); + const fetchFn = vi.fn( + async () => new Response(b.body, { status: 404 }), + ) as unknown as typeof fetch; + const summary = await loadFxRates(d1, { fetchedAt: FETCHED_AT, fetchFn }); + expect(summary.warnings.join(' ')).toMatch(/not served by frankfurter/); + expect(b.cancelled()).toBe(true); + } + + // Non-OK — the loader throws. + { + const { db, d1 } = fxDb(); + stageContract(db, 'USD', '2026-07-08'); + const b = openBody(); + const fetchFn = vi.fn( + async () => new Response(b.body, { status: 500 }), + ) as unknown as typeof fetch; + await expect(loadFxRates(d1, { fetchedAt: FETCHED_AT, fetchFn })).rejects.toThrow(/HTTP 500/); + expect(b.cancelled()).toBe(true); + } + + // Redirected to another host — the host pin throws. + { + const { db, d1 } = fxDb(); + stageContract(db, 'USD', '2026-07-08'); + const b = openBody(); + const fetchFn = vi.fn(async () => { + const res = new Response(b.body, { status: 200 }); + Object.defineProperty(res, 'url', { + value: 'https://evil.example/v1/2026-07-01..2026-07-08', + }); + return res; + }) as unknown as typeof fetch; + await expect(loadFxRates(d1, { fetchedAt: FETCHED_AT, fetchFn })).rejects.toThrow(); + expect(b.cancelled()).toBe(true); + } + }); + it('keeps successfully loaded currencies when another currency fails', async () => { const { db, d1 } = fxDb(); stageContract(db, 'USD', '2026-07-08'); diff --git a/packages/ingest/src/fx.ts b/packages/ingest/src/fx.ts index 7d78a4ba..f24f5323 100644 --- a/packages/ingest/src/fx.ts +++ b/packages/ingest/src/fx.ts @@ -4,6 +4,20 @@ // validation) live here once so the two implementations cannot drift; the Worker-native loader and // its coverage guard are D1-based and pure-fetch, safe for workerd (no Node APIs). +// A response body that is never read keeps its stream - and the connection behind it - open for the +// rest of the invocation; the collector is not a substitute. Deliberately NOT awaited: cancelling only +// needs to be INITIATED for the runtime to release the stream, and awaiting it would make the caller +// hostage to a cancel() that never settles. Kept private here rather than shared with apps/etl: this +// file is also loaded as raw TypeScript by the Node CLI (scripts/load-fx.mjs), where an extensionless +// relative import does not resolve and a `.ts` one needs allowImportingTsExtensions repo-wide. +function discardBody(res: Response): void { + try { + void res.body?.cancel().catch(() => {}); + } catch { + // Already consumed, locked, or errored - there is nothing left to release either way. + } +} + // Canonical host. The legacy api.frankfurter.app now 301-redirects here — with host-pinned // fetches (assertSameFinalHost) the legacy host would fail closed, so point at the target // directly. Same response shape; the /v1 prefix is required on the .dev host. @@ -220,15 +234,25 @@ export async function loadFxRates(db: D1Database, opts: LoadFxOptions): Promise< try { const url = fxSeriesUrl(gap.currency, start, end, api); const res = await fetchFn(url); - assertSameFinalHost(url, res.url); + try { + assertSameFinalHost(url, res.url); + } catch (err) { + discardBody(res); + throw err; + } if (res.status === 404) { // Frankfurter answers 404 for a base currency it does not serve — permanent, not // transient: warn and move on (CLI parity), never brick the cron on one odd currency. + // The body still has to be released — this `continue` is the most-travelled path here. + discardBody(res); load.status = 'unsupported'; summary.warnings.push(`currency ${gap.currency} not served by frankfurter`); continue; } - if (!res.ok) throw new Error(`HTTP ${res.status}`); + if (!res.ok) { + discardBody(res); + throw new Error(`HTTP ${res.status}`); + } const { rows, warnings } = parseFxSeries(await res.json(), gap.currency, `${start}..${end}`); summary.warnings.push(...warnings); await upsertFxRates(db, rows, opts.fetchedAt); diff --git a/packages/ingest/src/ocds.test.ts b/packages/ingest/src/ocds.test.ts index 014f65f9..fd8b15cd 100644 --- a/packages/ingest/src/ocds.test.ts +++ b/packages/ingest/src/ocds.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { classifyBucketKey, computeCatchupWindow, + fullDeriveIsSafe, releaseToAmendments, releaseToContracts, releaseToLots, @@ -288,6 +289,48 @@ describe('releaseToAmendments', () => { expect(rows).toHaveLength(1); expect(rows[0]?.contract_number).toBe('DOC-1'); }); + + // Issue #286: the УНП is absent from OCDS releases, so the amendment must carry tender.id (the EOP + // procedure id) for the ETL bridge to recover the УНП. And the release value is the pre-amendment + // value, so it is stored as value_before with a null value_after — an OCDS row must never masquerade + // as a fresh current_value. `unp` still holds the ocid here; rewriting it to the real УНП is the job + // of derive-amendments.sql / refresh-slice.sql, not the pure flattener's. + it('captures the tender.id bridge and records the release value as value_before (issue #286)', () => { + const rows = releaseToAmendments( + { + ...release, + tag: ['contractAmendment'], + tender: { ...release.tender, id: '425867' }, + contracts: [ + { + ...release.contracts![0]!, + id: '90029', + value: { amount: 21_602_081.98, currency: 'EUR' }, + amendments: [{ description: 'Анекс 1', rationale: 'ЗОП чл. 116' }], + }, + ], + }, + meta, + ); + + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + contract_number: '90029', + unp: 'ocds-bg-2026-000123', + tender_ext_id: '425867', + value_before: 21_602_081.98, + value_after: null, + value_delta: null, + currency: 'EUR', + description: 'Анекс 1', + }); + }); + + it('captures a null tender_ext_id when the release has no tender.id', () => { + const rows = releaseToAmendments({ ...release, tag: ['contractUpdate'] }, meta); + expect(rows).toHaveLength(1); + expect(rows[0]?.tender_ext_id).toBeNull(); + }); }); describe('OCDS enrichment mappers', () => { @@ -352,6 +395,27 @@ describe('bucket key and catchup helpers', () => { computeCatchupWindow({ maxLoadedDate: '2026-06-01', today: '2026-06-07', lookbackDays: 3 }), ).toEqual({ from: '2026-05-29', to: '2026-06-07' }); }); + + it('allows a full derive only when the window reaches the start of the feed', () => { + // Initial backfill: nothing to lose, any window is safe. + expect( + fullDeriveIsSafe({ windowFrom: '2026-06-10', feedStart: '2020-01-01', hasCorpus: false }), + ).toBe(true); + // Whole feed reloaded into staging — the rebuild is complete. + expect( + fullDeriveIsSafe({ windowFrom: '2020-01-01', feedStart: '2020-01-01', hasCorpus: true }), + ).toBe(true); + // A catch-up window over an existing corpus would drop everything before it. + expect( + fullDeriveIsSafe({ windowFrom: '2026-06-10', feedStart: '2020-01-01', hasCorpus: true }), + ).toBe(false); + }); + + it('rejects malformed days rather than silently allowing a full derive', () => { + expect(() => + fullDeriveIsSafe({ windowFrom: '10-06-2026', feedStart: '2020-01-01', hasCorpus: true }), + ).toThrow(/windowFrom/); + }); }); describe('splitSqlStatements', () => { diff --git a/packages/ingest/src/ocds.ts b/packages/ingest/src/ocds.ts index fb460cfa..131ba304 100644 --- a/packages/ingest/src/ocds.ts +++ b/packages/ingest/src/ocds.ts @@ -216,6 +216,11 @@ export interface AmendmentStagingRow { reason: string | null; circumstances: string | null; sme: string | null; + // The EOP procedure id (OCDS `tender.id`) — the bridge to the УНП. The OCID is a surrogate; the + // real УНП is not in the OCDS release, so the ETL recovers it via tender_ext_id → raw_tenders → unp + // in scripts/derive-amendments.sql (and scripts/refresh-slice.sql on the incremental path), mirroring + // the OCDS-lots bridge that lives in scripts/normalize-raw.sql. See issue #286. + tender_ext_id: string | null; } export interface PartyStagingRow { @@ -317,7 +322,7 @@ export function releaseToContracts(rel: OcdsRelease, meta: OcdsMeta): ContractSt dataset_variant: 'OCDS', seq_no: null, document_number: rel.id ?? null, - contract_number: c.id ?? null, + contract_number: clean(c.id), contract_date: dateOnly(c.dateSigned), published_at: ctx.published_at, unp: rel.ocid ?? null, @@ -363,7 +368,7 @@ export function releaseToAmendments(rel: OcdsRelease, meta: OcdsMeta): Amendment dataset_variant: 'OCDS', seq_no: null, document_number: rel.id ?? null, - contract_number: c.id ?? null, + contract_number: clean(c.id), contract_date: dateOnly(c.dateSigned), published_at: ctx.published_at, unp: rel.ocid ?? null, @@ -375,14 +380,23 @@ export function releaseToAmendments(rel: OcdsRelease, meta: OcdsMeta): Amendment contract_subject: c.title || sup.awardTitle || null, contractor_eik: sup.eik, contractor_name: sup.name, - value_before: null, - value_after: finiteNum(c.value?.amount), + // An OCDS contractAmendment/contractUpdate release carries the contract value as it stands in + // that release — measured against the EOP annex stream this is the value BEFORE the amendment, + // never a reliable "after" (issue #286). Record it as value_before with a null value_after so an + // OCDS row can never drive the derived current_value (derive-amendments.sql selects the latest + // non-null value_after); the authoritative after-value comes from the EOP annex. + value_before: finiteNum(c.value?.amount), + value_after: null, value_delta: null, currency: isoCurrency(c.value?.currency), description: amd?.description || null, reason: amd?.rationale || null, circumstances: null, sme: null, + // OCDS tender.id === the EOP procedure id; the bridge to the УНП (derive-amendments.sql, and + // refresh-slice.sql on the incremental path). The ocid stored in `unp` here is a surrogate that + // the ETL bridge rewrites to the real УНП. + tender_ext_id: clean(rel.tender?.id), }, ]; }); @@ -452,6 +466,27 @@ export function computeCatchupWindow({ return { from: from > today ? today : from, to: today }; } +/** + * A full derive rebuilds the domain from whatever the staging tables hold — `scripts/normalize-raw.sql` + * opens with `DELETE FROM contracts` — so every contract outside the loaded window is dropped. It is + * only sound when the window reaches back to the first day the feed is loaded from, or when there is + * no corpus yet (the initial backfill). A gap-aware catch-up window never does, which is why + * `--catchup` derives a slice. + */ +export function fullDeriveIsSafe({ + windowFrom, + feedStart, + hasCorpus, +}: { + windowFrom: string; + feedStart: string; + hasCorpus: boolean; +}): boolean { + validateDay(windowFrom, 'windowFrom'); + validateDay(feedStart, 'feedStart'); + return !hasCorpus || windowFrom <= feedStart; +} + export function daysInWindow(from: string, to: string): number { validateDay(from, 'from'); validateDay(to, 'to'); @@ -523,6 +558,7 @@ export const AMENDMENT_STAGING_COLS: (keyof AmendmentStagingRow)[] = [ 'reason', 'circumstances', 'sme', + 'tender_ext_id', ]; export const PARTY_STAGING_COLS: (keyof PartyStagingRow)[] = [ diff --git a/packages/ingest/src/refresh-full-clear.test.ts b/packages/ingest/src/refresh-full-clear.test.ts new file mode 100644 index 00000000..c90f9f4a --- /dev/null +++ b/packages/ingest/src/refresh-full-clear.test.ts @@ -0,0 +1,77 @@ +/// +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { fullClearTables } from './refresh'; + +// fullClearTables answers one question for scripts/import.mjs: which tables does a full derive empty, +// and therefore what does a partial window destroy? The guard that used to ask it named `contracts` +// alone while the clear had reached fourteen tables — so the tests that matter here are the ones that +// keep the answer tied to the SQL rather than to a copy of it. + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const normalizeRaw = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); + +describe('fullClearTables', () => { + it('reads the block out of the real normalize-raw.sql', () => { + const tables = fullClearTables(normalizeRaw); + // The base domain tables: whatever else the block grows, losing any of these is losing the corpus. + expect(tables).toEqual( + expect.arrayContaining(['contracts', 'lots', 'tenders', 'bidders', 'authorities']), + ); + }); + + it('stops before the per-run metadata resets further down the file', () => { + // normalize-raw.sql also clears data_freshness and pipeline_stats, which are rewritten every run. + // Counting them as corpus would make the guard refuse EVERY full derive, initial backfill included + // — a guard that always refuses gets deleted, so this boundary is load-bearing. + const tables = fullClearTables(normalizeRaw); + expect(tables).not.toContain('data_freshness'); + expect(tables).not.toContain('pipeline_stats'); + }); + + it('keeps the marker and the block adjacent', () => { + // If the marker is dropped or drifts away from the DELETEs, this returns [] and import.mjs throws + // rather than silently deciding the corpus is empty and letting the destructive path through. + expect(fullClearTables(normalizeRaw).length).toBeGreaterThanOrEqual(5); + }); + + it('takes only the marked block, and only unqualified deletes', () => { + const sql = [ + 'DELETE FROM before_the_marker;', + '-- @full-clear', + 'DROP TABLE IF EXISTS scratch;', + 'DELETE FROM contracts;', + "DELETE FROM lots WHERE id = 'x';", // scoped: not a full clear + 'DELETE FROM authorities;', + '', + 'DELETE FROM after_the_block;', + ].join('\n'); + expect(fullClearTables(sql)).toEqual(['contracts', 'authorities']); + }); + + it('sees a table however it is quoted', () => { + // `DELETE FROM "search_index";` is valid SQLite and reads as pure formatting. A bare-identifier + // matcher dropped it from the list, which silently reopened the data-loss hole: the guard would + // then wave through a corpus whose only populated table was the re-quoted one. + const sql = [ + '-- @full-clear', + 'DELETE FROM "search_index";', + 'DELETE FROM `flow_pairs`;', + 'DELETE FROM [home_totals];', + 'delete from contracts;', + '', + ].join('\n'); + expect(fullClearTables(sql)).toEqual([ + 'search_index', + 'flow_pairs', + 'home_totals', + 'contracts', + ]); + }); + + it('returns nothing when the marker is absent', () => { + expect(fullClearTables('DELETE FROM contracts;\nDELETE FROM lots;\n')).toEqual([]); + }); +}); diff --git a/packages/ingest/src/refresh-officials.test.ts b/packages/ingest/src/refresh-officials.test.ts index 55170ef2..2c0ebb8f 100644 --- a/packages/ingest/src/refresh-officials.test.ts +++ b/packages/ingest/src/refresh-officials.test.ts @@ -20,6 +20,9 @@ const migrations = [ '0000_init.sql', '0001_flow_pairs_bidder_index.sql', '0003_related_persons_foundation.sql', + // The officials batch now joins the Trade Register evidence seal (#279, ADR-0033) — without 0006 the + // real refresh-slice.sql this test executes cannot parse. + '0009_interest_link_evidence.sql', ].map((f) => resolve(root, 'packages/db/migrations', f)); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); @@ -67,6 +70,11 @@ INSERT INTO interest_links -- her — dropping the s.status='published' guard would collapse it behind an invisible self stake (false neg). ('il:ys','person:ЯНА|444','person:ЯНА','eik:444','444','ДЕЛТА ООД','exact_name_key','v1','C_hold','owns','private_ownership',0,'none',1,'2020','2023',1,20000,'2021','2021','held'), ('il:yf','person:ЯНА|444|family','person:ЯНА','eik:444','444','ДЕЛТА ООД','exact_name_key','v1','B_distinctive','related','family_ownership',1,'none',1,'2020','2023',1,20000,'2021','2021','published'); +-- Every link carries an evidence seal (#279, ADR-0033): the officials batch requires a publishing rung, +-- so a fixture without seals indexes NOBODY and every assertion below passes vacuously. Derived from +-- interest_links, so a row added later is sealed automatically. +INSERT INTO interest_link_evidence (link_key, evidence_kind, registry_role, matched_fact, lookup_date, rules_version, live_status) + SELECT link_key, 'document', 'owner', 'role:owner:CR_F_19_L', '2026-08-05', 'tr-rules-1', 'live' FROM interest_links; `; describe('ETL refresh-slice officials batch (the live parse-then-execute path)', () => { diff --git a/packages/ingest/src/refresh.ts b/packages/ingest/src/refresh.ts index 4159da90..0aed14ad 100644 --- a/packages/ingest/src/refresh.ts +++ b/packages/ingest/src/refresh.ts @@ -98,6 +98,12 @@ const LEGACY_TRANSIENT_STAGING_TABLES = [ 'raw_egov_amendments', ] as const; +// Scratch tables that live only for the span of a single derive step. Each is DROP-guarded at the top of +// its own script, so it self-heals on the next run; listing it here also sweeps it after an aborted run so +// it never lingers in D1 (review nikimilenkov LOW 2 — #306's value-resolver scratch table). Not part of +// work-staging-schema.sql, so it stays out of transientStagingStatements' recreate path. +const SCRATCH_TABLES = ['amendment_contract_resolve'] as const; + function touchesTransientStaging(statement: string): boolean { return TRANSIENT_STAGING_TABLES.some((table) => statement.includes(table)); } @@ -108,8 +114,47 @@ export function transientStagingStatements(workStagingSchemaSql: string): string ); } +const FULL_CLEAR_MARKER = /^--\s*@full-clear\b/i; +// All three SQLite quoting styles, not just the bare identifier. Rewriting one line as +// `DELETE FROM "search_index";` is a valid, invisible formatting change — and with a bare-only +// matcher it would drop that table out of the guard's list and quietly reopen the hole this parser +// exists to close. +const DELETE_FROM = + /^DELETE\s+FROM\s+(?:"([^"]+)"|`([^`]+)`|\[([^\]]+)\]|([A-Za-z_][A-Za-z0-9_]*))\s*;?\s*$/i; + +/** + * The tables `scripts/normalize-raw.sql` empties before rebuilding the domain from staging, read out + * of the SQL rather than restated in JS. The guard that consumes this list used to ask about + * `contracts` alone while the clear had grown to fourteen tables — a hardcoded copy of a destructive + * list is a data-loss bug on a timer, so the list has exactly one home. + * + * Scoped to the `@full-clear` block on purpose: the same file later resets `data_freshness` and + * `pipeline_stats`, which are per-run metadata. Counting those as corpus would make the guard refuse + * every full derive, including the initial backfill it is supposed to let through. + */ +export function fullClearTables(normalizeRawSql: string): string[] { + const tables: string[] = []; + let inBlock = false; + for (const line of normalizeRawSql.split(/\r?\n/)) { + const trimmed = line.trim(); + if (FULL_CLEAR_MARKER.test(trimmed)) { + inBlock = true; + continue; + } + if (!inBlock) continue; + const hit = trimmed.match(DELETE_FROM); + if (hit) { + tables.push((hit[1] ?? hit[2] ?? hit[3] ?? hit[4])!); + continue; + } + // Comments and the DROP TABLEs share the block; a blank line ends it. + if (trimmed === '') break; + } + return tables; +} + export function dropTransientStagingStatements(): string[] { - return [...TRANSIENT_STAGING_TABLES, ...LEGACY_TRANSIENT_STAGING_TABLES] + return [...SCRATCH_TABLES, ...TRANSIENT_STAGING_TABLES, ...LEGACY_TRANSIENT_STAGING_TABLES] .reverse() .map((table) => `DROP TABLE IF EXISTS ${table}`); } diff --git a/packages/ingest/tsconfig.json b/packages/ingest/tsconfig.json index b8ac7d61..9d4ac835 100644 --- a/packages/ingest/tsconfig.json +++ b/packages/ingest/tsconfig.json @@ -1,6 +1,8 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { + // base.ts imports './amendment-total.ts' with an explicit .ts extension (the ETL runtime consumes this + // source under plain Node). allowImportingTsExtensions is set in tsconfig.base.json for every consumer. "types": ["@cloudflare/workers-types"] }, "include": ["src"] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 304ec682..a8037bfb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,7 @@ overrides: sharp: ^0.35.3 postcss: ^8.5.18 valibot: ^1.4.2 + nanoid: ^3.3.17 importers: @@ -1643,8 +1644,8 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - nanoid@3.3.16: - resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} + nanoid@3.3.18: + resolution: {integrity: sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -3369,7 +3370,7 @@ snapshots: ms@2.1.3: {} - nanoid@3.3.16: {} + nanoid@3.3.18: {} node-releases@2.0.45: {} @@ -3406,7 +3407,7 @@ snapshots: postcss@8.5.24: dependencies: - nanoid: 3.3.16 + nanoid: 3.3.18 picocolors: 1.1.1 source-map-js: 1.2.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 200559f1..87ca4b8e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -38,6 +38,10 @@ overrides: # valibot <1.4.2 — GHSA-5qjj-4xww-7phc (MEDIUM); transitive, build-time only, never # ships to the Worker. valibot: '^1.4.2' + # nanoid <3.3.17 — GHSA-2v37-7h3g-55p8 (HIGH, 8.2); pulled in by postcss (itself pinned + # above), so postcss 8.5.24 keeps resolving 3.3.16 on its own. Build-time + # only, via vite/vitest — never ships to the Worker. + nanoid: '^3.3.17' onlyBuiltDependencies: - esbuild diff --git a/scripts/cacbg/audit.mjs b/scripts/cacbg/audit.mjs index 50ff2933..6ca5f3a7 100644 --- a/scripts/cacbg/audit.mjs +++ b/scripts/cacbg/audit.mjs @@ -7,11 +7,14 @@ import { DatabaseSync } from 'node:sqlite'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { nameDistinctiveness } from './classify.mjs'; +import fs from 'node:fs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; +import { RULES_VERSION, isSealedFact } from '../tr/evidence.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); +const STAGING = process.env.CACBG_STAGING || path.join(ROOT, 'scratch/cacbg/staging'); +const SNAPSHOT = path.join(STAGING, 'published-snapshot.json'); const { companyNameKey } = await import('../../packages/shared/src/company-name-key.ts'); const db = new DatabaseSync(DB, { readOnly: true }); @@ -31,12 +34,34 @@ const published = db ` SELECT il.id, il.link_key, il.person_id, il.eik, il.entity_key, il.match_method, il.publish_tier, il.bidder_id, il.relation, il.contemporaneous, il.contract_value_eur, - b.name AS bidder_name, b.eik_normalized AS bidder_eik, b.eik_valid AS bidder_eik_valid + b.name AS bidder_name, b.eik_normalized AS bidder_eik, b.eik_valid AS bidder_eik_valid, + -- LEFT JOIN, deliberately: a published link with NO seal is the finding, so it must reach the + -- loop rather than be filtered out of it. + e.evidence_kind, e.registry_role, e.matched_fact FROM interest_links il JOIN bidders b ON b.id = il.bidder_id + LEFT JOIN interest_link_evidence e ON e.link_key = il.link_key WHERE il.status = 'published'`, ) .all(); +// Every link's CURRENT status, for the monotonicity gate below. A link that left the published set +// because it was SUPPRESSED is still in this table — the takedown is a status flip, not a delete — and +// distinguishing that from a link that simply stopped being built is the difference between a declared +// removal and a silent recall regression. Read here, while the handle is open. +const statusNow = new Map( + db + .prepare('SELECT link_key, status FROM interest_links') + .all() + .map((r) => [r.link_key, r.status]), +); + +// The only two evidence rungs that publish (ADR-0033 decision 1). Everything else withholds. +const PUBLISHING_EVIDENCE = new Set(['document', 'confirmed']); +// The closed vocabulary for a sealed matched_fact lives in evidence.mjs, next to the code that WRITES +// it — one definition, so the gate can never permit a shape the writer has stopped emitting (or, worse, +// the other way round). `isSealedFact` bounds the seat leg to a settlement's one or two tokens, which is +// what stops a three-part Bulgarian name riding through behind the legitimate `seat:` prefix. + const findings = []; const flag = (link, axis, detail) => findings.push({ axis, link_key: link.link_key, eik: link.eik, detail }); @@ -50,12 +75,16 @@ for (const l of published) { // l.eik is a valid winner BEARING this name key (a stray/mis-attached ЕИК is still caught). The ЕИК+name // double-lock itself is re-proven in the provenance pass below (A_eik_no_provenance). if (!rec) flag(l, 'A_key_missing', `entity_key ${l.entity_key} not found in live bidder set`); - else if (l.publish_tier === 'A_eik') { + else if (l.match_method === 'declared_eik') { + // Identity here is the declarant-provided ЕИК, not the name (ADR-0028), so a name shared by more + // than one winner is legitimate — the ЕИК picks exactly one. Require instead that l.eik is a valid + // winner BEARING this name key, which still catches a stray or mis-attached ЕИК. The ЕИК+name + // double-lock is re-proven independently in the provenance pass below. if (!rec.valid.has(l.eik)) flag( l, 'A_eik_not_winner', - `A_eik published ${l.eik}, not among key ${l.entity_key}'s valid winners {${[...rec.valid].join(',')}}`, + `declared_eik published ${l.eik}, not among key ${l.entity_key}'s valid winners {${[...rec.valid].join(',')}}`, ); } else if (rec.valid.size !== 1) flag( @@ -72,12 +101,34 @@ for (const l of published) { if (!l.bidder_eik_valid) flag(l, 'B_eik_invalid', `published on eik_valid=0 bidder ${l.bidder_name}`); - // C. Tier honesty: B_distinctive must actually be distinctive by the same classifier that gated it. - if (l.publish_tier === 'B_distinctive' && nameDistinctiveness(l.entity_key) !== 'distinctive') + // C. Evidence honesty (#279, ADR-0033). The tier IS the evidence kind now, so the axis that used to + // re-derive name distinctiveness re-derives the publishing rule instead: a published link must + // carry a seal, and that seal must be one of the two rungs that publish. `nameDistinctiveness` + // no longer gates publication at all — it survives only as a withholding filter inside the + // loader — so re-checking it here would assert a rule that is no longer in force. + if (!l.evidence_kind) + flag(l, 'C_no_evidence', `published with no Trade Register evidence seal (${l.bidder_name})`); + else if (!PUBLISHING_EVIDENCE.has(l.evidence_kind)) + flag( + l, + 'C_withholding_evidence', + `published on evidence_kind='${l.evidence_kind}', which withholds (${l.bidder_name})`, + ); + else if (l.evidence_kind !== l.publish_tier) + flag( + l, + 'C_tier_evidence_mismatch', + `publish_tier='${l.publish_tier}' but sealed evidence_kind='${l.evidence_kind}'`, + ); + + // C2. The PII rail, audited rather than assumed: matched_fact is a CLOSED vocabulary and can never + // carry a name. The registry deed's names are read only to produce a boolean and must never reach + // a served column (#279 §9, ADR-0033 decision 5). A schema cannot enforce this; this does. + if (!isSealedFact(l.matched_fact)) flag( l, - 'C_not_distinctive', - `tier B_distinctive but nameDistinctiveness=${nameDistinctiveness(l.entity_key)} (${l.bidder_name})`, + 'C_matched_fact_shape', + `matched_fact='${l.matched_fact}' is outside the closed vocabulary — a name may have leaked`, ); } @@ -126,6 +177,63 @@ for (const l of nonExact) { db.close(); +// D. Monotonicity — ADR-0033 decision 6, the correction of #279 §8. +// +// §8 asked for a seal kept „forever" and strictly-additive recomputation. That is unachievable: labels +// flip, the deed cache expires, and a court can annul an entry (чл. 29 ЗТРРЮЛНЦ) with no rules change +// at all. So the seal is NOT a store — it is re-derived every run — and monotonicity is enforced HERE, +// as a gate, against the export load.mjs writes immediately before it drops the CACBG tables. +// +// The rule: a link that was published last run and is not published now is a REGRESSION unless the +// rules themselves changed. Under an unchanged rules_version nothing licensed the removal, so it is a +// hard finding. Under a changed one it is an intentional event and degrades to a printed diff. +// +// ship-related-persons.mjs's count floor cannot do this job: it compares a COUNT, so a one-for-one +// swap — one true link silently dropped, one gained — leaves it perfectly quiet. +const publishedNow = new Set(published.map((l) => l.link_key)); +let priorPublished = null; +try { + priorPublished = JSON.parse(fs.readFileSync(SNAPSHOT, 'utf8')); +} catch (e) { + // ENOENT is the legitimate first run — there is no prior surface to regress from. Anything else + // (unreadable, malformed) must not be swallowed into a silent pass of the gate. + if (e.code !== 'ENOENT') throw e; +} +const vanished = (priorPublished ?? []).filter((p) => !publishedNow.has(p.link_key)); + +// The three grounds decision 6 sanctions for a removal. Anything else that vanished is a regression. +// +// Getting this set right is what stops the gate from deadlocking the mechanisms it points at. The +// rules bump is the obvious one. The other two are not optional extras — each is the ONLY expressible +// form of a removal the ADR names in prose: +// +// suppressed — the ADR-0031 takedown path, which decision 6 wires the court-annulled entry +// (чл. 29 ЗТРРЮЛНЦ) to by name. It flips status published → suppressed with rules_version +// untouched, so without this branch the one removal the ADR explicitly licenses hard-fails. +// corrected — "a correction of wrong input". Suppression cannot express it: correcting the input +// unbuilds the link, and load.mjs's B3 unused-suppression gate then fails the build for a +// fingerprint that matched nothing. So the two sanctioned removals would fail in OPPOSITE +// directions, leaving a real correction with no path at all. The flag is set by load.mjs from the +// version-controlled, fingerprinted corrections list — a human decision recorded in git, not a +// condition the audit can infer. +// +// Both are declared, both are reviewed, and both are still PRINTED below: a withdrawn public claim is +// never silent, it just is not a build failure. +const declaredRemoval = (p) => + p.rules_version !== RULES_VERSION || + p.corrected === true || + statusNow.get(p.link_key) === 'suppressed'; +const regressions = vanished.filter((p) => !declaredRemoval(p)); +for (const p of regressions) + findings.push({ + axis: 'D_monotonicity', + link_key: p.link_key, + eik: p.link_key.split('|')[1] ?? '', + // The link_key is named explicitly: it is the only handle a human has to go and look at which + // claim disappeared, and the shared axis report prints the ЕИК alone. + detail: `${p.link_key} published last run under rules_version ${p.rules_version} (unchanged) and is not published now — nothing licensed this removal`, + }); + // Report const byAxis = {}; for (const f of findings) (byAxis[f.axis] ??= []).push(f); @@ -137,6 +245,29 @@ for (const [axis, fs] of Object.entries(byAxis)) { for (const f of fs.slice(0, 20)) console.log(` - [${f.eik}] ${f.detail}`); console.log(''); } +if (priorPublished === null) { + console.log(`## monotonicity — no prior export at ${SNAPSHOT}; treating this as a first run\n`); +} else { + const declared = vanished.filter(declaredRemoval); + console.log( + `## monotonicity — ${priorPublished.length} published last run, ${publishedNow.size} now; ` + + `${regressions.length} regression(s), ${declared.length} declared removal(s)\n`, + ); + // Declared removals are not findings, but they are never silent: withdrawing a named public claim is + // exactly when a human should read WHICH claims went and on what ground. The ground is printed too — + // "removed" alone would flatten a court annulment, a corrected parse and a rules bump into one line. + for (const p of declared) { + const ground = + p.rules_version !== RULES_VERSION + ? `a rules change (${p.rules_version} → ${RULES_VERSION})` + : statusNow.get(p.link_key) === 'suppressed' + ? 'a suppression (ADR-0031 takedown path)' + : 'an acknowledged input correction'; + console.log(` - ${p.link_key} removed under ${ground}`); + } + if (declared.length) console.log(''); +} + console.log(`## non-exact provenance (${provenance.length}) — verify each cross-check by eye`); for (const p of provenance) { console.log(` - ${p.method} [${p.eik}] winner="${p.winner}"`); diff --git a/scripts/cacbg/audit.test.mjs b/scripts/cacbg/audit.test.mjs index df36baa8..e0f8af4b 100644 --- a/scripts/cacbg/audit.test.mjs +++ b/scripts/cacbg/audit.test.mjs @@ -26,10 +26,15 @@ const dirs = []; // Build a fixture DB (bidders + declarations + declared_interests + interest_links), run audit.mjs against // it as a subprocess, and return { threw, out } — threw=true iff the audit exited non-zero (a hard finding). -function buildAndAudit({ bidders, decls = [], dis = [], links }) { +function buildAndAudit({ bidders, decls = [], dis = [], links, seals = [], snapshot = null }) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-audit-')); dirs.push(dir); const DB = path.join(dir, 'fixture.sqlite'); + const staging = path.join(dir, 'staging'); + fs.mkdirSync(staging, { recursive: true }); + // The pre-wipe export load.mjs writes before it drops the CACBG tables. Absent === a first run. + if (snapshot) + fs.writeFileSync(path.join(staging, 'published-snapshot.json'), JSON.stringify(snapshot)); const db = new DatabaseSync(DB); db.exec(` CREATE TABLE bidders(id TEXT PRIMARY KEY, name TEXT, eik_normalized TEXT, eik_valid INT); @@ -38,10 +43,16 @@ function buildAndAudit({ bidders, decls = [], dis = [], links }) { CREATE TABLE interest_links( id TEXT PRIMARY KEY, link_key TEXT, person_id TEXT, eik TEXT, entity_key TEXT, match_method TEXT, publish_tier TEXT, bidder_id TEXT, relation TEXT, contemporaneous INT, contract_value_eur REAL, status TEXT); + -- The evidence seal (#279, migration 0006). The audit LEFT JOINs it, so it must exist even when a + -- case deliberately leaves a link unsealed — an unsealed published link is itself a finding. + CREATE TABLE interest_link_evidence( + link_key TEXT PRIMARY KEY, evidence_kind TEXT, registry_role TEXT, matched_fact TEXT, + entry_number TEXT, entry_date TEXT, lookup_date TEXT, rules_version TEXT, live_status TEXT); ${bidders.map((b) => `INSERT INTO bidders VALUES (${b});`).join('\n')} ${decls.map((d) => `INSERT INTO declarations VALUES (${d});`).join('\n')} ${dis.map((d) => `INSERT INTO declared_interests(declaration_id, entity_raw) VALUES (${d});`).join('\n')} ${links.map((l) => `INSERT INTO interest_links VALUES (${l});`).join('\n')} + ${seals.map((e) => `INSERT INTO interest_link_evidence(link_key,evidence_kind,registry_role,matched_fact,lookup_date,rules_version,live_status) VALUES (${e});`).join('\n')} `); db.close(); @@ -51,7 +62,12 @@ function buildAndAudit({ bidders, decls = [], dis = [], links }) { out = execFileSync( 'node', ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'audit.mjs')], - { cwd: ROOT, env: { ...process.env, CACBG_DB: DB }, encoding: 'utf8', stdio: 'pipe' }, + { + cwd: ROOT, + env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: staging }, + encoding: 'utf8', + stdio: 'pipe', + }, ); } catch (e) { threw = true; // execFileSync throws on non-zero exit (a hard finding fired) @@ -73,8 +89,9 @@ test('A_eik behind a colliding name, backed by a real ЕИК+name double-lock, P // the declarant wrote BOTH the ЕИК and the фирма → the double-lock the loader required dis: [`'d1','„ОБЩ" ЕООД, ЕИК 100000001'`], links: [ - `'il1','p1|100000001','p1','100000001','${KEY}','declared_eik','A_eik','b1','owns',0,1000,'published'`, + `'il1','p1|100000001','p1','100000001','${KEY}','declared_eik','confirmed','b1','owns',0,1000,'published'`, ], + seals: [`'p1|100000001','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], }); assert.equal( threw, @@ -93,8 +110,9 @@ test('A_eik published on a ЕИК that is NOT a winner bearing the name key → dis: [`'d1','„ОБЩ" ЕООД, ЕИК 300000003'`], // entity_key claims the colliding name, but the published eik/bidder is the unrelated 300000003 links: [ - `'il1','p1|300000003','p1','300000003','${KEY}','declared_eik','A_eik','b3','owns',0,1000,'published'`, + `'il1','p1|300000003','p1','300000003','${KEY}','declared_eik','confirmed','b3','owns',0,1000,'published'`, ], + seals: [`'p1|300000003','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], }); assert.equal(threw, true, 'a stray-ЕИК A_eik link must fail the gate'); assert.equal( @@ -110,8 +128,9 @@ test('A_eik with no declaration carrying its ЕИК+name → A_eik_no_provenance decls: [`'d1','p1'`], dis: [`'d1','нещо съвсем друго без ЕИК'`], // no ЕИК, no фирма → the double-lock cannot be re-proven links: [ - `'il1','p1|100000001','p1','100000001','${K('„ОБЩ" ЕООД')}','declared_eik','A_eik','b1','owns',0,1000,'published'`, + `'il1','p1|100000001','p1','100000001','${K('„ОБЩ" ЕООД')}','declared_eik','confirmed','b1','owns',0,1000,'published'`, ], + seals: [`'p1|100000001','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], }); assert.equal( threw, @@ -130,9 +149,325 @@ test('a NON-A_eik (name-based) colliding link STILL fails A_multi_eik — the na bidders: COLLIDING_BIDDERS, // exact_name_key published on a name that maps to 2 ЕИК — exactly what the gate must keep rejecting links: [ - `'il1','p1|100000001','p1','100000001','${KEY}','exact_name_key','B_distinctive','b1','owns',0,1000,'published'`, + `'il1','p1|100000001','p1','100000001','${KEY}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'`, ], }); assert.equal(threw, true, 'a name-based colliding published link must still fail'); assert.equal(/A_multi_eik/.test(out), true, 'invariant A still fires for non-A_eik links'); }); + +// ── C: evidence honesty (#279, ADR-0033) ───────────────────────────────────────────────────────── +// The axis that used to re-derive name distinctiveness now re-derives the PUBLISHING rule, because the +// publishing rule changed: identity rests on a registry fact, and nameDistinctiveness survives only as +// a withholding filter inside the loader. Re-checking distinctiveness here would assert a rule that is +// no longer in force — a test that passes while guarding nothing. +test('a published link with NO evidence seal is a hard finding', () => { + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [], // the whole point: nothing sealed it + }); + assert.equal(threw, true, 'publishing without evidence must fail the gate'); + assert.equal(/C_no_evidence/.test(out), true, out); +}); + +test('a link published on a WITHHOLDING rung is a hard finding', () => { + // bar_joint_stock / unknown / outside_tr never publish. If one ever reaches status='published', the + // loader and the seal disagree and a claim is on the surface that no evidence supports. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','unknown','b1','owns',0,1000,'published'`, + ], + seals: [`'p1|100000001','unknown',NULL,NULL,'2026-08-05','tr-rules-1','live'`], + }); + assert.equal(threw, true); + assert.equal(/C_withholding_evidence/.test(out), true, out); +}); + +test('a tier that disagrees with its own seal is a hard finding', () => { + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [`'p1|100000001','confirmed',NULL,'eik','2026-08-05','tr-rules-1','live'`], + }); + assert.equal(threw, true); + assert.equal(/C_tier_evidence_mismatch/.test(out), true, out); +}); + +test('a matched_fact outside the closed vocabulary is a hard finding — the name-leak rail', () => { + // #279 §9: the deed's names are read only to produce a boolean and must never reach a served column. + // A schema cannot enforce a vocabulary, so the audit does. This is the shape a leak would take. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','ИВАН ПЕТРОВ ТЕСТОВ','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'a name in matched_fact must fail the gate'); + assert.equal(/C_matched_fact_shape/.test(out), true, out); +}); + +// …and the rail must catch the shape a leak would ACTUALLY take. `seat:` is a legitimate member +// of the vocabulary, so an unbounded `seat:[\p{Lu} -]+` admits `seat:ИВАН ПЕТРОВ ГЕОРГИЕВ` — a full +// three-part Bulgarian name (ЗГР чл. 9) wearing the prefix of a fact we allow. That is precisely the +// value the rail exists to reject, and it is the one a mis-split of the seat field would produce. +// A Bulgarian settlement is one or two tokens („СОФИЯ", „ВЕЛИКО ТЪРНОВО", „ГЕНЕРАЛ ТОШЕВО"); the +// three-part name is exactly three. The bound is deliberately tight: a rarer 3-token seat trips the +// audit and a human adjudicates, which is the correct direction for a rail whose failure mode is +// publishing somebody's name. +test('a THREE-TOKEN seat is a name shape, not a settlement — the rail must catch it', () => { + const { threw, out } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','confirmed','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','confirmed',NULL,'seat:ИВАН ПЕТРОВ ГЕОРГИЕВ','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'a three-part name behind seat: must fail the gate'); + assert.equal(/C_matched_fact_shape/.test(out), true, out); +}); + +test('a real two-token settlement still passes — the bound must not empty the seat rung', () => { + const { threw } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','confirmed','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','confirmed',NULL,'seat:ВЕЛИКО ТЪРНОВО','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, false, 'ВЕЛИКО ТЪРНОВО is a settlement and must survive the rail'); +}); + +test('a well-formed seal passes every C axis (positive control)', () => { + // Without this, all four negatives above would still pass if the axes fired unconditionally. + const { threw } = buildAndAudit({ + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'`, + ], + }); + assert.equal(threw, false, 'a correctly sealed link must not fail the gate'); +}); + +// ── D. Monotonicity (ADR-0033 decision 6) ──────────────────────────────────────────────────────── +// #279 §8 asked for a seal kept „forever" and strictly-additive recomputation. ADR-0033 showed that is +// false (labels flip, the cache expires, a court can annul an entry) and replaced the STORE with a +// GATE: seals are re-derived every run, and the audit compares the current published set against the +// pre-wipe export. A link that vanishes under an UNCHANGED rules_version is a hard finding — nothing +// about the rules changed, so its disappearance is a regression, not a decision. +// +// Why this needs its own gate and ship's count floor cannot serve: assertShipFloor checks a COUNT. +// A one-for-one swap — one link lost, one gained — leaves the count identical and the floor silent +// while a true published link is dropped. Only a per-key comparison sees it. +const SEALED = [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'`, +]; +const ONE_LINK = { + bidders: [`'b1','УНИК ТЕХ 7 ЕООД','100000001',1`, `'b2','ВТОРА ФИРМА ЕООД','200000002',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('УНИК ТЕХ 7 ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: SEALED, +}; + +test('D — a previously published link that vanished under an UNCHANGED rules_version is a hard finding', () => { + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, // published last run, gone now + ], + }); + assert.equal(threw, true, 'a silent recall regression must fail the build'); + assert.match(out, /D_monotonicity/); + assert.match(out, /p9\|200000002/); +}); + +test('D — the same disappearance under a CHANGED rules_version is a printed diff, not a finding', () => { + // Removal stays an intentional event: bumping the rules version is how you declare one. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [{ link_key: 'p9|200000002', rules_version: 'tr-rules-0' }], + }); + assert.equal(threw, false, 'a declared rules change must not fail the build'); + assert.match(out, /p9\|200000002/, 'but it must still be reported'); +}); + +test('D — no snapshot (a first run) neither fires nor crashes', () => { + const { threw } = buildAndAudit(ONE_LINK); + assert.equal(threw, false); +}); + +// The gate must also let the two removals ADR-0033 decision 6 actually SANCTIONS through, or it +// deadlocks the mechanisms it points at. Both are declared events with a reviewed paper trail in git; +// neither is a silent recall regression, and the gate exists to tell those apart. + +test('D — a link taken down through the ADR-0031 suppression path is a declared removal, not a regression', () => { + // Decision 6 names the court-annulled entry (чл. 29 ЗТРРЮЛНЦ) and wires it to ADR-0031. That path + // flips status published → suppressed, so the link leaves the published set with rules_version + // unchanged — firing the gate on the ONE removal the ADR explicitly licenses. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + links: [ + ...ONE_LINK.links, + `'il2','p9|200000002','p9','200000002','${K('ВТОРА ФИРМА ЕООД')}','exact_name_key','document','b2','owns',0,1000,'suppressed'`, + ], + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, + ], + }); + assert.equal(threw, false, 'the sanctioned takedown path must not fail the build'); + assert.match(out, /p9\|200000002/, 'but a withdrawn public claim is never silent'); +}); + +test('D — a snapshot entry acknowledged as a corrected input is a declared removal', () => { + // The other sanctioned ground: the link should never have been published because its INPUT was + // wrong. Suppression cannot express it — correcting the input unbuilds the link, and load.mjs's B3 + // gate then fails the build for a suppression that matched nothing. Without this the two sanctioned + // removals fail in opposite directions and there is no way to clear either. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1', corrected: true }, + ], + }); + assert.equal(threw, false, 'an acknowledged correction must not fail the build'); + assert.match(out, /p9\|200000002/); +}); + +test('D — neither escape hatch fires on its own: an unacknowledged, unsuppressed drop still hard-fails', () => { + // The mutation control for the two tests above. A gate that accepted every disappearance would pass + // both of them, and this is the assertion that says it did not. + const { threw } = buildAndAudit({ + ...ONE_LINK, + links: [ + ...ONE_LINK.links, + `'il2','p9|200000002','p9','200000002','${K('ВТОРА ФИРМА ЕООД')}','exact_name_key','document','b2','owns',0,1000,'held'`, + ], + snapshot: [ + { link_key: 'p1|100000001', rules_version: 'tr-rules-1' }, + { link_key: 'p9|200000002', rules_version: 'tr-rules-1' }, + ], + }); + assert.equal( + threw, + true, + 'held is not a sanctioned removal — the evidence simply stopped licensing it', + ); +}); + +test('D positive control — an unchanged published set produces no monotonicity finding', () => { + // Without this, a gate that never fires would pass both negatives above. + const { threw, out } = buildAndAudit({ + ...ONE_LINK, + snapshot: [{ link_key: 'p1|100000001', rules_version: 'tr-rules-1' }], + }); + assert.equal(threw, false); + assert.doesNotMatch(out, /D_monotonicity/); +}); + +// The four axes below fire on shapes that no test exercised. Each is a hard finding — the audit is the +// last gate before a named claim ships — so an axis that silently stopped firing would be invisible. + +test('A_key_missing: a link whose entity_key is in NO live bidder → hard finding', () => { + // The key resolved when the link was built and does not now: the winner was renamed, re-keyed or + // dropped from the corpus. The link still names an official against a company we can no longer find, + // so it must stop the run rather than ship pointing at nothing. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('ИЗЧЕЗНАЛ ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'an unresolvable entity_key must fail the gate'); + assert.equal(/A_key_missing/.test(out), true); +}); + +test('A_eik_mismatch: a name-resolved key pointing at a DIFFERENT ЕИК than it resolves to → hard', () => { + // The libel case in its purest form: the name resolves to exactly one winner, and the link published a + // different company against it. Everything on the card — contracts, money, the ЕИК link — would be the + // wrong company's, under a real official's name. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`, `'b2','ДРУГ ЕООД','200000002',1`], + links: [ + `'il1','p1|200000002','p1','200000002','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b2','owns',0,1000,'published'`, + ], + seals: [ + `'p1|200000002','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true); + assert.equal(/A_eik_mismatch/.test(out), true); +}); + +test('B_bidder_eik: the stored bidder row disagrees with the link ЕИК → hard finding', () => { + // Row integrity. The card renders the BIDDER's name and money but links out on the link's ЕИК, so a + // disagreement means the reader is shown one company and sent to another. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`, `'b2','ДРУГ ЕООД','200000002',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b2','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true); + assert.equal(/B_bidder_eik/.test(out), true); +}); + +test('B_eik_invalid: publishing against a checksum-INVALID ЕИК → hard finding', () => { + // eik_valid=0 means the ЕИК failed its control digit, so it identifies no company at all. This axis is + // the rail that keeps such a row off the public surface, and nothing else re-checks it downstream. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',0`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, true, 'an invalid-ЕИК publish must fail the gate'); + assert.equal(/B_eik_invalid/.test(out), true); +}); + +test('the four axes above are a BOUND: a clean, valid, sealed link passes them all', () => { + // POSITIVE CONTROL for the whole block. Four assertions that something fails prove nothing unless the + // correct shape passes — an audit that flagged everything would satisfy every test above. + const { threw, out } = buildAndAudit({ + bidders: [`'b1','РЕАЛЕН ЕООД','100000001',1`], + links: [ + `'il1','p1|100000001','p1','100000001','${K('РЕАЛЕН ЕООД')}','exact_name_key','document','b1','owns',0,1000,'published'`, + ], + seals: [ + `'p1|100000001','document','owner','role:owner:CR_F_19_L','2026-08-12','tr-rules-1','live'`, + ], + }); + assert.equal(threw, false, `a clean link must pass: ${out}`); + for (const axis of ['A_key_missing', 'A_eik_mismatch', 'B_bidder_eik', 'B_eik_invalid']) + assert.equal(new RegExp(axis).test(out), false, `${axis} must not fire on a clean link`); +}); diff --git a/scripts/cacbg/classify.mjs b/scripts/cacbg/classify.mjs index fe979345..d066a3fa 100644 --- a/scripts/cacbg/classify.mjs +++ b/scripts/cacbg/classify.mjs @@ -16,6 +16,7 @@ const FORM_TOKENS = new Set([ 'КД', 'СД', 'АДСИЦ', + 'КДА', 'КООПЕРАЦИЯ', 'ФОНДАЦИЯ', 'СДРУЖЕНИЕ', @@ -32,6 +33,24 @@ const FORM_TOKENS = new Set([ // generic/withhold — the safe side), never fabricates a joint-stock exclusion. Operates on UPPERCASE input. const SEAT_MARKER = /\s+(?:ГР|С|ОБЩ|ОБЛ|Ж\.К)\.\s*\S[^,]*$/u; const hasFormToken = (s) => s.split(/[^А-ЯЁ]+/).some((t) => FORM_TOKENS.has(t)); + +// The legal forms that TERMINATE a фирма (ЗТРРЮЛНЦ writes them as a suffix), so anything after one is a +// seat or a qualifier — never part of the name. ЕТ/СД/КД/КООПЕРАЦИЯ/ФОНДАЦИЯ/СДРУЖЕНИЕ are deliberately +// ABSENT: those PRECEDE the фирма („ЕТ Алекс Петров Димитров"), and truncating after them would eat the +// name itself — turning a distinctive ЕТ into a generic one and withholding a true link. +const SUFFIX_FORMS = new Set(['ЕООД', 'ООД', 'ЕАД', 'АД', 'АДСИЦ', 'КДА', 'ДЗЗД']); + +// Cut everything after the LAST фирма-terminating form token. This is the case the comma-peel and the +// marker strip both miss: „ТРЕЙС ГРУП ХОЛД АД София" has neither a comma nor a „гр." dot, so it survived +// both, stopped ending in its form, and defeated every end-anchored form test downstream. Token-exact +// (never a substring), so „КАДИЕВ ГЛОБАЛ ЕООД" and „АД-ХОК ЕООД" are untouched. +function stripAfterSuffixForm(s) { + const re = /[А-ЯЁ]+/gu; + let last = null; + for (let m = re.exec(s); m !== null; m = re.exec(s)) if (SUFFIX_FORMS.has(m[0])) last = m; + return last === null ? s : s.slice(0, last.index + last[0].length).trim(); +} + function stripSeatSuffix(upper) { let s = String(upper).trim(); // Peel trailing comma-clauses right-to-left while the clause bears no legal form (i.e. it's a seat, not @@ -44,7 +63,7 @@ function stripSeatSuffix(upper) { ) { s = m[1].trim(); } - return s.replace(SEAT_MARKER, '').trim(); + return stripAfterSuffixForm(s.replace(SEAT_MARKER, '').trim()); } /** @@ -71,49 +90,55 @@ const norm = (s) => .replace(/[\s.\-–—]+/g, ' ') .trim(); -// Joint-stock / listed legal form (АД / ЕАД / АДСИЦ) as the TRAILING form token. In BG company names the +// Joint-stock / share-issuing legal form (АД / ЕАД / АДСИЦ / КДА) as the TRAILING form token. In BG company names the // legal form is always the suffix, so anchor to the end (optionally followed by quotes/whitespace); a whole // token bounded on the left by string edge, whitespace or quotes — NOT hyphens/dots, so „АД-ХОК ЕООД" (a // hyphenated ООД name) is not misread. Anchoring to the suffix is what stops „АД ГРУП ООД" (an ООД whose // NAME begins with the token „АД") being wrongly excluded as joint-stock — the form there is ООД. -const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ)[\s"„“”«»]*$/u; +// Twinned byte-for-byte by JOINT_SUFFIX in scripts/tr/deed.mjs — the TR parser cannot import from this +// directory without closing a cacbg↔tr cycle. A test in deed.test.mjs pins the two identical; change one +// and that test fails rather than the two silently diverging on a legal form. +export const JOINT_STOCK = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; +// The same four forms as whole tokens. Kept in step with JOINT_STOCK above by classify.test.mjs, and with +// deed.mjs's JOINT_SUFFIX twin by deed.test.mjs — three spellings of one rule, all three pinned. +const JOINT_STOCK_FORMS = new Set(['АД', 'ЕАД', 'АДСИЦ', 'КДА']); /** * Materiality by legal form. The public ownership surface is CLOSELY-HELD companies only (ООД/ЕООД/ЕТ/ - * КД/СД/ДЗЗД or a form-unspecified name from the closely-held table). Joint-stock forms (АД/ЕАД/АДСИЦ) are + * КД/СД/ДЗЗД or a form-unspecified name from the closely-held table). Joint-stock forms (АД/ЕАД/АДСИЦ/КДА, + * the last a командитно дружество с акции — it issues shares, so it belongs with them) are * public-float securities — a declared parcel of listed shares is NOT a material ownership conflict, and * presenting it as one defames (the „11 Trace shares → €88M" trap). Excludes only an explicit АД-form token, * so it withholds rather than fabricates. @returns {boolean} true ⇒ material/closely-held. */ export function closelyHeldForm(name) { - return !JOINT_STOCK.test( - stripSeatSuffix( - String(name ?? '') - .normalize('NFC') - .toUpperCase(), - ), - ); + // LAST-FORM-TOKEN-WINS, not an end anchor. The anchor asked „does the name END in a joint-stock form?", + // which a declarant-typed cell can defeat just by appending a seat — and `stripSeatSuffix` cannot be + // trusted to have removed every shape of one. Asking instead „which legal form is the name's LAST?" + // is position-independent: a trailing seat, a stray qualifier, or nothing at all leaves the verdict + // unchanged, while „АД" leading („АД ГРУП ООД") or glued („АД-ХОК ЕООД") still isn't the form. + // + // This is why the predicate no longer uses JOINT_STOCK directly while deed.mjs's twin still does: that + // twin reads the deed envelope's `fullName` — a REGISTRY-clean name that genuinely ends in its form — + // whereas this one reads a free-text cell a human typed. Same rule, different input hygiene. + const tokens = stripSeatSuffix( + String(name ?? '') + .normalize('NFC') + .toUpperCase(), + ) + .split(/[^А-ЯЁ]+/u) + .filter(Boolean); + const lastForm = tokens.filter((t) => FORM_TOKENS.has(t)).at(-1); + // No form token at all ⇒ nothing says joint-stock ⇒ material, exactly as the anchor behaved. The bar + // only ever fires on an EXPLICIT joint-stock form, so it withholds rather than fabricates. + return lastForm === undefined || !JOINT_STOCK_FORMS.has(lastForm); } -/** Seat proof: declared seat and winner settlement both present and equal ⇒ same entity (deterministic). */ -export function seatConfirmed(declSeat, winnerSettlement) { - const a = norm(declSeat); - const b = norm(winnerSettlement); - return a.length > 0 && b.length > 0 && a === b; -} - -/** - * Publish tier for a single-winner-ЕИК match: - * 'A_eik' — a declarant-provided ЕИК resolved the winner: the national unique identifier, - * deterministic even behind a generic/colliding name. Assigned by the loader for - * declared_eik matches (not this function — it needs the match method); ADR-0016. - * 'A_seat' — seat-confirmed: deterministic, publishable even for generic names. - * 'B_distinctive' — single-ЕИК + structurally distinctive name: publishable (disclosed heuristic). - * 'C_hold' — generic name, no seat/ЕИК proof: withhold pending TR name-census. - */ -export function publishTier({ seatOk, distinctiveness }) { - if (seatOk) return 'A_seat'; - return distinctiveness === 'distinctive' ? 'B_distinctive' : 'C_hold'; -} +// seatConfirmed() and publishTier() lived here until #279. The publish tiers they produced +// (A_seat / B_distinctive / C_hold) are superseded by the Trade Register evidence ladder in +// scripts/tr/evidence.mjs — identity now rests on a checkable registry fact rather than on the shape +// of the declared name (ADR-0033, superseding ADR-0009). The seat comparison moved with it, because a +// declared seat is now matched against the REGISTERED seat rather than against the winner row's +// settlement column. nameDistinctiveness above survives, narrowed to an AND-gate on the weakest rung. /** * Temporal relation of a contract to the years a stake was declared (asset decls are annual snapshots). diff --git a/scripts/cacbg/classify.test.mjs b/scripts/cacbg/classify.test.mjs index 7d2bddd1..f0f971ba 100644 --- a/scripts/cacbg/classify.test.mjs +++ b/scripts/cacbg/classify.test.mjs @@ -2,11 +2,10 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { nameDistinctiveness, - seatConfirmed, - publishTier, temporalStatus, localityToken, closelyHeldForm, + JOINT_STOCK, } from './classify.mjs'; test('nameDistinctiveness: numbers / Latin / ≥3 words are distinctive; bare 1-2 word Cyrillic is generic', () => { @@ -32,19 +31,6 @@ test('nameDistinctiveness: numbers / Latin / ≥3 words are distinctive; bare 1- assert.equal(nameDistinctiveness('„ДОМИНО" ЕООД'), 'generic'); // quoted single word + form }); -test('seatConfirmed: equal non-empty seats confirm; empty or mismatched do not', () => { - assert.equal(seatConfirmed('Шумен', 'ШУМЕН'), true); - assert.equal(seatConfirmed('София', 'Пловдив'), false); - assert.equal(seatConfirmed('', 'София'), false); // sparse winner/declared seat never confirms - assert.equal(seatConfirmed('София', ''), false); -}); - -test('publishTier: seat proof wins; else distinctiveness decides publish vs hold', () => { - assert.equal(publishTier({ seatOk: true, distinctiveness: 'generic' }), 'A_seat'); - assert.equal(publishTier({ seatOk: false, distinctiveness: 'distinctive' }), 'B_distinctive'); - assert.equal(publishTier({ seatOk: false, distinctiveness: 'generic' }), 'C_hold'); -}); - test('temporalStatus: contract within declared-year span is contemporaneous', () => { assert.equal(temporalStatus([2020, 2021, 2022], 2021), 'contemporaneous'); assert.equal(temporalStatus([2020, 2022], 2024), 'after_last_decl'); @@ -71,6 +57,25 @@ test('closelyHeldForm: ООД/ЕООД/ЕТ material; АД/ЕАД/АДСИЦ (li assert.equal(closelyHeldForm('АД СТИЛ ЕООД'), true); }); +test('closelyHeldForm: КДА (командитно дружество с акции) is joint-stock and must be excluded', () => { + // КДА issues shares like an АД — the shareholder book is not public, so a declared parcel is neither + // verifiable nor necessarily material, which is the whole basis of the exclusion. It was missing from + // both JOINT_STOCK and FORM_TOKENS, so a КДА read as closely-held and its holder could be published + // as an owner. #279 rung 1 names it explicitly alongside АД and ЕАД. + assert.equal(closelyHeldForm('ФИНАНС КДА'), false); + assert.equal(closelyHeldForm('"АЛФА ИНВЕСТ" КДА'), false); + assert.equal(closelyHeldForm('АЛФА КДА, гр. София'), false); // seat suffix must not rescue it + // …and the mirror: „КДА" inside a word or as a leading token is not the form. + assert.equal(closelyHeldForm('КДА-ТРЕЙД ЕООД'), true); + assert.equal(closelyHeldForm('КДА ГРУП ООД'), true); +}); + +test('nameDistinctiveness: КДА counts as a legal form, not a content word', () => { + // FORM_TOKENS feeds the content-word count. A form token counted as content inflates distinctiveness, + // which is the direction that publishes prematurely. + assert.equal(nameDistinctiveness('ФИНАНС КДА'), 'generic'); +}); + test('closelyHeldForm: a trailing седалище after the form does not flip an АД to closely-held (libel)', () => { // The declarant appended the seat to the name cell. Without stripping it, the end-anchored form test // misses the АД and returns closely-held=true → a listed-АД parcel presented as a material conflict. @@ -86,6 +91,56 @@ test('closelyHeldForm: a trailing седалище after the form does not flip assert.equal(closelyHeldForm('СТРОЙ, ИНВЕСТ ООД'), true); // trailing clause has ООД → closely-held }); +test('the token set and the JOINT_STOCK regex name the SAME four forms', () => { + // closelyHeldForm now decides on the last form TOKEN while deed.mjs's envelope test still uses the + // end-anchored REGEX. Two spellings of one rule drift silently — the day one gains a form the other + // lacks, a joint-stock company passes one gate and is barred by the other. Pin them to each other. + // (deed.mjs's third spelling is pinned to the regex by deed.test.mjs:378.) + for (const form of ['АД', 'ЕАД', 'АДСИЦ', 'КДА']) { + assert.equal(JOINT_STOCK.test(`ФИРМА ${form}`), true, `regex misses ${form}`); + assert.equal(closelyHeldForm(`ФИРМА ${form}`), false, `token set misses ${form}`); + // …and the mirror: every form the regex accepts must be one the token set bars, seat or no seat. + assert.equal(closelyHeldForm(`ФИРМА ${form} София`), false, `token set misses ${form} + seat`); + } + // Every FORM_TOKEN that is NOT one of the four must read as closely-held. + for (const form of ['ЕООД', 'ООД', 'ЕТ', 'ДЗЗД', 'КД', 'СД']) { + assert.equal(closelyHeldForm(`ФИРМА ${form}`), true, `${form} wrongly barred`); + } +}); + +test('closelyHeldForm: a seat with NO comma and NO „гр." marker still cannot flip an АД (libel)', () => { + // The gap the marker/comma rules leave open. `SEAT_MARKER` requires a literal dot and the comma-peel + // requires a comma, so „ТРЕЙС ГРУП ХОЛД АД София" — the plainest way a declarant writes it — survives + // both, no longer ENDS in the form, and the end-anchored JOINT_STOCK test misses it. A listed АД then + // reads as closely-held: the „11 акции на Trace → €88M" trap, from the one input shape nothing strips. + assert.equal(closelyHeldForm('ТРЕЙС ГРУП ХОЛД АД София'), false); + assert.equal(closelyHeldForm('ТРЕЙС ГРУП ХОЛД АД СОФИЯ'), false); + assert.equal(closelyHeldForm('Транспроект ЕАД Пловдив'), false); + assert.equal(closelyHeldForm('НЕС АДСИЦ Варна'), false); + assert.equal(closelyHeldForm('АЛФА КДА София'), false); + // POSITIVE CONTROLS — the bar must stay a bound, not become a blanket. A predicate that always returned + // false would pass every assertion above; these are what distinguish the fix from that (ADR-0027). + assert.equal(closelyHeldForm('Вамос ООД Русе'), true); // dot-less seat on a closely-held form + assert.equal(closelyHeldForm('ЕНЕРДЖИ СЪПЛАЙ ЕООД Бургас'), true); + assert.equal(closelyHeldForm('АД ГРУП ООД'), true); // leading „АД" is not the form + assert.equal(closelyHeldForm('АД-ХОК ЕООД'), true); // „АД" glued by a hyphen is not a form token + assert.equal(closelyHeldForm('КДА ГРУП ООД'), true); + assert.equal(closelyHeldForm('КАДИЕВ ГЛОБАЛ ЕООД'), true); // „АД" inside a word + assert.equal(closelyHeldForm('ЕТ Алекс'), true); // ЕТ leads the фирма — nothing after it is a seat +}); + +test('nameDistinctiveness: a dot-less trailing city is not counted as a content word either', () => { + // Same blind spot, and here it fails toward PUBLISHING: an uncounted seat token inflates the content-word + // count to 3 ⇒ 'distinctive'. Since #279 rung 2 gates an uncorroborated „Документ" publish on exactly this + // predicate (ADR-0035), a seat read as a content word is a false company-identity claim, not just noise. + assert.equal(nameDistinctiveness('СТРОЙ ИНВЕСТ ООД София'), 'generic'); + assert.equal(nameDistinctiveness('НИКАС КОМЕРС ЕООД Пловдив'), 'generic'); + // POSITIVE CONTROLS: a genuinely ≥3-content-word фирма stays distinctive, and a leading-form ЕТ name + // keeps its content words — nothing after „ЕТ" is a seat, so the strip must not reach them. + assert.equal(nameDistinctiveness('ХИДРО СТРОЙ МОНТАЖ ЕООД София'), 'distinctive'); + assert.equal(nameDistinctiveness('ЕТ АЛЕКС ПЕТРОВ ДИМИТРОВ'), 'distinctive'); +}); + test('nameDistinctiveness: a trailing city is not counted as a content word (no premature publish)', () => { // The exact over-publish case: 2 real content words + a seat token would read as 3 → distinctive. assert.equal(nameDistinctiveness('СТРОЙ ИНВЕСТ ООД, СОФИЯ'), 'generic'); // was distinctive via +СОФИЯ diff --git a/scripts/cacbg/guard.mjs b/scripts/cacbg/guard.mjs index 97a326bd..9502ac96 100644 --- a/scripts/cacbg/guard.mjs +++ b/scripts/cacbg/guard.mjs @@ -9,8 +9,15 @@ import { fileURLToPath } from 'node:url'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); export const SCRATCH = path.join(ROOT, 'scratch', 'cacbg'); -export function assertScratchIgnored() { - const probe = path.join('scratch', 'cacbg', '.probe'); +/** + * Assert that a `scratch/` tree is git-ignored, before anything writes PII into it. + * Parameterised rather than copied: the Trade Register leg needs the identical rail for its deed + * cache (owner names, company addresses — ADR-0033 decision 5), and a second copy of a safety rail + * drifts from the original. Existing no-argument callers are unaffected. + * @param {string} [subdir] directory under scratch/ to probe + */ +export function assertScratchIgnored(subdir = 'cacbg') { + const probe = path.join('scratch', subdir, '.probe'); try { execFileSync('git', ['check-ignore', '-q', probe], { cwd: ROOT }); } catch { diff --git a/scripts/cacbg/link-corrections.jsonl b/scripts/cacbg/link-corrections.jsonl new file mode 100644 index 00000000..e69de29b diff --git a/scripts/cacbg/load-ambiguous.test.mjs b/scripts/cacbg/load-ambiguous.test.mjs index 336cfb30..01cf8f7a 100644 --- a/scripts/cacbg/load-ambiguous.test.mjs +++ b/scripts/cacbg/load-ambiguous.test.mjs @@ -45,12 +45,33 @@ function buildAndLoad(bidderRows) { fs.writeFileSync(path.join(STAGING, 'holdings.jsonl'), ''); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + // An EMPTY Trade Register cache, pointed at explicitly (#279, ADR-0033). This fixture declares no + // holdings, so it resolves no links and the coverage gate is satisfied by an empty candidate set — + // but the cache FILE must still exist, because a missing one refuses the whole load. + // + // Explicit rather than defaulted, and that matters: load.mjs falls back to the repo's real + // scratch/tr/tr-cache.sqlite, so a test that omits TR_CACHE_DB silently runs against whatever the + // developer's last live crawl left behind. That is exactly how this test passed locally and failed + // in CI, where no such file exists. + const trDb = path.join(dir, 'tr-cache.sqlite'); + new DatabaseSync(trDb).close(); + let threw = false; try { execFileSync( 'node', ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], - { cwd: ROOT, env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, stdio: 'pipe' }, + { + cwd: ROOT, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: trDb, + TR_RAW_DIR: path.join(dir, 'tr-deeds'), + }, + stdio: 'pipe', + }, ); } catch { threw = true; // execFileSync throws on a non-zero exit code (a fired hard-gate) diff --git a/scripts/cacbg/load-collision.test.mjs b/scripts/cacbg/load-collision.test.mjs index 32991193..43fb1df4 100644 --- a/scripts/cacbg/load-collision.test.mjs +++ b/scripts/cacbg/load-collision.test.mjs @@ -16,7 +16,7 @@ import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad() { execFileSync( @@ -24,15 +24,93 @@ function runLoad() { ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], { cwd: ROOT, - env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + }, stdio: 'pipe', }, ); } + +/** + * Minimal Trade Register evidence for this fixture (#279, ADR-0033). Publishing now rests on a registry + * fact, so a loader test without a cache would only ever exercise the fail-closed path. Each winner's + * deed names its own declarant as съдружник, which is the „Документ" rung. + */ +function buildTrCache(owners) { + fs.mkdirSync(TR_RAW, { recursive: true }); + const cache = new DatabaseSync(TR_DB); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + for (const [eik, spec] of Object.entries(owners)) { + const { name, seat } = spec; + const deed = { + uic: eik, + fullName: '"ФИКС" ЕООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: `

${name}

`, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + // The REGISTERED seat, matching what this official declared. Both фирми here are + // generic („КОМПАНИЯ ЕДНО/ДВЕ" — two content words), so under ADR-0035 a name match + // alone cannot establish which company was declared; the agreeing seat is what does. + // Without it this fixture would exercise the withholding path instead of the + // cross-folder attribution it exists to test. + { + nameCode: 'CR_F_5_L', + htmlData: `

Държава: БЪЛГАРИЯ
Населено място: гр. ${seat}

`, + fieldEntryNumber: '20110502101008', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], + }; + fs.writeFileSync(path.join(TR_RAW, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code,legal_form_verdict,latest_own_entry_date) VALUES(?,?,?,?,?,?,?,?)', + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + 4, + 'closely_held', + '2011-05-02', + ); + } + cache.close(); +} const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-collision-')); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); fs.mkdirSync(STAGING, { recursive: true }); @@ -45,7 +123,7 @@ before(() => { CREATE TABLE contracts(id TEXT PRIMARY KEY, tender_id TEXT, bidder_id TEXT, signed_at TEXT, amount_eur REAL); INSERT INTO authorities VALUES ('auth:1','ВЕДОМСТВО ТЕСТ'); INSERT INTO tenders VALUES ('t1','auth:1'),('t2','auth:1'); - -- Two distinct seat-confirmed single-ЕИК winners → both publish (A_seat), no ambiguity. + -- Two distinct single-ЕИК winners, each named in its own deed AND seat-corroborated → both publish. INSERT INTO bidders VALUES ('eik:100000001','КОМПАНИЯ ЕДНО ЕООД','100000001',1,'София'); INSERT INTO bidders VALUES ('eik:200000002','КОМПАНИЯ ДВЕ ЕООД','200000002',1,'Пловдив'); INSERT INTO contracts VALUES ('c1','t1','eik:100000001','2021-05-01',50000); @@ -92,6 +170,11 @@ before(() => { holdings.map((h) => JSON.stringify(h)).join('\n') + '\n', ); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + + buildTrCache({ + 100000001: { name: 'ИВАН ПЪРВИ ТЕСТОВ', seat: 'София' }, + 200000002: { name: 'ПЕТЪР ВТОРИ ПРОБЕН', seat: 'Пловдив' }, + }); }); after(() => fs.rmSync(dir, { recursive: true, force: true })); diff --git a/scripts/cacbg/load-divestment-nonwinner.test.mjs b/scripts/cacbg/load-divestment-nonwinner.test.mjs index 226be07c..fac967b5 100644 --- a/scripts/cacbg/load-divestment-nonwinner.test.mjs +++ b/scripts/cacbg/load-divestment-nonwinner.test.mjs @@ -17,7 +17,7 @@ import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad() { execFileSync( @@ -25,15 +25,86 @@ function runLoad() { ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], { cwd: ROOT, - env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + }, stdio: 'pipe', }, ); } + +/** + * Minimal Trade Register evidence for this fixture (#279, ADR-0033). Publishing now rests on a registry + * fact, so a loader test without a cache would only ever exercise the fail-closed path. Each winner's + * deed names its own declarant as съдружник — the „Документ" rung. + */ +function buildTrCache(owners) { + fs.mkdirSync(TR_RAW, { recursive: true }); + const cache = new DatabaseSync(TR_DB); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + for (const [eik, names] of Object.entries(owners)) { + const html = [] + .concat(names) + .map((n) => `

${n}

`) + .join(`
`); + const deed = { + uic: eik, + fullName: '"ФИКС" ЕООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: html, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], + }; + fs.writeFileSync(path.join(TR_RAW, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code,legal_form_verdict,latest_own_entry_date) VALUES(?,?,?,?,?,?,?,?)', + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + 4, + 'closely_held', + '2011-05-02', + ); + } + cache.close(); +} + const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-divest-')); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); fs.mkdirSync(STAGING, { recursive: true }); @@ -64,7 +135,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Диан Дивестов', + person: 'Диан Иванов Дивестов', position: '', entity: 'ДИВ ТЕХ 5 ЕООД', kind: 'shares', @@ -80,7 +151,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Диан Дивестов', + person: 'Диан Иванов Дивестов', position: '', entity: 'НЕПОБЕДИМ КОМПАНИ ООД', kind: 'shares', @@ -99,7 +170,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Верен Държателев', + person: 'Верен Иванов Държателев', position: '', entity: 'ДРУГ ВИН 6 ЕООД', kind: 'shares', @@ -114,6 +185,13 @@ before(() => { holdings.map((h) => JSON.stringify(h)).join('\n') + '\n', ); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + + buildTrCache({ + // Диан DIVESTED, so the live deed must name somebody else — otherwise §7's reconciliation + // correctly overturns his declared termination and the case stops testing what it is for. + 100000001: 'НОВ ИВАНОВ СОБСТВЕНИК', + 200000002: 'ВЕРЕН ИВАНОВ ДЪРЖАТЕЛЕВ', + }); // filings.jsonl — one record per declaration (as extract.mjs emits it), carrying the declaration type. The // divest horizon is built from this: Диан's 2022 assets declaration (listing only the non-winner) advances // his assets horizon to 2022 → the 2019 ДИВ ТЕХ 5 winner stake is withdrawn. Верен has only a 2019 filing. @@ -143,8 +221,8 @@ test('a later NON-winner ownership filing still withdraws a divested winner stak ) .get(eik, person); - const dian = link('100000001', 'Диан Дивестов'); - const veren = link('200000002', 'Верен Държателев'); + const dian = link('100000001', 'Диан Иванов Дивестов'); + const veren = link('200000002', 'Верен Иванов Държателев'); // The divested winner stake is dated to its last declaration and excluded from the public surface. assert.equal(dian.status, 'withdrawn'); diff --git a/scripts/cacbg/load-homonym.test.mjs b/scripts/cacbg/load-homonym.test.mjs index 395b62ed..7f38b12b 100644 --- a/scripts/cacbg/load-homonym.test.mjs +++ b/scripts/cacbg/load-homonym.test.mjs @@ -18,19 +18,94 @@ import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad() { execFileSync( 'node', ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], - { cwd: ROOT, env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING }, stdio: 'pipe' }, + { + cwd: ROOT, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + }, + stdio: 'pipe', + }, ); } + +/** + * Minimal Trade Register evidence for this fixture (#279, ADR-0033). Publishing now rests on a registry + * fact, so a loader test without a cache would only ever exercise the fail-closed path. Each winner's + * deed names its own declarant as съдружник — the „Документ" rung. + */ +function buildTrCache(owners) { + fs.mkdirSync(TR_RAW, { recursive: true }); + const cache = new DatabaseSync(TR_DB); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + for (const [eik, names] of Object.entries(owners)) { + const html = [] + .concat(names) + .map((n) => `

${n}

`) + .join(`
`); + const deed = { + uic: eik, + fullName: '"ФИКС" ЕООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: html, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], + }; + fs.writeFileSync(path.join(TR_RAW, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code,legal_form_verdict,latest_own_entry_date) VALUES(?,?,?,?,?,?,?,?)', + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + 4, + 'closely_held', + '2011-05-02', + ); + } + cache.close(); +} + const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-homonym-')); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); fs.mkdirSync(STAGING, { recursive: true }); @@ -59,7 +134,7 @@ before(() => { template: 'assets', category: '', institution: 'ОБЩИНА СОФИЯ', - person: 'Георги Иванов', + person: 'Георги Иванов Петров', position: 'Кмет', entity: 'ВИН ЕДНО 5 ЕООД', kind: 'shares', @@ -77,7 +152,7 @@ before(() => { template: 'assets', category: '', institution: 'МИНИСТЕРСТВО НА ТЕСТА', - person: 'Георги Иванов', + person: 'Георги Иванов Петров', position: 'Директор', entity: 'ВИН ДВЕ 6 ЕООД', kind: 'shares', @@ -95,7 +170,7 @@ before(() => { template: 'assets', category: '', institution: 'ОБЩИНА СОФИЯ', - person: 'Георги Иванов', + person: 'Георги Иванов Петров', position: 'Кмет', entity: 'ВИН ЕДНО 5 ЕООД', kind: 'shares', @@ -110,6 +185,11 @@ before(() => { holdings.map((h) => JSON.stringify(h)).join('\n') + '\n', ); fs.writeFileSync(path.join(STAGING, 'related.jsonl'), ''); + + buildTrCache({ + 100000001: 'ГЕОРГИ ИВАНОВ ПЕТРОВ', + 200000002: 'ГЕОРГИ ИВАНОВ ПЕТРОВ', + }); }); after(() => fs.rmSync(dir, { recursive: true, force: true })); @@ -120,7 +200,7 @@ test('same-named officials at different institutions do NOT merge into one perso // Two DISTINCT persons named „Георги Иванов" — one per institution — never one merged identity. const persons = db - .prepare("SELECT id, name FROM persons WHERE name = 'Георги Иванов' ORDER BY id") + .prepare("SELECT id, name FROM persons WHERE name = 'Георги Иванов Петров' ORDER BY id") .all(); assert.equal(persons.length, 2, 'two distinct namesake officials, not one merged person'); assert.notEqual(persons[0].id, persons[1].id); diff --git a/scripts/cacbg/load.mjs b/scripts/cacbg/load.mjs index 4dc360b2..ca7c030c 100644 --- a/scripts/cacbg/load.mjs +++ b/scripts/cacbg/load.mjs @@ -14,24 +14,65 @@ import { DatabaseSync } from 'node:sqlite'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +// nameDistinctiveness is deliberately NOT imported: the evidence ladder replaced it in the publish +// path (ADR-0033). It survives in classify.mjs for the review queue, not for a publishing decision. import { - nameDistinctiveness, - seatConfirmed, - publishTier, temporalStatus, localityToken, closelyHeldForm, + nameDistinctiveness, } from './classify.mjs'; +import { openCache, readDeed, coverage } from '../tr/cache.mjs'; +import { TR_DB, TR_RAW, deedPath } from '../tr/paths.mjs'; +import { + evidenceVerdict, + isSealedFact, + reconcileTermination, + RULES_VERSION, +} from '../tr/evidence.mjs'; import { companyCandidates, declaredEiks } from './extract-companies.mjs'; -import { fingerprint, loadSuppressions, SUPPRESSION_KEY_VERSION } from './suppressions.mjs'; +import { + fingerprint, + loadCorrections, + loadSuppressions, + SUPPRESSION_KEY_VERSION, +} from './suppressions.mjs'; import { canonicalInstitution } from './institutions.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); const STAGING = process.env.CACBG_STAGING || path.join(ROOT, 'scratch/cacbg/staging'); const MIGRATION = path.join(ROOT, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// 0006 attaches the Trade Register evidence seal (#279, ADR-0033). Applied here as well as 0003 +// because this loader rebuilds the CACBG tables from the migrations on every run — a seal table +// missing from the work DB would make every evidence write fail at ship time instead of at load. +const MIGRATION_EVIDENCE = path.join( + ROOT, + 'packages/db/migrations/0009_interest_link_evidence.sql', +); const REPORT = path.join(STAGING, 'findings.md'); -const MATCHER_VERSION = 'cnk-1+classify-1'; // bump when the normalizer or classify logic changes +// Bumped for #279: classify-2 (КДА added to the joint-stock bar) + tr-1 (identity now rests on a +// Trade Register fact, not on name distinctiveness). RULES_VERSION versions the EVIDENCE rules +// separately — §8's monotonicity gate keys on that one, not on this. +const MATCHER_VERSION = 'cnk-1+classify-2+tr-1'; +const TR_CACHE_DB = process.env.TR_CACHE_DB || TR_DB; +const TR_RAW_DIR = process.env.TR_RAW_DIR || TR_RAW; +// A deliberate, logged override for the coverage gate below. Without it a single permanently +// unreachable ЕИК would deadlock the pipeline forever; with it, the operator states that they know. +const ALLOW_PARTIAL_TR = process.argv.includes('--allow-partial-tr'); +// Bootstrap mode: write the crawl's input list and stop, successfully. The decision run and the register +// crawl now share one job (they must — the raw deeds hold third-party names and cannot travel between +// runners), and that job has to be able to start from nothing: the list is derived from the resolved +// corpus, so only this script can produce it, but the full run refuses without the very cache the list +// is used to fill. Ignoring the refusal's exit code instead would erase the difference between „no cache +// yet" and „this run is broken". +// +// POINT THIS AT A SCRATCH COPY OF THE WORK DB. It is not a read-only pass: reaching the candidate list +// means rebuilding the corpus tables, so it drops and repopulates persons/declarations/declared_interests +// and leaves interest_links EMPTY. Empty is the safe end state (the ship floor refuses it, and no link +// can be published without evidence it never gathered), but it is not the state a subsequent real run +// should inherit. The one thing it must never touch either way is the monotonicity snapshot — see below. +const EMIT_CANDIDATES_ONLY = process.argv.includes('--emit-candidates'); const { companyNameKey, isMatchableKey } = await import('../../packages/shared/src/company-name-key.ts'); @@ -55,7 +96,24 @@ const readJsonl = (f) => .map((l) => JSON.parse(l)) : []; -const db = new DatabaseSync(DB); +// Bootstrap mode works on a THROWAWAY COPY, and that is not a convenience — it is the correctness of the +// monotonicity gate. Reaching the candidate list means rebuilding the corpus tables, which drops +// interest_links; the pass itself never publishes, so it would leave the table EMPTY. The next real run +// would then read that empty table as its prior-published set, write an empty snapshot, and the gate — +// whose only job is to notice a published claim disappearing — would pass unconditionally, for ever. +// Copying here rather than asking the caller to do it keeps the flag safe wherever it is invoked from. +const WORK_DB = EMIT_CANDIDATES_ONLY ? `${DB}.bootstrap` : DB; +if (EMIT_CANDIDATES_ONLY) { + for (const suffix of ['', '-wal', '-shm']) { + // -wal/-shm may legitimately be absent (a cleanly closed DB has neither); anything else must surface. + try { + fs.copyFileSync(`${DB}${suffix}`, `${WORK_DB}${suffix}`); + } catch (e) { + if (e.code !== 'ENOENT') throw e; + } + } +} +const db = new DatabaseSync(WORK_DB); db.exec('PRAGMA foreign_keys=ON'); // Suppressions live in a VERSION-CONTROLLED, HMAC-fingerprinted list (ADR-0031), NOT a DB table — so a // takedown survives a fresh-CI-runner rebuild and never ships the „who was taken down" signal to prod. @@ -68,9 +126,91 @@ const suppressedFp = new Set(suppEntries.map((e) => e.fp)); // B3 unused-suppression gate: every listed fingerprint MUST match exactly one built link. Track which get // used; a fingerprint that matched nothing (a stale/mis-keyed takedown) fails the build after the load loop. const usedSuppressions = new Set(); +// Export the CURRENT published surface before anything is dropped — ADR-0033 decision 6. +// +// The rebuild below is total, so the previous run's published set exists only in this instant. The +// audit compares against this file and hard-fails on a link that was published last run and is not +// published now under an UNCHANGED rules_version: nothing licensed that removal, so it is a silent +// recall regression. rules_version travels per key, because a link that vanished under a rules BUMP is +// an intentional removal and must degrade to a printed diff instead. +// +// Held and withdrawn links are deliberately excluded: they were never a public claim, so their absence +// next run is not a regression. On a first run the tables do not exist yet and the export is an empty +// set — written, not skipped, so a missing file means „the loader never ran", not „nothing published". +const priorPublished = (() => { + try { + return db + .prepare( + `SELECT il.link_key AS link_key, e.rules_version AS rules_version + FROM interest_links il + LEFT JOIN interest_link_evidence e ON e.link_key = il.link_key + WHERE il.status = 'published'`, + ) + .all() + .map((r) => ({ link_key: r.link_key, rules_version: r.rules_version ?? RULES_VERSION })); + } catch (e) { + // A first run: interest_links does not exist yet. Any OTHER failure must surface — swallowing it + // would turn a broken export into a permanently silent gate. + if (!/no such table/i.test(e.message)) throw e; + return []; + } +})(); +// Decision 6's SECOND sanctioned removal: „a correction of wrong input". A link whose input was wrong +// should never have been published, but correcting the input UNBUILDS it — so a suppression on it +// would match no built link and trip the B3 gate above, while doing nothing leaves a permanent hard +// finding. The acknowledgement is therefore recorded here, against the set the gate actually compares: +// each prior-published key whose fingerprint is listed is exported flagged, and audit.mjs reads the +// flag as a declared removal. Fingerprinted for ADR-0031's reason — `pid|eik` in git would record which +// named official was tied to which company for ever. +const CORRECTIONS_LIST = + process.env.CACBG_CORRECTIONS_LIST || path.join(ROOT, 'scripts/cacbg/link-corrections.jsonl'); +const correctedFp = new Set( + loadCorrections(CORRECTIONS_LIST, SUPP_SALT, SUPPRESSION_KEY_VERSION).map((e) => e.fp), +); +const usedCorrections = new Set(); +const snapshot = priorPublished.map((p) => { + if (correctedFp.size === 0) return p; + const fp = fingerprint(p.link_key, SUPP_SALT); + if (!correctedFp.has(fp)) return p; + usedCorrections.add(fp); + return { ...p, corrected: true }; +}); +// The B3 rail, mirrored — and it matters MORE here. A stale suppression silently un-suppresses; a +// stale acknowledgement silently pre-clears a FUTURE disappearance of that same link, which is exactly +// the regression the gate exists to catch, with nobody having decided it. An acknowledgement is +// one-shot by construction: once the corrected link stops being published it also stops appearing in +// the prior set, so the entry must be deleted from the list in the same change that lands the fix. +if (!EMIT_CANDIDATES_ONLY) { + const unusedCorr = [...correctedFp].filter((fp) => !usedCorrections.has(fp)); + if (unusedCorr.length > 0) { + db.close(); + throw new Error( + `${unusedCorr.length} correction(s) matched NO previously published link — a stale acknowledgement ` + + `would clear a future disappearance of that link, which is the regression this gate exists to ` + + `catch. Delete the entry once its fix has shipped (an acknowledgement is one-shot). ` + + `Unmatched fingerprints: ${unusedCorr.map((f) => f.slice(0, 12) + '…').join(', ')}`, + ); + } +} +// Not written by the bootstrap pass, which works on a throwaway copy and has no business restating what +// the real run is about to record. +// +// tmp + rename, not a bare write: a crash mid-write leaves a truncated file, and audit.mjs deliberately +// does NOT swallow a parse failure here (only ENOENT is a legitimate first run). A torn snapshot would +// therefore wedge every subsequent audit until a human cleared the file by hand. The raw deeds already +// land this way; the gate's own input deserves the same. +if (!EMIT_CANDIDATES_ONLY) { + const snapPath = path.join(STAGING, 'published-snapshot.json'); + const snapTmp = `${snapPath}.tmp`; + fs.writeFileSync(snapTmp, JSON.stringify(snapshot, null, 2) + '\n'); + fs.renameSync(snapTmp, snapPath); +} + // Full idempotent rebuild that also picks up schema changes: drop the CACBG tables (children first — // FK-safe) and re-apply the migration. Nothing to preserve — suppressions are external now. for (const t of [ + // FIRST: interest_link_evidence references interest_links, so it must go before its parent. + 'interest_link_evidence', 'interest_link_authorities', 'interest_links', 'declared_interests', @@ -80,6 +220,7 @@ for (const t of [ ]) db.exec(`DROP TABLE IF EXISTS ${t}`); db.exec(fs.readFileSync(MIGRATION, 'utf8')); +db.exec(fs.readFileSync(MIGRATION_EVIDENCE, 'utf8')); // A link is suppressed when its fingerprint is in the list. Only compute the HMAC when the list is // non-empty (size>0 ⇒ salt present, else the loader above threw), so the empty common path skips crypto. const isSuppressed = (linkKey) => { @@ -219,14 +360,41 @@ const familyMaterialByTemplate = new Map(); // declare a stake ONLY in the interests declaration). One record per declaration incl. empty / no-material ones, // so a same-type divest-to-ZERO still advances the horizon. An absent/typeless file just yields no match ⇒ the // link is kept (fail-safe: never withdraw on missing evidence). +// +// A filing whose is unreadable falls back to its FOLDER year (#279 §1.3). Dropping the record +// instead — the previous behaviour — is not the fail-safe it resembles: an undated filing that vanishes +// never advances the horizon, so `divested` stays false and a stake the official has since SOLD keeps +// naming them on the public surface. That is a stale claim about a real person, which is the failure this +// surface can least afford. The folder year is an APPROXIMATION — it is the publication year and runs +// ahead of the declared year (migration 0003) — so it can advance the horizon by up to a year early. That +// errs toward WITHDRAWING a claim we are no longer sure of, which is the safe direction here. +// +// A filing datable by NEITHER field is still ignored: the fallback dates a filing, it does not invent one. const filingMaxByPersonType = new Map(); +let filingFolderDated = 0, + filingUndatable = 0; for (const f of readJsonl(path.join(STAGING, 'filings.jsonl'))) { if (!isMatchableKey(companyNameKey(f.person))) continue; - const fy = yr(f.year); - if (!Number.isFinite(fy)) continue; + let fy = yr(f.year); + if (!Number.isFinite(fy)) { + fy = yr(f.folder); + if (!Number.isFinite(fy)) { + // Counted, not silently dropped. A horizon we failed to advance is invisible in the output — the + // link simply stays up — so without this the only symptom of a corpus-wide date regression would be + // a surface that quietly stopped withdrawing anything. + filingUndatable++; + continue; + } + filingFolderDated++; + } const k = `${personId(f.person, f.institution)}|${f.template ?? ''}`; filingMaxByPersonType.set(k, Math.max(filingMaxByPersonType.get(k) ?? fy, fy)); } +if (filingFolderDated > 0 || filingUndatable > 0) { + console.log( + ` filings: ${filingFolderDated} dated by FOLDER (unreadable ), ${filingUndatable} undatable (ignored — no horizon)`, + ); +} db.exec('BEGIN'); for (const h of readJsonl(path.join(STAGING, 'holdings.jsonl'))) { @@ -384,6 +552,12 @@ for (const r of readJsonl(path.join(STAGING, 'related.jsonl'))) { db.exec('COMMIT'); // --- enrich each (person,eik) → interest_links (+ per-authority breakdown) ----------------------- +// THE WRITER of contract_count / contract_value_eur. Its join shape (contracts→tenders→authorities→ +// bidders) is mirrored by CONTRACT_JOIN in packages/db/src/queries/related-persons.ts, so that the +// read-time subset can never exceed what was stored. `authorities` here IS projected (the per-authority +// breakdown needs the name), unlike on the read side where it looks dead — that asymmetry is why the two +// have to be pinned to each other rather than reasoned about separately. related-persons-sql.test.ts +// asserts they agree; change one only by changing both, and re-baseline ADR-0033 §10 when you do. const contractStmt = db.prepare( "SELECT strftime('%Y', c.signed_at) yr, a.id auth_id, a.name authority, c.amount_eur eur FROM contracts c JOIN tenders t ON t.id=c.tender_id JOIN authorities a ON a.id=t.authority_id JOIN bidders b ON b.id=c.bidder_id WHERE b.eik_normalized=?", ); @@ -393,6 +567,12 @@ const insLink = db.prepare( const insILA = db.prepare( 'INSERT OR IGNORE INTO interest_link_authorities(link_key,authority_id,authority_name,contract_count,value_eur,own) VALUES(?,?,?,?,?,?)', ); +// The evidence seal (#279 §8, migration 0006). Written for EVERY link, not only published ones — the +// seals on held and withdrawn links are what let the review queue explain itself. `matched_fact` is a +// closed vocabulary and must NEVER carry a name; the audit enforces that with a pattern check. +const insEvidence = db.prepare( + 'INSERT OR REPLACE INTO interest_link_evidence(link_key,evidence_kind,registry_role,matched_fact,entry_number,entry_date,lookup_date,rules_version,live_status) VALUES(?,?,?,?,?,?,?,?,?)', +); // classify one authority (whose name may be a ';'-joined blob) against the official's institutions. // exact = deterministic name equality; name_contains/locality = DISCLOSED heuristics (candidate, not proof). const OWN_RANK = { exact: 3, name_contains: 2, locality: 1, none: 0 }; @@ -416,6 +596,90 @@ function authOwn(authorityName, instNorms, instNormsLong, locTokens) { } // Distinct officials who declared each company (ЕИК). A private interest has ONE owner-declarant; a // public body's board is declared by MANY rotating members — the deterministic ex-officio tell (ADR-0019). +// ── Trade Register evidence: the candidate set, the fail-closed gate, and the deed reader ───────── +// Identity now rests on a checkable registry fact rather than on the shape of the declared name +// (#279, ADR-0033). Two consequences the loader has to enforce, both fail-closed: +// +// 1. NO cache ⇒ throw. Publishing without evidence is precisely what this change abolishes. +// 2. PARTIAL cache ⇒ throw. This is the silent one. An 80%-restored cache yields roughly 80 +// published links, which is ABOVE ship-related-persons.mjs's floor of 50 — so it would sail +// through that guard, ship a decimated surface, and wipe the rest of the live links. +// +// The candidate set is every resolved ЕИК across ALL aggregates, not just the ones that end up +// published: a link held for want of evidence still needs its deed to say so. +const candidateEiks = [...new Set([...agg.values()].map((r) => r.eik))].sort(); +fs.writeFileSync(path.join(STAGING, 'candidate-eiks.txt'), candidateEiks.join('\n') + '\n'); + +if (EMIT_CANDIDATES_ONLY) { + // Stop BEFORE the TR gate and before anything is written to the domain. A bootstrap pass that built + // links would leave a surface resting on no evidence at all, and a failure between this pass and the + // real one would leave that surface sitting in the work DB, shippable. + console.log( + `${candidateEiks.length} candidate ЕИК written for the crawler; stopping (--emit-candidates)`, + ); + db.close(); + for (const suffix of ['', '-wal', '-shm']) fs.rmSync(`${WORK_DB}${suffix}`, { force: true }); // the copy has served its purpose + process.exit(0); +} + +if (!fs.existsSync(TR_CACHE_DB)) { + db.close(); + throw new Error( + `REFUSE TO LOAD: no Trade Register cache at ${TR_CACHE_DB}. Every publishing decision now rests ` + + `on a registry fact (ADR-0033); without the cache there is no evidence to rest on. Run ` + + `scripts/tr/fetch-deeds.mjs --eiks-file ${path.join(STAGING, 'candidate-eiks.txt')} first.`, + ); +} +const trCache = openCache(TR_CACHE_DB); +const trCoverage = coverage(trCache, candidateEiks); +console.log( + `TR cache: ${trCoverage.covered}/${trCoverage.wanted} covered ` + + `(fetched ${trCoverage.fetched}, outside ТР ${trCoverage.outsideTr}, missing ${trCoverage.missing})`, +); +if (trCoverage.missing > 0 && !ALLOW_PARTIAL_TR) { + trCache.close(); + db.close(); + throw new Error( + `REFUSE TO LOAD: the Trade Register cache covers ${trCoverage.covered} of ${trCoverage.wanted} ` + + `candidate ЕИК. A partial cache does not fail loudly downstream — it publishes a decimated ` + + `surface that still clears the ship floor and then wipes the rest of the live links. Finish ` + + `the crawl, or pass --allow-partial-tr to state that a smaller surface is intended.`, + ); +} + +// Deeds are read from git-ignored scratch and cached in memory for the run. The parsed deed carries +// third-party names; they are used ONLY inside evidenceVerdict's boolean comparisons and never reach +// a column, a log line or the report (ADR-0033 decision 5). +// The lookup date sealed on every link: when the evidence was gathered, not when it was interpreted. +// It is the freshness bound the methodology page has to state, so it comes from the cache rather than +// from `now` — a re-run over an unchanged cache must not make the evidence look fresher than it is. +const trLookupDate = (() => { + const row = trCache.prepare('SELECT MAX(fetched_at) m FROM deeds').get(); + return row?.m ? String(row.m).slice(0, 10) : new Date().toISOString().slice(0, 10); +})(); + +const deedCache = new Map(); +function deedFor(eik) { + if (deedCache.has(eik)) return deedCache.get(eik); + const row = readDeed(trCache, eik); + let entry; + if (!row) entry = { deed: null, outsideTr: false, missing: true }; + else if (row.status === 'outside_tr') entry = { deed: null, outsideTr: true, missing: false }; + else { + // RE-DERIVED from the ЕИК through safeEik, never the stored raw_path. The cache index is written by + // the crawler but travels between runs — and once it does, a stored path is attacker-influenced + // input joined straight onto a filesystem root, which is a traversal read. purgeExpired already + // re-derives for exactly this reason; this was the one read that did not. deedPath also throws on a + // malformed ЕИК rather than quietly reading some other company's deed (R8). + const file = deedPath(eik, TR_RAW_DIR); + entry = fs.existsSync(file) + ? { deed: JSON.parse(fs.readFileSync(file, 'utf8')), outsideTr: false, missing: false } + : { deed: null, outsideTr: false, missing: true }; + } + deedCache.set(eik, entry); + return entry; +} + const declarantsByEik = new Map(); for (const rec of agg.values()) { if (rec.scope !== 'self') continue; // ex-officio tell counts SELF declarants of a public board only @@ -460,7 +724,20 @@ for (const rec of agg.values()) { a.count++; if (r.eur != null) a.value += r.eur; } - const seatOk = [...rec.seats].some((s) => seatConfirmed(s, rec.bidder.settlement)); + // ── the evidence ladder replaces the publish tiers (ADR-0033 decision 1) ──────────────────────── + // rec.seats is keyed on `pid|eik|scope`, so it already holds ONLY the seats this person declared for + // THIS company — which is what #279 §5 rung 3 requires: 4.9% of company-name keys carry more than one + // distinct declared seat, so a company-only key would let one person's seat confirm another's link. + const { deed, outsideTr, missing } = deedFor(rec.eik); + if (missing && !ALLOW_PARTIAL_TR) { + // Unreachable via the coverage gate above; kept as a belt-and-braces refusal so a future change + // that loosens the gate cannot silently publish a link with no evidence behind it. + trCache.close(); + db.close(); + throw new Error( + `no cached deed for ЕИК ${rec.eik} — the coverage gate should have caught this`, + ); + } // A declarant-provided ЕИК is the national unique identifier (ЗТРРЮЛНЦ) — it resolves the winner // deterministically even behind a generic or winner-colliding name, so a declared_eik match publishes // on its own basis (A_eik), never held for name-genericness. This is at least as certain as the seat @@ -468,14 +745,60 @@ for (const rec of agg.values()) { // Name-only methods (exact_name_key / extracted_name) still ride the distinctiveness/seat gate below: // a globally non-unique winner name (e.g. „Водоснабдяване и канализация ЕАД" → 2 valid ЕИК in different // towns) can never be name-distinctive, so it publishes only if the declared SEAT disambiguates, else held. + // The filters that can only WITHHOLD are retained as an AND-gate on the weakest rung only + // (ADR-0033 decision 2): a nationally shared company name cannot ride „Потвърдено". The stronger + // „Документ" rung is deliberately not gated — the register named this person in THIS company, which + // makes the name key moot. Near-zero recall cost, and it preserves ADR-0017's outcome. + // ADR-0017 carried forward, and NARROWED to what it actually held: a name backing more than one valid + // ЕИК cannot support a name-derived identity claim. It gates the SEAT leg of rung 3 only — never the + // declared-ЕИК leg (ADR-0028: the ЕИК is the identity), and never rung 2 (the register named this + // person in THIS company). nameDistinctiveness is deliberately NOT part of this gate: the seat rung + // exists precisely to rescue a generic name, so requiring distinctiveness would empty it. const nameUnique = nameGloballyUnique(rec.key); - const tier = - rec.method === 'declared_eik' - ? 'A_eik' - : publishTier({ - seatOk, - distinctiveness: nameUnique ? nameDistinctiveness(rec.key) : 'generic', - }); + // „Неизвестна" — the withholding verdict, used for every way of ending up with no usable evidence. + const noEvidence = () => ({ + kind: 'unknown', + publishable: false, + registryRole: null, + matchedFact: null, + entryNumber: null, + entryDate: null, + rulesVersion: RULES_VERSION, + }); + // With --allow-partial-tr the operator has accepted an incomplete cache. An uncached ЕИК then yields + // no evidence at all, which is „Неизвестна" — held. It must never be read as a reason to publish. + let verdict; + if (missing) verdict = noEvidence(); + else { + try { + verdict = evidenceVerdict({ + deed, + outsideTr, + declarantName: rec.person, + declaredSeats: [...rec.seats], + declaredEik: rec.method === 'declared_eik', + firstDeclaredYear: declYears.length ? Math.min(...declYears) : null, + scope: rec.scope, + nameGloballyUnique: nameUnique, + // ADR-0035. The resolver picked this winner by фирма; `nameGloballyUnique` above only says no OTHER + // WINNER shares that name, which says nothing about the register at large. So an uncorroborated + // rung 2 additionally asks whether the фирма is one a national twin is unlikely to share. + // `rec.key` is the WINNER's registered name — the canonical spelling of the company we actually + // looked up, which is the identity in question. + companyNameDistinctive: nameDistinctiveness(rec.key) === 'distinctive', + }); + } catch (err) { + // A deed we cannot parse is a deed we cannot reason about — so this link withholds, exactly as an + // uncached one does, and the run continues. Failing the whole load instead would let one malformed + // deed out of ~400 decide the fate of every other link, and the ship floor would then refuse the + // reduced surface — turning a single bad payload into a total outage. Loud, per link, fail-closed. + console.error( + ` ${rec.eik}: evidence UNREADABLE — ${err instanceof Error ? err.message : err} (link held)`, + ); + verdict = noEvidence(); + } + } + const tier = verdict.kind; const contemporaneous = [...years].some( (cy) => temporalStatus(declYears, cy) === 'contemporaneous', ) @@ -537,15 +860,33 @@ for (const rec of agg.values()) { // non-surfaced class ('internal'), not 'published'. cValue can legitimately be 0 with cCount>0 (contracts // whose amount is unknown/NULL) — that is a real conflict of unknown value, so gate on COUNT, not value. const surfaces = (iClass === 'private_ownership' || iClass === 'family_ownership') && cCount > 0; + // §7: „terminated" is an inference from SILENCE, and its commonest cause is a finished mandate, not a + // sale. Before ADR-0021 E11's withdrawal takes effect, a terminated OWN stake is reconciled against the + // live deed: a person still registered as an owner has not divested. Family stakes are never reconciled + // — the registered owner there is the relative, whose name we neither store nor check. + // PHASE 1 uses only `terminated`; the „и към днешна дата" label is computed and deliberately not + // rendered (ADR-0033 decision 4 — it asserts a present tense behind an LIA addendum). + const recon = + divested && rec.scope === 'self' + ? reconcileTermination({ deed, declarantName: rec.person, scope: rec.scope }) + : { terminated: divested, label: null }; + const terminatedEffective = recon.terminated; + + // Order matters and differs from the old ladder: `internal` is now decided BEFORE `published`, so a + // non-surfaced class (ex-officio board, management-only, zero-contract) can never land in `held`. + // `held` is the REVIEW QUEUE, and its population is exactly the evidence rungs that withhold — + // bar_joint_stock, unknown and outside_tr. const status = isSuppressed(linkKey) ? 'suppressed' - : divested - ? 'withdrawn' - : tier === 'C_hold' - ? 'held' - : surfaces - ? 'published' - : 'internal'; + : verdict.kind === 'refuted' + ? 'withdrawn' // §5.4 — own stakes only; evidence.mjs refuses to refute a family stake + : terminatedEffective + ? 'withdrawn' + : !surfaces + ? 'internal' + : verdict.publishable + ? 'published' + : 'held'; const yrs = [...years]; insLink.run( `il:${linkKey}`, @@ -570,6 +911,35 @@ for (const rec of agg.values()) { yrs.length ? String(Math.max(...yrs)) : null, status, ); + // live_status is RE-DERIVED every run and never treated as part of the permanent seal (ADR-0033 R3): + // it asserts a present tense whose freshness is bounded by the cache refresh cycle. + const liveStatus = !terminatedEffective + ? recon.label === 'owner_today' + ? 'terminated_owner_still' + : 'live' + : recon.label === 'manager_today' + ? 'terminated_manager_still' + : 'terminated'; + // Refuse at WRITE time, not only in the post-hoc audit. The audit runs after the whole domain is + // built; by then the name is already in a table, and a run that dies on a suppressed-by-default axis + // could ship it. `matched_fact` is the one sealed column derived from a deed's text, so it is the one + // place a third-party name can reach a served row — the rail belongs where the value is produced. + if (!isSealedFact(verdict.matchedFact)) + throw new Error( + `REFUSE TO SEAL: matched_fact for ${linkKey} is outside the closed vocabulary — a name may have ` + + `leaked out of a deed (#279 §9, ADR-0033 decision 5)`, + ); + insEvidence.run( + linkKey, + verdict.kind, + verdict.registryRole, + verdict.matchedFact, + verdict.entryNumber, + verdict.entryDate, + trLookupDate, + verdict.rulesVersion, + liveStatus, + ); for (const [auth_id, a] of perAuth) insILA.run(linkKey, auth_id, a.name, a.count, a.value || null, a.own); } @@ -686,6 +1056,21 @@ const S = { r.n, ]), ), + // Every evidence rung's count, publishing and withholding alike (ADR-0033 decision 1). `publish_tier` + // IS the verdict kind since #279. + by_evidence_kind: Object.fromEntries( + q('SELECT publish_tier, COUNT(*) n FROM interest_links GROUP BY publish_tier').map((r) => [ + r.publish_tier, + r.n, + ]), + ), + // ADR-0035's residual: links where the register named this person in the resolved company but nothing + // established that the company is the declared one. This is the number F8's hand-labelled sample reads + // to decide whether the distinctiveness gate tightens to strict ЕИК/seat corroboration. Reported + // separately from the map above because it is a DECISION input, not just a tally. + document_uncorroborated: one( + "SELECT COUNT(*) n FROM interest_links WHERE publish_tier='document_uncorroborated'", + ).n, ambiguous_name_keys: ambiguousKeys.length, ambiguous_name_key_examples: ambiguousKeys, noMatch, diff --git a/scripts/cacbg/load.test.mjs b/scripts/cacbg/load.test.mjs index d795bf59..987cbdd8 100644 --- a/scripts/cacbg/load.test.mjs +++ b/scripts/cacbg/load.test.mjs @@ -10,11 +10,13 @@ import path from 'node:path'; import os from 'node:os'; import { fileURLToPath } from 'node:url'; import { fingerprint } from './suppressions.mjs'; +// The seal vocabulary is asserted with the PRODUCTION predicate, imported from the module that writes it. +import { isSealedFact } from '../tr/evidence.mjs'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, '..', '..'); const SUPP_SALT = 'test-salt-9f3a'; // stand-in for the CI secret SUPPRESSION_SALT -let dir, DB, STAGING; +let dir, DB, STAGING, TR_DB, TR_RAW; function runLoad(extraEnv = {}) { execFileSync( @@ -22,17 +24,105 @@ function runLoad(extraEnv = {}) { ['--import', path.join(HERE, 'register-ts.mjs'), path.join(HERE, 'load.mjs')], { cwd: ROOT, - env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING, ...extraEnv }, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: TR_DB, + TR_RAW_DIR: TR_RAW, + ...extraEnv, + }, stdio: 'pipe', }, ); } + +/** + * Build a Trade Register cache + raw deeds covering the fixture's winners. + * + * `spec[eik]` describes one deed: `owners` / `managers` are full names placed in SEPARATE registry + * entities (so the entity-boundary rule is exercised end to end), `form` is the numeric legalForm and + * `suffix` the ЗТРРЮЛНЦ form on fullName. Anything omitted from `spec` is still cached — as a deed + * naming somebody else — because the loader must FAIL CLOSED on a cache that does not cover every + * candidate, and a test that silently left ЕИК uncovered would exercise that path by accident. + */ +function buildTrCache(dbFile, rawDir, spec = {}, { omit = [] } = {}) { + fs.mkdirSync(rawDir, { recursive: true }); + const cache = new DatabaseSync(dbFile); + cache.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT)`); + const src = new DatabaseSync(DB, { readOnly: true }); + const eiks = src + .prepare('SELECT eik_normalized e FROM bidders WHERE eik_normalized IS NOT NULL') + .all() + .map((r) => r.e); + src.close(); + + const container = (t) => + `

${t}

`; + const joinEntities = (names) => names.map(container).join(`
`); + + for (const eik of eiks) { + if (omit.includes(eik)) continue; + const d = spec[eik] ?? {}; + if (d.outsideTr) { + cache + .prepare( + 'INSERT OR REPLACE INTO deeds(eik,status,fetched_at,outside_reason) VALUES(?,?,?,?)', + ) + .run(eik, 'outside_tr', '2026-08-05T00:00:00Z', 'HTTP 200, empty body'); + continue; + } + const fields = []; + const push = (nameCode, names, entryDate) => + names?.length && + fields.push({ + nameCode, + htmlData: joinEntities(names), + fieldEntryNumber: '20110502101007', + fieldEntryDate: `${entryDate ?? '2011-05-02'}T00:00:00`, + }); + push('CR_F_19_L', d.owners ?? ['НЯКОЙ ДРУГ СОБСТВЕНИК'], d.ownEntryDate); + push('CR_F_7_L', d.managers, d.ownEntryDate); + if (d.seat) push('CR_F_5_L', [`Населено място: ${d.seat}`], d.seatEntryDate ?? d.ownEntryDate); + const deed = { + uic: eik, + fullName: `"ФИКС" ${d.suffix ?? 'ООД'}`, + legalForm: d.form ?? 4, + sections: [{ subDeeds: [{ groups: [{ fields }] }] }], + }; + fs.writeFileSync(path.join(rawDir, `${eik}.json`), JSON.stringify(deed)); + cache + .prepare( + `INSERT OR REPLACE INTO deeds(eik,status,http_status,fetched_at,raw_path,legal_form_code, + legal_form_verdict,seat_normalized,latest_own_entry_date) + VALUES(?,?,?,?,?,?,?,?,?)`, + ) + .run( + eik, + 'fetched', + 200, + '2026-08-05T00:00:00Z', + `${eik}.json`, + d.form ?? 4, + d.suffix && /АД|КДА/.test(d.suffix) ? 'joint_stock' : 'closely_held', + d.seat ? d.seat.replace(/^гр\.\s*/, '').toUpperCase() : null, + d.ownEntryDate ?? '2011-05-02', + ); + } + cache.close(); +} const open = () => new DatabaseSync(DB, { readOnly: true }); before(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-load-')); DB = path.join(dir, 'fixture.sqlite'); STAGING = path.join(dir, 'staging'); + TR_DB = path.join(dir, 'tr-cache.sqlite'); + TR_RAW = path.join(dir, 'tr-deeds'); fs.mkdirSync(STAGING, { recursive: true }); // minimal slice of the winner schema that load.mjs joins @@ -82,6 +172,22 @@ before(() => { -- stake at all. The empty filing advances his horizon past 2019 → the 2019 stake is withdrawn. INSERT INTO bidders VALUES ('eik:101010104','ДИВЕСТ ЗЕРО 4 ЕООД','101010104',1,'София'); INSERT INTO contracts VALUES ('c11','t1','eik:101010104','2019-05-01',150000); + -- §1.3 unparseable filing YEAR: Безгодин owns ДИВЕСТ БЕЗГОД (2019), then files a later declaration whose + -- is unreadable while its FOLDER carries 2023. The folder must supply the horizon, or the filing + -- is dropped, the horizon never advances, and a sold stake stays published — a stale public claim. + INSERT INTO bidders VALUES ('eik:212121218','ДИВЕСТ БЕЗГОД 9 ЕООД','212121218',1,'София'); + INSERT INTO contracts VALUES ('c17','t1','eik:212121218','2019-05-01',120000); + -- §1.3 positive control: Дрънкан's later filing is datable by NEITHER NOR folder. It must be + -- ignored, never guessed — an undatable filing is no evidence of a sale, so this stake stays published. + INSERT INTO bidders VALUES ('eik:232323231','ДРЪНКАН ТЕХ 10 ЕООД','232323231',1,'София'); + INSERT INTO contracts VALUES ('c18','t1','eik:232323231','2019-05-01',110000); + -- ADR-0035 winner-vs-non-winner homonym: „ХОМОНИМ ТРЕЙД" is a GENERIC фирма (two content words) and the + -- sole WINNER holding it. Хомоним Иванов Тестов declared a stake in a company of that name — but the one + -- he owns never bid, so the resolver lands on this winner instead. This winner's deed happens to name a + -- HOMONYM (identical three tokens), which under a name-only rung 2 „proves" a link false in both halves. + -- Nothing corroborates the company (no declared ЕИК, no declared seat), so it must be withheld. + INSERT INTO bidders VALUES ('eik:242424248','ХОМОНИМ ТРЕЙД ЕООД','242424248',1,'София'); + INSERT INTO contracts VALUES ('c19','t1','eik:242424248','2023-05-01',95000); -- N10 canonicalization: Канонов owns КАНОН ТЕХ 5, filing „МВР" one year and the full ministry name the -- next → ONE identity, ONE link (not a split). Distinctive name (number) → tier B publishable. INSERT INTO bidders VALUES ('eik:131313136','КАНОН ТЕХ 5 ЕООД','131313136',1,'София'); @@ -99,6 +205,12 @@ before(() => { -- not twice — the load-side sibling of the UI conflictHeadline per-ЕИК money dedup. INSERT INTO bidders VALUES ('eik:181818187','ПАРТНЬОРИ 5 ЕООД','181818187',1,'София'); INSERT INTO contracts VALUES ('c15','t1','eik:181818187','2022-06-01',600000); + -- FAMILY positive control (#279): identical in shape to Кмет's case but the official DECLARED the + -- company's seat, which is what confirms the company's identity when the registered owner is the + -- relative whose name we never hold. Without this case „family published: 0" would be + -- indistinguishable from a structurally dead path (ADR-0027's false-zero lesson). + INSERT INTO bidders VALUES ('eik:191919199','СЕМЕЕН ДОМ ЕООД','191919199',1,'Русе'); + INSERT INTO contracts VALUES ('c16','t4','eik:191919199','2023-07-01',180000); `); db.close(); @@ -128,7 +240,7 @@ before(() => { template: 'assets', category: '', institution: 'X', - person: 'Мария Иванова', + person: 'Мария Иванова Петрова', position: '', entity: '"ГЕНЕРИК" ООД', kind: 'shares', @@ -145,7 +257,7 @@ before(() => { template: 'assets', category: '', institution: 'Y', - person: 'Петър Николов', + person: 'Петър Иванов Николов', position: '', entity: 'СИЙ ЕООД', kind: 'shares', @@ -162,7 +274,7 @@ before(() => { template: 'assets', category: '', institution: 'Z', - person: 'Георги Стоянов', + person: 'Георги Иванов Стоянов', position: '', entity: 'СИЙ ЕООД', kind: 'shares', @@ -181,7 +293,7 @@ before(() => { template: 'assets', category: '', institution: 'W', - person: 'Стефан Колев', + person: 'Стефан Иванов Колев', position: '', entity: '"ГЕНЕРИК" ООД, ЕИК 222222229', kind: 'shares', @@ -198,7 +310,7 @@ before(() => { template: 'assets', category: '', institution: 'V', - person: 'Радка Илиева', + person: 'Радка Иванова Илиева', position: '', entity: '"ГЕНЕРИК" ООД, ЕИК 333333338', kind: 'shares', @@ -215,7 +327,7 @@ before(() => { template: 'interests', category: '', institution: 'U', - person: 'Борис Манолов', + person: 'Борис Иванов Манолов', position: 'член на съвет', entity: 'ХОЛДИНГ 9 ЕАД', kind: 'management', @@ -231,7 +343,7 @@ before(() => { template: 'interests', category: '', institution: 'U', - person: 'Виктор Асенов', + person: 'Виктор Иванов Асенов', position: 'член на съвет', entity: 'ХОЛДИНГ 9 ЕАД', kind: 'management', @@ -249,7 +361,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Николай Дивестов', + person: 'Николай Иванов Дивестов', position: '', entity: 'ДИВЕСТ 1 ЕООД', kind: 'shares', @@ -265,7 +377,7 @@ before(() => { template: 'assets', category: '', institution: 'T', - person: 'Николай Дивестов', + person: 'Николай Иванов Дивестов', position: '', entity: 'ДИВЕСТ 2 ЕООД', kind: 'shares', @@ -283,7 +395,7 @@ before(() => { template: 'assets', category: '', institution: 'ОБЩИНА ТЕСТ', - person: 'Кмет Тестов', + person: 'Кмет Иванов Тестов', position: 'кмет', entity: 'ЕВРОСТРОЙ 21 ЕООД', kind: 'shares', @@ -293,6 +405,24 @@ before(() => { holderRelation: 'related', controlHash: 'H11', }, + // FAMILY positive control: same as Кмет, but with the seat declared → rung 3 confirms the company. + { + folder: '2024', + xmlFile: 'K2.xml', + year: '2023', + template: 'assets', + category: '', + institution: 'ОБЩИНА ТЕСТ', + person: 'Кметица Иванова Втора', + position: 'кмет', + entity: 'СЕМЕЕН ДОМ ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: 'Русе', + holderRelation: 'related', + controlHash: 'H11b', + }, // SECURITIES: Акционер holds LISTED joint-stock shares (kind securities) → excluded, no ownership link. { folder: '2024', @@ -301,7 +431,7 @@ before(() => { template: 'assets', category: '', institution: 'S', - person: 'Акционер Тестов', + person: 'Акционер Иванов Тестов', position: '', entity: 'ЛИСТЕД ТЕСТ АД', kind: 'securities', @@ -320,7 +450,7 @@ before(() => { template: 'assets', category: '', institution: 'N2', - person: 'Нула Тестов', + person: 'Нула Иванов Тестов', position: '', entity: 'НУЛА ТЕХ 3 ЕООД', kind: 'shares', @@ -340,7 +470,7 @@ before(() => { template: 'assets', category: '', institution: 'T2', - person: 'Пълен Дивестов', + person: 'Пълен Иванов Дивестов', position: '', entity: 'ДИВЕСТ ЗЕРО 4 ЕООД', kind: 'shares', @@ -350,6 +480,61 @@ before(() => { holderRelation: 'self', controlHash: 'H14', }, + // §1.3 unparseable filing year: Безгодин's 2019 stake. His later filing (in filings.jsonl, no holdings + // row) carries an unreadable but a 2023 FOLDER — the fallback that must withdraw this stake. + { + folder: '2020', + xmlFile: 'BG0.xml', + year: '2019', + template: 'assets', + category: '', + institution: 'T3', + person: 'Безгодин Иванов Дивестов', + position: '', + entity: 'ДИВЕСТ БЕЗГОД 9 ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: 'гр. София', + holderRelation: 'self', + controlHash: 'H20', + }, + // §1.3 positive control: Дрънкан's 2019 stake, whose only later filing is undatable (see filings.jsonl). + { + folder: '2020', + xmlFile: 'DR0.xml', + year: '2019', + template: 'assets', + category: '', + institution: 'T4', + person: 'Дрънкан Иванов Тестов', + position: '', + entity: 'ДРЪНКАН ТЕХ 10 ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: '', + holderRelation: 'self', + controlHash: 'H21', + }, + // ADR-0035: Хомоним's declared stake in a generic-named company, with NO ЕИК and NO seat declared. + { + folder: '2024', + xmlFile: 'HOM.xml', + year: '2023', + template: 'assets', + category: '', + institution: 'T5', + person: 'Хомоним Иванов Тестов', + position: '', + entity: 'ХОМОНИМ ТРЕЙД ЕООД', + kind: 'shares', + detail: '100%', + timing: 'annual', + seat: '', + holderRelation: 'self', + controlHash: 'H22', + }, // B4 UNKNOWN holder: the holder cell is neither confidently the declarant's own name nor a relative's // (an ambiguous 1-token-different cell). classifyHolder → 'unknown' → this forms NO link (counted // nowhere), so a phantom relative never enters a published family figure. @@ -360,7 +545,7 @@ before(() => { template: 'assets', category: '', institution: 'N3', - person: 'Двусмислен Тестов', + person: 'Двусмислен Иванов Тестов', position: '', entity: 'ДИСТИНКТ ТЕХ 7 ЕООД', kind: 'shares', @@ -379,7 +564,7 @@ before(() => { template: 'assets', category: '', institution: 'МВР', - person: 'Канонов Тестов', + person: 'Канонов Иванов Тестов', position: '', entity: 'КАНОН ТЕХ 5 ЕООД', kind: 'shares', @@ -396,7 +581,7 @@ before(() => { template: 'assets', category: '', institution: 'Министерство на вътрешните работи', - person: 'Канонов Тестов', + person: 'Канонов Иванов Тестов', position: '', entity: 'КАНОН ТЕХ 5 ЕООД', kind: 'shares', @@ -414,7 +599,7 @@ before(() => { template: 'assets', category: '', institution: '', - person: 'Безинст Тестов', + person: 'Безинст Иванов Тестов', position: '', entity: 'БЕЗИНСТ ТЕХ 6 ЕООД', kind: 'shares', @@ -434,7 +619,7 @@ before(() => { template: 'interests', category: '', institution: 'INT', - person: 'Интер Тестов', + person: 'Интер Иванов Тестов', position: '', entity: 'ИНТЕР ТЕХ 8 ЕООД', kind: 'shares', @@ -454,7 +639,7 @@ before(() => { template: 'assets', category: '', institution: 'ТЕСТ ВЕДОМСТВО', - person: 'Алфа Партньоров', + person: 'Алфа Иванов Партньоров', position: '', entity: 'ПАРТНЬОРИ 5 ЕООД', kind: 'shares', @@ -471,7 +656,7 @@ before(() => { template: 'assets', category: '', institution: 'ДРУГО ВЕДОМСТВО', - person: 'Бета Партньоров', + person: 'Бета Иванов Партньоров', position: '', entity: 'ПАРТНЬОРИ 5 ЕООД', kind: 'shares', @@ -503,7 +688,7 @@ before(() => { xmlFile: 'ZE1.xml', year: '2023', template: 'assets', // Пълен's later EMPTY ASSET filing — same type as his 2019 asset stake ⇒ divests it - person: 'Пълен Дивестов', + person: 'Пълен Иванов Дивестов', institution: 'T2', }); // #226 (Todor B1) cross-type: Интер declares his stake ONLY in an INTERESTS declaration (2020); his later @@ -515,13 +700,68 @@ before(() => { xmlFile: 'INTA.xml', year: '2023', template: 'assets', - person: 'Интер Тестов', + person: 'Интер Иванов Тестов', institution: 'INT', }); + // §1.3: the divesting filing whose is UNREADABLE. `folder` carries 2023 and is the only thing that + // can date it. Dropping the record leaves the horizon at 2019, `divested` false, and a sold stake on the + // public surface — the failure direction that matters here, since the claim names a real official. + filings.push({ + folder: '2023', + xmlFile: 'BG1.xml', + year: 'н/д', + template: 'assets', + person: 'Безгодин Иванов Дивестов', + institution: 'T3', + }); + // POSITIVE CONTROL for the same fallback: a filing datable by NEITHER field must still be ignored, not + // guessed at. Дрън's stake stays published — an undatable filing is no evidence of a sale. + filings.push({ + folder: 'архив', + xmlFile: 'DR1.xml', + year: '', + template: 'assets', + person: 'Дрънкан Иванов Тестов', + institution: 'T4', + }); fs.writeFileSync( path.join(STAGING, 'filings.jsonl'), filings.map((f) => JSON.stringify(f)).join('\n') + '\n', ); + + // The Trade Register evidence each link now has to rest on (#279, ADR-0033). Shaped so every + // existing case keeps the INTENT it was written for, under the new rule rather than the old one: + // • a person the register names as owner/manager → „Документ" + // • a declared seat matching the registered seat → „Потвърдено" + // • a declared ЕИК → „Потвърдено" (never name-gated, ADR-0028) + // • nobody we can match and nothing to confirm → „Неизвестна", held + buildTrCache(TR_DB, TR_RAW, { + 111111119: { managers: ['ИВАН ПЕТРОВ ТЕСТОВ'] }, // manages → document/manager (class keeps it internal) + 444444447: { seat: 'гр. Бургас' }, // Петър declared Бургас → confirmed; Георги declared none → held + 555555556: { + managers: ['БОРИС ИВАНОВ МАНОЛОВ', 'ВИКТОР ИВАНОВ АСЕНОВ'], + suffix: 'ЕАД', + form: 5, + }, + 666666665: { owners: ['СЪВСЕМ ДРУГ СОБСТВЕНИК'] }, // Николай absent → his divestment stands + 777777773: { owners: ['НИКОЛАЙ ИВАНОВ ДИВЕСТОВ'] }, // still the registered owner → document + 888888884: { owners: ['РОДНИНА КМЕТОВА'] }, // family: the RELATIVE owns it, not the official + 999999998: { suffix: 'АД', form: 5 }, + 101010104: { owners: ['ДРУГ СОБСТВЕНИК'] }, // Пълен absent → divest-to-zero stands + 131313136: { owners: ['КАНОНОВ ИВАНОВ ТЕСТОВ'] }, + 161616163: { owners: ['ИНТЕР ИВАНОВ ТЕСТОВ'] }, + 181818187: { owners: ['АЛФА ИВАНОВ ПАРТНЬОРОВ', 'БЕТА ИВАНОВ ПАРТНЬОРОВ'] }, + 121212129: { owners: ['НУЛА ИВАНОВ ТЕСТОВ'] }, + 191919199: { owners: ['РОДНИНА ВТОРА'], seat: 'гр. Русе' }, // family + declared seat → confirmed + // Безгодин is ABSENT from the deed (so §7 reconciliation cannot reverse the divestment) but his + // declared seat matches the registered one, so rung 3 says „Потвърдено" and the link PUBLISHES. + // Only the folder-dated divestment withdraws it — making the pre-fix failure the dangerous one. + 212121218: { owners: ['ДРУГ СОБСТВЕНИК СЪВСЕМ'], seat: 'гр. София' }, + 232323231: { owners: ['ДРЪНКАН ИВАНОВ ТЕСТОВ'] }, // still the owner → the undatable filing changes nothing + // The homonym: the deed names someone with Хомоним's exact three tokens. Rung 2 matches — and must + // still withhold, because nothing says this is the company he declared. + 242424248: { owners: ['ХОМОНИМ ИВАНОВ ТЕСТОВ'] }, + }); }); after(() => fs.rmSync(dir, { recursive: true, force: true })); @@ -540,7 +780,7 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // management_role never surfaces → status 'internal', NOT 'published' (a direct D1 reader must not see a // non-surfaced official+company row labelled published; the served query also filters by interest_class). assert.equal(ivan.status, 'internal'); - assert.equal(ivan.publish_tier, 'B_distinctive'); + assert.equal(ivan.publish_tier, 'document'); // the register names him a manager of this company assert.equal(ivan.relation, 'manages'); assert.equal(ivan.interest_class, 'management_role'); // manages, sole declarant → ambiguous, not headline assert.equal(ivan.own_institution, 'exact'); @@ -557,8 +797,8 @@ test('resolves publish/held/quarantine tiers deterministically', () => { assert.equal(blob.value_eur, 25000); // bare collision name (no ЕИК in text) → quarantined, Мария gets no link - assert.equal(link('222222229', 'Мария Иванова'), undefined); - assert.equal(link('333333338', 'Мария Иванова'), undefined); + assert.equal(link('222222229', 'Мария Иванова Петрова'), undefined); + assert.equal(link('333333338', 'Мария Иванова Петрова'), undefined); // the only links onto the colliding ЕИК come from declared_eik (Стефан/Радка), never exact_name_key assert.equal( db @@ -569,14 +809,14 @@ test('resolves publish/held/quarantine tiers deterministically', () => { 0, ); - const petar = link('444444447', 'Петър Николов'); - assert.equal(petar.publish_tier, 'A_seat'); // generic name rescued by seat match + const petar = link('444444447', 'Петър Иванов Николов'); + assert.equal(petar.publish_tier, 'confirmed'); // declared seat == registered seat assert.equal(petar.status, 'published'); assert.equal(petar.interest_class, 'private_ownership'); // declared a share → the headline conflict signal // two officials manage the SAME company → deterministically classed ex-officio (public board), not private - const boris = link('555555556', 'Борис Манолов'); - const viktor = link('555555556', 'Виктор Асенов'); + const boris = link('555555556', 'Борис Иванов Манолов'); + const viktor = link('555555556', 'Виктор Иванов Асенов'); assert.equal(boris.interest_class, 'ex_officio_board'); assert.equal(viktor.interest_class, 'ex_officio_board'); assert.equal(boris.relation, 'manages'); @@ -584,14 +824,14 @@ test('resolves publish/held/quarantine tiers deterministically', () => { assert.equal(boris.status, 'internal'); assert.equal(viktor.status, 'internal'); - const georgi = link('444444447', 'Георги Стоянов'); - assert.equal(georgi.publish_tier, 'C_hold'); // generic, no seat → held + const georgi = link('444444447', 'Георги Иванов Стоянов'); + assert.equal(georgi.publish_tier, 'unknown'); // same company, but he declared no seat → nothing confirms assert.equal(georgi.status, 'held'); // E11 divestment: Николай's 2019 stake in ДИВЕСТ 1 is superseded by a 2022 filing that omits it → withdrawn; // his current ДИВЕСТ 2 stake stays published. A later ownership filing that drops a company ends that link. - const gone = link('666666665', 'Николай Дивестов'); - const kept = link('777777773', 'Николай Дивестов'); + const gone = link('666666665', 'Николай Иванов Дивестов'); + const kept = link('777777773', 'Николай Иванов Дивестов'); assert.equal(gone.status, 'withdrawn'); // divested — excluded from the published surface assert.equal(gone.interest_class, 'private_ownership'); assert.equal(gone.last_declared_year, '2019'); // dated to its last declaration, never asserted "current" @@ -601,15 +841,15 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // certain ЕИК (declared_eik) behind a colliding name, no seat → the declarant-provided ЕИК is the // national unique identifier, so identity is deterministic → publishes as A_eik, NOT held for // name-genericness (ADR-0016; the ЕИК is at least as certain as the seat that rescues Радка below). - const stefan = link('222222229', 'Стефан Колев'); + const stefan = link('222222229', 'Стефан Иванов Колев'); assert.equal(stefan.match_method, 'declared_eik'); // ЕИК resolution IS certain - assert.equal(stefan.publish_tier, 'A_eik'); // ЕИК = unique identifier → deterministic, not name-gated + assert.equal(stefan.publish_tier, 'confirmed'); // the declared ЕИК confirms the company, never name-gated assert.equal(stefan.status, 'published'); // private_ownership (20% ООД share) → surfaces // same colliding name, resolved by her declared ЕИК (with seat as extra corroboration) → the ЕИК is the // identity, so A_eik (not A_seat); publishable. A_seat's own path stays covered by Петър above. - const radka = link('333333338', 'Радка Илиева'); + const radka = link('333333338', 'Радка Иванова Илиева'); assert.equal(radka.match_method, 'declared_eik'); - assert.equal(radka.publish_tier, 'A_eik'); + assert.equal(radka.publish_tier, 'confirmed'); assert.equal(radka.status, 'published'); // FAMILY: a close relative's declared stake in a winner that sold to the official's OWN institution. @@ -617,10 +857,17 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // relative's declared stake in a procurement winner is the same public-interest signal. class // family_ownership, relation 'related', own_institution exact. It has real contract money (€250k, cCount>0) // so the zero-contract gate keeps it. - const family = link('888888884', 'Кмет Тестов'); + const family = link('888888884', 'Кмет Иванов Тестов'); assert.equal(family.relation, 'related'); assert.equal(family.interest_class, 'family_ownership'); - assert.equal(family.status, 'published'); // ADR-0032: family surfaces like self + // #279 NARROWS the family surface, and this is where it shows. The registered owner of a family + // stake is the RELATIVE, whose name we deliberately never store (ADR-0010 item 4, ADR-0032 #2) — so + // rung 2 („Документ") can never fire for a family link, by construction. Its identity can only be + // confirmed by something the OFFICIAL declared: the seat, or the ЕИК. Кмет declared neither, so his + // link is now HELD rather than published. ADR-0032's decision is untouched — family publishes on the + // named surface exactly like self — but it now needs the same registry evidence as everything else. + assert.equal(family.status, 'held'); + assert.equal(family.publish_tier, 'unknown'); assert.equal(family.own_institution, 'exact'); // relative's company sold to the official's own institution assert.equal(family.contemporaneous, 1); assert.equal(family.contract_value_eur, 250000); @@ -645,8 +892,8 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // SUM(contract_value_eur) over published links double-counts that winner — the load-side twin of the UI // conflictHeadline bug. Prove the fixture actually exercises the collision (naive per-link sum strictly // exceeds the per-ЕИК-deduped sum), then assert the reported totals equal the deduped figure. - const alfa = link('181818187', 'Алфа Партньоров'); - const beta = link('181818187', 'Бета Партньоров'); + const alfa = link('181818187', 'Алфа Иванов Партньоров'); + const beta = link('181818187', 'Бета Иванов Партньоров'); assert.equal(alfa.status, 'published'); assert.equal(beta.status, 'published'); assert.equal(alfa.interest_class, 'private_ownership'); @@ -683,14 +930,14 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // ZERO-CONTRACT gate (I5): a distinctive winner with NO contracts is collected but never published — the // card would read „0 договори · 0 €", which is no procurement conflict. status 'internal', not 'published'. - const zero = link('121212129', 'Нула Тестов'); + const zero = link('121212129', 'Нула Иванов Тестов'); assert.equal(zero.contract_count, 0); assert.equal(zero.interest_class, 'private_ownership'); // it IS own material ownership … - assert.equal(zero.publish_tier, 'B_distinctive'); // … and tier-B by name … + assert.equal(zero.publish_tier, 'document'); // … with registry evidence … assert.equal(zero.status, 'internal'); // … but the zero-contract gate withholds it from the surface // SECURITIES/materiality: a self holding of LISTED joint-stock shares forms NO ownership link. - assert.equal(link('999999998', 'Акционер Тестов'), undefined); + assert.equal(link('999999998', 'Акционер Иванов Тестов'), undefined); // but it is still recorded as a declared interest (census), tagged kind securities assert.equal( db.prepare("SELECT kind FROM declared_interests WHERE entity_raw='ЛИСТЕД ТЕСТ АД'").get().kind, @@ -700,7 +947,7 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // B1 divest-to-ZERO: Пълен owned ДИВЕСТ ЗЕРО in 2019, then filed an EMPTY declaration in 2023 (no holdings // row — only a filings.jsonl entry). The empty filing advances his horizon to 2023, so the 2019 stake is // WITHDRAWN. Without the filing horizon (pre-B1) his scope-max would be 2019 and this would stay published. - const divZero = link('101010104', 'Пълен Дивестов'); + const divZero = link('101010104', 'Пълен Иванов Дивестов'); assert.equal(divZero.interest_class, 'private_ownership'); assert.equal(divZero.last_declared_year, '2019'); // dated to its last declaration, never asserted current assert.equal(divZero.status, 'withdrawn'); // caught by the empty later filing (B1) @@ -710,18 +957,47 @@ test('resolves publish/held/quarantine tiers deterministically', () => { // that asset-declaration silence as a sale and WITHDRAWS the stake; the per-type horizon must not — no later // INTERESTS filing omits the company. This link must stay PUBLISHED. (Guards against dropping a true link: // 13% of holders declare a stake only in the interests declaration.) - const crossType = link('161616163', 'Интер Тестов'); + // ADR-0035 — the CRITICAL, end to end. Хомоним declared a stake in „ХОМОНИМ ТРЕЙД ЕООД"; the company he + // actually owns never bid, so `resolveEntity` resolved his declaration to the same-named WINNER, whose + // deed names a person with his exact three tokens. Rung 2 matches. It must NOT publish: the register + // proves someone of that name owns THIS company, not that this is the company he declared. `nameGlobally- + // Unique` cannot catch it — it ranges over bidders, and this winner is the only bidder with the name. + const homonym = link('242424248', 'Хомоним Иванов Тестов'); + assert.equal(homonym.publish_tier, 'document_uncorroborated'); + assert.notEqual(homonym.status, 'published'); + // The seal must record WHY it was withheld, and must not carry the role the rung refused to assert. + const homonymSeal = db + .prepare( + 'SELECT evidence_kind, registry_role, matched_fact FROM interest_link_evidence WHERE link_key=?', + ) + .get(homonym.link_key); + assert.equal(homonymSeal.evidence_kind, 'document_uncorroborated'); + assert.equal(homonymSeal.registry_role, null); + assert.equal(homonymSeal.matched_fact, null); + + // §1.3 unparseable filing YEAR: Безгодин's later declaration has an unreadable ('н/д') but a 2023 + // FOLDER. Dropping that record — the pre-fix behaviour — leaves his horizon at 2019, so `divested` stays + // false and a stake he no longer holds keeps naming him on the public surface. The folder must date it. + const noYear = link('212121218', 'Безгодин Иванов Дивестов'); + assert.equal(noYear.interest_class, 'private_ownership'); + assert.equal(noYear.status, 'withdrawn'); + // POSITIVE CONTROL: datable by NEITHER field ⇒ ignored, not guessed. The fallback must not become a + // licence to invent a horizon — an undatable filing is no evidence of a sale, so this link stays up. + const noDate = link('232323231', 'Дрънкан Иванов Тестов'); + assert.equal(noDate.status, 'published'); + + const crossType = link('161616163', 'Интер Иванов Тестов'); assert.equal(crossType.interest_class, 'private_ownership'); assert.equal(crossType.status, 'published'); // NOT withdrawn — the later asset filing is a different type // B4 UNKNOWN holder: an ambiguous holder cell forms NO link at all (counted nowhere) — it must never // reach the leaderboard, self or family (ADR-0032). Двусмислен gets no interest_link. - assert.equal(link('111111119', 'Двусмислен Тестов'), undefined); + assert.equal(link('111111119', 'Двусмислен Иванов Тестов'), undefined); // but the person + declared_interest are still recorded (census), and it is neither self nor family. assert.equal( db .prepare( - "SELECT COUNT(*) n FROM interest_links il JOIN persons p ON p.id=il.person_id WHERE p.name='Двусмислен Тестов'", + "SELECT COUNT(*) n FROM interest_links il JOIN persons p ON p.id=il.person_id WHERE p.name='Двусмислен Иванов Тестов'", ) .get().n, 0, @@ -738,10 +1014,10 @@ test('resolves publish/held/quarantine tiers deterministically', () => { .n, 1, ); - const kanon = link('131313136', 'Канонов Тестов'); + const kanon = link('131313136', 'Канонов Иванов Тестов'); assert.equal(kanon.status, 'published'); // distinctive, private ownership, has a contract // N10 empty-institution: an empty institution cannot distinguish homonyms → Безинст forms NO link. - assert.equal(link('141414141', 'Безинст Тестов'), undefined); + assert.equal(link('141414141', 'Безинст Иванов Тестов'), undefined); db.close(); }); @@ -772,14 +1048,17 @@ test('re-run is idempotent and honors the suppression list (contested link stays 'suppressed', ); // idempotent: still exactly the same number of links + persons after a clean rebuild. - // 16 links: 12 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero - // 'withdrawn' + Интер's per-type-kept published link) + 1 family + Канонов's canonicalized single link + + // 20 links: 15 self (incl. withdrawn/held + the zero-contract 'internal' + Пълен's divest-to-zero + // 'withdrawn' + Безгодин's folder-dated 'withdrawn' + Дрънкан's undatable-filing 'published' + + // Хомоним's ADR-0035 'document_uncorroborated' hold + Интер's per-type-kept published link) + 2 family (Кмет's, now held for want of registry evidence, + // and Кметица's seat-confirmed one) + Канонов's canonicalized single link + // Алфа & Бета (two officials on one winner, ПАРТНЬОРИ 5); Мария (quarantined), Акционер (securities), // Двусмислен (unknown holder) & Безинст (empty institution) none. - assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 16); - // 19 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, - // zero-contract Нула and the two ПАРТНЬОРИ co-owners; Канонов's two institution-variant filings fold to ONE. - assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 19); + assert.equal(db.prepare('SELECT COUNT(*) n FROM interest_links').get().n, 20); + // 23 persons: everyone who declared a holding, incl. no-link Мария, Акционер, Двусмислен, Безинст, + // zero-contract Нула, the two ПАРТНЬОРИ co-owners, the two §1.3 filing-date cases and Хомоним; + // Канонов's two institution-variant filings fold to ONE. + assert.equal(db.prepare('SELECT COUNT(*) n FROM persons').get().n, 23); db.close(); }); @@ -879,3 +1158,362 @@ test('a suppression on a ROTATED key_version FAILS the build (B3 no silent salt (err) => /key_version/.test(String(err.stderr ?? '') + String(err.message ?? '')), ); }); + +// ── the fail-closed evidence gates (ADR-0033 decision 7) ───────────────────────────────────────── +// Both directions of "no evidence ⇒ no publish", because the partial case is the dangerous one: it does +// NOT fail loudly downstream. An 80%-restored cache yields roughly 80 published links, which clears +// ship-related-persons.mjs's floor of 50 — so it would ship a decimated surface and then wipe the rest +// of the live links. The loader is the only place that can still tell the difference. +test('a MISSING Trade Register cache refuses the whole load', () => { + const gone = path.join(dir, 'no-such-cache.sqlite'); + assert.throws( + () => runLoad({ TR_CACHE_DB: gone }), + /REFUSE TO LOAD[\s\S]*no Trade Register cache/, + ); +}); + +test('a PARTIAL cache refuses the load rather than publishing a decimated surface', () => { + const partialDb = path.join(dir, 'partial-cache.sqlite'); + const partialRaw = path.join(dir, 'partial-deeds'); + // Cover everything EXCEPT two winners — the shape a resumed-but-unfinished crawl leaves behind. + buildTrCache( + partialDb, + partialRaw, + { 111111119: { managers: ['ИВАН ПЕТРОВ ТЕСТОВ'] } }, + { omit: ['444444447', '777777773'] }, + ); + assert.throws( + () => runLoad({ TR_CACHE_DB: partialDb, TR_RAW_DIR: partialRaw }), + /REFUSE TO LOAD[\s\S]*covers \d+ of \d+/, + ); +}); + +test('--allow-partial-tr is the deliberate, stated override', () => { + // Without an override a single permanently unreachable ЕИК would deadlock the pipeline forever. + const partialDb = path.join(dir, 'partial-ok.sqlite'); + const partialRaw = path.join(dir, 'partial-ok-deeds'); + buildTrCache(partialDb, partialRaw, {}, { omit: ['121212129'] }); + assert.doesNotThrow(() => + execFileSync( + 'node', + [ + '--import', + path.join(HERE, 'register-ts.mjs'), + path.join(HERE, 'load.mjs'), + '--allow-partial-tr', + ], + { + cwd: ROOT, + env: { + ...process.env, + CACBG_DB: DB, + CACBG_STAGING: STAGING, + TR_CACHE_DB: partialDb, + TR_RAW_DIR: partialRaw, + }, + stdio: 'pipe', + }, + ), + ); + runLoad(); // restore the full-cache state for any later reader +}); + +test('the candidate ЕИК list is written for the crawler, covering held links too', () => { + runLoad(); + const listed = fs + .readFileSync(path.join(STAGING, 'candidate-eiks.txt'), 'utf8') + .split('\n') + .filter(Boolean); + const db = open(); + const all = db + .prepare('SELECT DISTINCT eik FROM interest_links') + .all() + .map((r) => r.eik); + db.close(); + // Every resolved ЕИК, not just the published ones: a link held for want of evidence still needs a + // deed to explain why it is held. + for (const e of all) assert.ok(listed.includes(e), `${e} missing from candidate-eiks.txt`); + assert.equal(new Set(listed).size, listed.length, 'no duplicates — each ЕИК costs one request'); +}); + +test('every link carries an evidence seal, and no seal carries a name', () => { + runLoad(); + const db = open(); + const links = db.prepare('SELECT link_key, publish_tier, status FROM interest_links').all(); + const seals = db.prepare('SELECT * FROM interest_link_evidence').all(); + assert.equal(seals.length, links.length, 'a seal for EVERY link, held and withdrawn included'); + + // The PRODUCTION predicate, imported from the module that WRITES the vocabulary — a restated regex + // here was looser than the real one and would have passed a seat token carrying a full name. + for (const s of seals) { + assert.ok(isSealedFact(s.matched_fact), s.matched_fact); + assert.ok(s.rules_version.length > 0); + assert.match(s.lookup_date, /^\d{4}-\d{2}-\d{2}$/); + } + // The rail, stated as an assertion rather than a convention. Scope it precisely: `link_key` carries + // the OFFICIAL's own identity by design — they are named on the public surface from their own + // declaration — so the rail is about everyone else. No name that exists only inside a registry deed + // (a co-owner, a manager, the relative who actually holds a family stake) may reach a sealed column. + const evidenceOnly = seals.map(({ link_key: _ignored, ...rest }) => rest); + const blob = JSON.stringify(evidenceOnly).toUpperCase(); + for (const thirdParty of [ + 'РОДНИНА', // the relative who owns the family company + 'СЪВСЕМ ДРУГ СОБСТВЕНИК', // a registry co-owner nobody declared + 'НЯКОЙ ДРУГ', + 'ДРУГ СОБСТВЕНИК', + ]) + assert.ok(!blob.includes(thirdParty), `a third-party name reached the seal: ${thirdParty}`); + // …and the declarant's name must not be duplicated into the evidence fields either. + for (const surname of ['ТЕСТОВ', 'ДИВЕСТОВ', 'ПАРТНЬОРОВ']) + assert.ok(!blob.includes(surname), `a declarant name reached an evidence column: ${surname}`); + db.close(); +}); + +test('a family stake publishes ONLY when the official confirmed the company themselves', () => { + // The positive control for the narrowest published path. A family link can never earn „Документ" — + // the registered owner is the relative, whose name we never hold — so it stands or falls on the seat + // or ЕИК the OFFICIAL declared. Without this case „family published: 0" would be indistinguishable + // from a structurally dead path (ADR-0027). + runLoad(); + const db = open(); + const withSeat = db + .prepare( + "SELECT il.status, il.publish_tier FROM interest_links il JOIN persons p ON p.id=il.person_id WHERE il.eik='191919199' AND p.name='Кметица Иванова Втора'", + ) + .get(); + assert.equal(withSeat.status, 'published'); + assert.equal(withSeat.publish_tier, 'confirmed'); + const seal = db + .prepare( + "SELECT matched_fact FROM interest_link_evidence WHERE link_key LIKE '%191919199|family'", + ) + .get(); + assert.equal(seal.matched_fact, 'seat:РУСЕ'); + db.close(); +}); + +test('the published surface is exported BEFORE the wipe, so the audit can gate monotonicity', () => { + // ADR-0033 decision 6. The loader rebuilds the CACBG tables from scratch every run, so the previous + // published set exists only in the instant before the DROP. Without this export the audit has + // nothing to compare against and the monotonicity gate can never fire — which is precisely the state + // review found: rules_version was written and never read. + // A FIRST run, on its own database: interest_links does not exist yet, so the export must be an + // empty set that is WRITTEN rather than skipped — a missing file has to keep meaning „the loader + // never ran", not „nothing was published". + const fresh = fs.mkdtempSync(path.join(os.tmpdir(), 'cacbg-load-first-')); + fs.mkdirSync(path.join(fresh, 'staging'), { recursive: true }); + for (const f of fs.readdirSync(STAGING)) + fs.copyFileSync(path.join(STAGING, f), path.join(fresh, 'staging', f)); + // The base DB (bidders, contracts) comes from the main pipeline and is a precondition of the load — + // a genuinely empty file is not a first run, it is a broken one. So: copy the base and drop the + // CACBG tables, which is exactly the state before the loader has ever run against it. + const firstDb = path.join(fresh, 'first.sqlite'); + fs.copyFileSync(DB, firstDb); + const fdb = new DatabaseSync(firstDb); + for (const t of [ + 'interest_link_evidence', + 'interest_link_authorities', + 'interest_links', + 'declared_interests', + 'related_persons_internal', + 'declarations', + 'persons', + ]) + fdb.exec(`DROP TABLE IF EXISTS ${t}`); + fdb.close(); + runLoad({ CACBG_DB: firstDb, CACBG_STAGING: path.join(fresh, 'staging') }); + assert.deepEqual( + JSON.parse(fs.readFileSync(path.join(fresh, 'staging', 'published-snapshot.json'), 'utf8')), + [], + ); + fs.rmSync(fresh, { recursive: true, force: true }); + + const SNAP = path.join(STAGING, 'published-snapshot.json'); + runLoad(); + const db = open(); + const published = db + .prepare("SELECT link_key FROM interest_links WHERE status='published'") + .all() + .map((r) => r.link_key); + db.close(); + assert.ok(published.length > 0, 'fixture must publish something for this test to mean anything'); + + runLoad(); + const snap = JSON.parse(fs.readFileSync(SNAP, 'utf8')); + assert.deepEqual( + snap.map((s) => s.link_key).sort(), + [...published].sort(), + 'the export must hold exactly what was published before the wipe', + ); + // rules_version travels WITH each key: the gate distinguishes „vanished under unchanged rules" + // (a regression) from „vanished under a rules bump" (an intentional removal), and it cannot do that + // from the current version alone. + for (const s of snap) + assert.ok(s.rules_version?.length > 0, 'each exported key carries its rules'); + // Held and withdrawn links must NOT be exported — they were never a public claim, so their absence + // next run is not a regression to gate on. + assert.equal(snap.length, published.length); +}); + +// The crawl input and the crawl's consumer are the same script, which used to make the two workflows +// deadlock: the decision run refused without a deed cache, the refresh run refused without a candidate +// list, and each produced only what the other needed. Merged into one job, the job still has to be able +// to BOOTSTRAP — produce the candidate list on a runner that has no cache yet — and it cannot do that by +// running the full load and ignoring a non-zero exit, because that exit is also how a genuinely broken +// run reports itself. Hence an explicit mode that stops at the list and succeeds. +test('--emit-candidates writes the crawl list and exits 0 with NO Trade Register cache', () => { + const gone = path.join(dir, 'bootstrap-absent.sqlite'); + const listFile = path.join(STAGING, 'candidate-eiks.txt'); + fs.rmSync(listFile, { force: true }); + assert.doesNotThrow(() => + execFileSync( + 'node', + [ + '--import', + path.join(HERE, 'register-ts.mjs'), + path.join(HERE, 'load.mjs'), + '--emit-candidates', + ], + { + cwd: ROOT, + env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING, TR_CACHE_DB: gone }, + stdio: 'pipe', + }, + ), + ); + const listed = fs.readFileSync(listFile, 'utf8').split('\n').filter(Boolean); + assert.ok(listed.length > 0, 'the bootstrap must actually produce candidates'); + assert.equal(new Set(listed).size, listed.length); + runLoad(); // restore the full built state for any later reader +}); + +function emitCandidates(trCacheDb) { + execFileSync( + 'node', + [ + '--import', + path.join(HERE, 'register-ts.mjs'), + path.join(HERE, 'load.mjs'), + '--emit-candidates', + ], + { + cwd: ROOT, + env: { ...process.env, CACBG_DB: DB, CACBG_STAGING: STAGING, TR_CACHE_DB: trCacheDb }, + stdio: 'pipe', + }, + ); +} + +test('a bootstrap pass leaves the REAL work DB untouched — it runs on a throwaway copy', () => { + // Reaching the candidate list means rebuilding the corpus tables, and the pass never publishes, so + // against the real DB it would leave interest_links empty. That is the damage: not the pass itself, + // but what the NEXT run then reads. + runLoad(); + const db = open(); + const before = db.prepare('SELECT COUNT(*) AS n FROM interest_links').get().n; + db.close(); + assert.ok(before > 0, 'the fixture must actually publish something'); + + emitCandidates(path.join(dir, 'bootstrap-absent-2.sqlite')); + + const after = open(); + assert.equal( + after.prepare('SELECT COUNT(*) AS n FROM interest_links').get().n, + before, + 'the bootstrap pass must not empty the domain it was pointed at', + ); + after.close(); + assert.equal(fs.existsSync(`${DB}.bootstrap`), false, 'the throwaway copy must be cleaned up'); +}); + +test('the monotonicity gate still sees a prior surface AFTER a bootstrap pass', () => { + // The end-to-end shape of the merged workflow: real run → bootstrap (to produce the crawl list) → + // real run. If the bootstrap emptied interest_links, the second real run would export an EMPTY + // prior-published set, and the gate — whose only job is to notice a published claim disappearing — + // would pass unconditionally, for ever. + const snapshot = path.join(STAGING, 'published-snapshot.json'); + runLoad(); + emitCandidates(path.join(dir, 'bootstrap-absent-3.sqlite')); + runLoad(); + const prior = JSON.parse(fs.readFileSync(snapshot, 'utf8')); + assert.ok( + prior.length > 0, + 'the snapshot went empty — the gate is now vacuous and would never fire again', + ); +}); + +// ── the corrections list — ADR-0033 decision 6's second sanctioned removal ──────────────────────── +// Decision 6 licenses removal by „a rules-version bump, or a correction of wrong input". Only the +// first was expressible. Suppression cannot carry a correction: correcting the input UNBUILDS the +// link, and the B3 unused-suppression gate above then fails the build for a fingerprint that matched +// nothing — so the two sanctioned removals failed in opposite directions and a real correction had no +// path at all. The list is fingerprinted for the same reason ADR-0031's is: `pid|eik` in git would +// record which named official was linked to which company, for ever. + +test('a corrections entry marks its key in the snapshot, so the audit reads a declared removal', () => { + runLoad(); + const db = open(); + const key = db + .prepare("SELECT link_key FROM interest_links WHERE status='published' LIMIT 1") + .get().link_key; + db.close(); + + const corrFile = path.join(dir, 'corr.jsonl'); + fs.writeFileSync( + corrFile, + JSON.stringify({ + fp: fingerprint(key, SUPP_SALT), + key_version: '1', + reason: 'the declaration row was misparsed; the stake was never declared', + corrected_at: '2026-08-11', + }) + '\n', + ); + runLoad({ CACBG_CORRECTIONS_LIST: corrFile, SUPPRESSION_SALT: SUPP_SALT }); + + const snap = JSON.parse(fs.readFileSync(path.join(STAGING, 'published-snapshot.json'), 'utf8')); + const marked = snap.find((s) => s.link_key === key); + assert.ok(marked, 'the key must still be exported — the gate needs to SEE it leave'); + assert.equal(marked.corrected, true); + // Every other key stays unflagged: an acknowledgement is per-link, never a blanket amnesty. + assert.equal( + snap.filter((s) => s.corrected === true).length, + 1, + 'one acknowledgement must not clear the whole surface', + ); +}); + +test('a corrections entry matching NO previously published link fails the build', () => { + // The B3 rail, mirrored. A stale acknowledgement is worse than a missing one: it sits in the list + // and would clear a FUTURE disappearance of the same link — the exact regression the gate exists to + // catch — with nobody having decided that. + const corrFile = path.join(dir, 'corr-stale.jsonl'); + fs.writeFileSync( + corrFile, + JSON.stringify({ + fp: fingerprint('p-nobody|999999999', SUPP_SALT), + key_version: '1', + reason: 'stale', + corrected_at: '2026-08-11', + }) + '\n', + ); + runLoad(); + assert.throws( + () => runLoad({ CACBG_CORRECTIONS_LIST: corrFile, SUPPRESSION_SALT: SUPP_SALT }), + (err) => + /correction/i.test(String(err.stderr ?? '') + String(err.message ?? '')) && + /matched NO/i.test(String(err.stderr ?? '') + String(err.message ?? '')), + ); +}); + +test('corrections are fail-closed on a missing salt, exactly like suppressions', () => { + const corrFile = path.join(dir, 'corr-nosalt.jsonl'); + fs.writeFileSync( + corrFile, + JSON.stringify({ fp: 'deadbeef', key_version: '1', reason: 'x', corrected_at: '2026-08-11' }) + + '\n', + ); + assert.throws( + () => runLoad({ CACBG_CORRECTIONS_LIST: corrFile, SUPPRESSION_SALT: '' }), + (err) => /SUPPRESSION_SALT is unset/.test(String(err.stderr ?? '') + String(err.message ?? '')), + ); +}); diff --git a/scripts/cacbg/parse.mjs b/scripts/cacbg/parse.mjs index bf95f154..6e89e242 100644 --- a/scripts/cacbg/parse.mjs +++ b/scripts/cacbg/parse.mjs @@ -82,10 +82,18 @@ function cellsByNum(row) { } // find the @_Num of the first column whose @_Description matches `re` (labels live on the header row) function colNum(firstRow, re, fallback) { + const found = colNumOrNull(firstRow, re); + return found ?? fallback; +} +// The same resolution WITHOUT the fallback, for the one column where "I could not find it" and "I found it +// and it was blank" must stay distinguishable. `colNum` collapses them: an unresolvable column resolves to +// a Num that is not in the row, `by[col]` is undefined, and the caller reads an empty cell it never found. +// For the HOLDER column those two readings differ by who owns the stake (§1.4) — see its call site. +function colNumOrNull(firstRow, re) { for (const c of asArray(firstRow?.Cell)) { if (c?.['@_Num'] && re.test(String(c?.['@_Description'] ?? ''))) return c['@_Num']; } - return fallback; + return null; } const year4 = (s) => String(s ?? '').match(/\b(20\d{2})\b/)?.[1] ?? null; @@ -159,15 +167,20 @@ function parseAssets(pp) { ? colNum(rows[0], /наименование.*дружеств|фирма/i, '4') : colNum(rows[0], /емитент/i, '6'); const cSeat = colNum(rows[0], /седалище/i, '5'); - const cHolder = colNum(rows[0], /собствено.*фамил/i, isOod ? '7' : '8'); + // NO fallback here, unlike every other column (§1.4). A blank holder cell MEANS something — the stake is + // the declarant's own (classifyHolder) — so a column we failed to resolve must not imitate one. With a + // fallback, a renumbered table resolves the holder to a missing column, reads '' and calls a RELATIVE's + // stake the official's own, which then publishes as their private_ownership. Unresolvable ⇒ 'unknown', + // which forms no link at all: we did not read the holder, so we make no claim about them. + const cHolder = colNumOrNull(rows[0], /собствено.*фамил/i); const cEgn = colNum(rows[0], /^егн$/i, isOod ? '8' : '9'); for (const row of rows) { const by = cellsByNum(row); const company = by[cCompany] ?? ''; if (!company) continue; if ((by[cEgn] ?? '').length > 0) egnPresent = true; - const holder = by[cHolder] ?? ''; - const holderRelation = classifyHolder(holder, declarant); + const holderRelation = + cHolder === null ? 'unknown' : classifyHolder(by[cHolder] ?? '', declarant); if (holderRelation === 'related') familyHoldingCount += 1; const seat = isOod ? (by[cSeat] ?? '') : ''; interests.push({ diff --git a/scripts/cacbg/parse.test.mjs b/scripts/cacbg/parse.test.mjs index 30495481..6a9b835e 100644 --- a/scripts/cacbg/parse.test.mjs +++ b/scripts/cacbg/parse.test.mjs @@ -249,3 +249,35 @@ test('parseList: only values shaped like a declaration filename are announced', assert.equal(shape('Уведомление'), 0, 'a title slotted into the filename field'); assert.equal(shape('AAAA.pdf'), 0, 'not a declaration document'); }); + +test('§1.4: an UNRESOLVABLE holder column is unknown, not an own stake', () => { + // `colNum` always returns a fallback, so a table that renumbers the holder column away resolves it to a + // column that is not there: `by[cHolder]` is undefined → '' → classifyHolder('') → 'self'. A RELATIVE's + // stake then enters the OWN-only path and publishes as the official's private_ownership — the surface's + // worst failure, since the official did not declare that stake as theirs. + // + // The required distinction is column RESOLVABLE-BUT-EMPTY (→ self, a blank cell means the declarant) + // versus column NOT RESOLVABLE (→ unknown, we have not read the holder at all). + const renumbered = ` + 1 + "РЕНОМЕР" ЕООД + Мария Спасова Роднинска`; + const d = parseDeclaration(assetDecl({ rows: renumbered }), 'REN.xml'); + const it = d.interests.find((i) => i.entity === '"РЕНОМЕР" ЕООД'); + assert.equal(it.holderRelation, 'unknown'); + + // POSITIVE CONTROL 1 — the column resolves and the cell is genuinely EMPTY: still the declarant's own + // stake. Collapsing both cases to 'unknown' would silently drop every stake declared this way. + const blankHolder = ` + 1 + "ПРАЗНА" ЕООД + `; + const blank = parseDeclaration(assetDecl({ rows: blankHolder }), 'BLK.xml'); + assert.equal(blank.interests[0].holderRelation, 'self'); + + // POSITIVE CONTROL 2 — a resolvable column still classifies a relative and the declarant correctly, so + // the change bounds one case rather than blanketing the parser. + const ok = parseDeclaration(assetDecl({ rows: selfRow + familyRow }), 'OK.xml'); + assert.equal(ok.interests.find((i) => i.entity === '"ТЕСТ АГРО" ЕООД').holderRelation, 'self'); + assert.equal(ok.interests.find((i) => i.entity === '"ФАМИЛНА" ЕООД').holderRelation, 'related'); +}); diff --git a/scripts/cacbg/suppressions.mjs b/scripts/cacbg/suppressions.mjs index b1096c60..2c67c93d 100644 --- a/scripts/cacbg/suppressions.mjs +++ b/scripts/cacbg/suppressions.mjs @@ -33,6 +33,25 @@ export function fingerprint(linkKey, salt) { * An absent or empty list needs no salt (nothing to fingerprint), so the common path stays friction-free. */ export function loadSuppressions(listPath, salt, keyVersion = SUPPRESSION_KEY_VERSION) { + return loadFingerprintedList(listPath, salt, keyVersion, 'suppression'); +} + +/** + * The MONOTONICITY-gate twin (ADR-0033 decision 6): links whose removal is licensed because their + * INPUT was wrong, not because the rules changed. + * + * A separate list from the suppressions, because the two are not the same act and cannot share one. + * A suppression keeps a built link OUT of the public surface; a correction says the link should never + * have been built at all — so correcting the input unbuilds it, and a suppression on it would then + * match nothing and trip the B3 unused gate. Same fingerprinting for the same reason: the raw + * `pid|eik` records which named official was tied to which company, which is precisely what must not + * live in git history. Consumed by load.mjs, which flags the matching keys in the pre-wipe snapshot. + */ +export function loadCorrections(listPath, salt, keyVersion = SUPPRESSION_KEY_VERSION) { + return loadFingerprintedList(listPath, salt, keyVersion, 'correction'); +} + +function loadFingerprintedList(listPath, salt, keyVersion, label) { const entries = existsSync(listPath) ? readFileSync(listPath, 'utf8') .split('\n') @@ -42,14 +61,14 @@ export function loadSuppressions(listPath, salt, keyVersion = SUPPRESSION_KEY_VE : []; if (entries.length > 0 && !salt) { throw new Error( - `${entries.length} suppression(s) in ${listPath} but SUPPRESSION_SALT is unset — refusing to build ` + + `${entries.length} ${label}(s) in ${listPath} but SUPPRESSION_SALT is unset — refusing to build ` + `(would silently un-suppress contested links). Set SUPPRESSION_SALT and retry.`, ); } for (const e of entries) { if (String(e.key_version ?? '') !== String(keyVersion)) { throw new Error( - `suppression ${String(e.fp ?? '').slice(0, 12)}… has key_version ${JSON.stringify(e.key_version)} ` + + `${label} ${String(e.fp ?? '').slice(0, 12)}… has key_version ${JSON.stringify(e.key_version)} ` + `but the current SUPPRESSION_KEY_VERSION is ${keyVersion} — refusing to build (a rotated salt ` + `would silently un-suppress it). Re-fingerprint every entry under the current salt + key_version.`, ); diff --git a/scripts/cacbg/tr-census.mjs b/scripts/cacbg/tr-census.mjs deleted file mode 100644 index e7b8c66d..00000000 --- a/scripts/cacbg/tr-census.mjs +++ /dev/null @@ -1,195 +0,0 @@ -// TR name-uniqueness census (ADR-0015). Promotes tier-C held interest_links — generic company names -// with a single WINNER namesake — to 'published' iff the name is GLOBALLY unique in the Trade Register. -// -// Source: the Commercial Register open-data dump on data.egov.bg (DPA-safe: ЕГН/ЛНЧ hashed out, company -// name + ЕИК retained). We build companyNameKey(name) → {distinct ЕИК} over ALL active entities using -// the SAME normalizer as the matcher, so the key spaces are identical. Promotion is deterministic: -// key count == 1 AND that ЕИК == the matched winner ЕИК. Anything else stays held. No heuristic. -// -// Field-detection is structural (find the 9/13-digit ЕИК; find the company-name field), so it is robust -// to the exact open-data column names — confirm the mapping against the real dump before a production run. -// -// Run: node --import ./scripts/cacbg/register-ts.mjs scripts/cacbg/tr-census.mjs --dump -import { DatabaseSync } from 'node:sqlite'; -import fs from 'node:fs'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); -const DB = process.env.CACBG_DB || path.join(ROOT, 'data/work/backfill.sqlite'); -const arg = (n) => { - const i = process.argv.indexOf(`--${n}`); - return i >= 0 ? process.argv[i + 1] : undefined; -}; -const DUMP = arg('dump') || process.env.TR_DUMP; -const { companyNameKey } = await import('../../packages/shared/src/company-name-key.ts'); - -const isEik = (v) => typeof v === 'string' && /^\d{9}$|^\d{13}$/.test(v.trim()); - -// The register stores CompanyName WITHOUT the legal form and LegalForm as a code; our matcher keys on the -// name WITH its Bulgarian form (bidders.name = «…» ЕООД), so we must reconstruct it or every lookup misses. -// Codes seen in the real dump: EOOD OOD ASSOC CC K EAD ET AD KCHT FOUND KD SD DPK EDPK IAD IEAD. -const LEGAL_FORM = { - EOOD: 'ЕООД', - OOD: 'ООД', - EAD: 'ЕАД', - AD: 'АД', - ET: 'ЕТ', - KD: 'КД', - K: 'КД', - SD: 'СД', - KCHT: 'КДА', - IAD: 'ИАД', - IEAD: 'ИЕАД', - DPK: 'ПК', - EDPK: 'ЕПК', - CC: 'кооперация', - ASSOC: 'СНЦ', - FOUND: 'фондация', -}; -// A Bulgarian trade name reconstructed to match the matcher's key space. Unknown form → append the raw -// code (harmless: it just forms its own key that no bidder lookup will hit). -const bgName = (companyName, legalForm) => - `${String(companyName ?? '').trim()} ${LEGAL_FORM[legalForm] ?? String(legalForm ?? '').trim()}`.trim(); - -// Extract {eik, name} from ONE TR deed's attribute object ({CompanyName, LegalForm, UIC, DeedStatus, …}). -// Also tolerates a flat {eik/uic, name} record (test fixtures). ЕИК = the 9/13-digit UIC. -export function extractEntity(rec) { - const uic = rec.UIC ?? rec.uic ?? rec.eik ?? rec.EIK; - const eik = isEik(String(uic ?? '')) ? String(uic).trim() : null; - const name = - rec.CompanyName != null ? bgName(rec.CompanyName, rec.LegalForm) : (rec.name ?? null); - return { eik, name: name || null }; -} - -// Read a TR dump as deed-attribute records. The real data.egov.bg export nests them at -// Message[].Body[].Deeds[].Deed[], each carrying its fields under `$` (fast-xml/attr style). Falls back to -// a plain JSON array / {data:[…]} / JSONL of already-flat records so tests and other shapes still work. -function* records(dump) { - const raw = fs.readFileSync(dump, 'utf8'); - const trimmed = raw.trimStart(); - if (trimmed.startsWith('{') || trimmed.startsWith('[')) { - const parsed = JSON.parse(raw); - const deeds = parsed?.Message?.[0]?.Body?.[0]?.Deeds?.[0]?.Deed; - if (Array.isArray(deeds)) { - for (const d of deeds) yield d?.['$'] ?? d; - return; - } - const arr = Array.isArray(parsed) - ? parsed - : (parsed.data ?? parsed.records ?? parsed.result ?? []); - yield* arr; - } else { - for (const line of raw.split('\n')) { - const t = line.trim(); - if (t) yield JSON.parse(t); - } - } -} - -// Build companyNameKey → Set(ЕИК) over one or more dump files (accumulates — call with every daily file to -// approach the full register). Deduped by ЕИК within a key, so the same company appearing on many days -// (or a name→one company) counts once; two DISTINCT companies sharing a name yield size>1 = not unique. -export function buildCensus(dumps) { - const census = new Map(); - for (const dump of Array.isArray(dumps) ? dumps : [dumps]) { - for (const rec of records(dump)) { - const { eik, name } = extractEntity(rec); - if (!eik || !name) continue; - const key = companyNameKey(name); - if (!census.has(key)) census.set(key, new Set()); - census.get(key).add(eik); - } - } - return census; -} - -// Promote held tier-C links the census proves globally unique: name-key maps to EXACTLY one ЕИК AND that -// ЕИК is the matched winner's (a national namesake, even at a different ЕИК, keeps the link held). With -// dryRun, report the would-promote set without writing — used to validate against a PARTIAL census, which -// must never actually promote (an un-ingested namesake would make a false-unique claim = libel). -export function promote(db, census, { dryRun = false, minEik = null, forcePartial = false } = {}) { - // Partial-census guard (libel gate). A census smaller than the real ТР register can make a genuinely - // non-unique name look unique (`eiks.size === 1`) → a false, libelous attribution. A real promote must - // ASSERT coverage in code, not by convention: pass `--min-eik ≥ register size` (the census's distinct-ЕИК - // count must meet it) or the explicit `--force-partial` escape hatch. dryRun is exempt — it writes nothing - // and exists precisely to inspect a partial census. - if (!dryRun && !forcePartial) { - if (!Number.isInteger(minEik) || minEik < 1) { - throw new Error( - 'refusing to promote without --min-eik : a partial census silently fabricates ' + - 'false-unique attributions. Pass --min-eik ≥ the known ТР register size, or --force-partial to override.', - ); - } - const distinctEik = new Set(); - for (const s of census.values()) for (const e of s) distinctEik.add(e); - if (distinctEik.size < minEik) { - throw new Error( - `refusing to promote from a partial census: ${distinctEik.size} distinct ЕИК < --min-eik ${minEik}. ` + - 'Ingest more ТР dumps until coverage meets the register size, or pass --force-partial to override.', - ); - } - } - const held = db - .prepare( - "SELECT link_key, eik, entity_key FROM interest_links WHERE status='held' AND publish_tier='C_hold'", - ) - .all(); - const upd = db.prepare( - "UPDATE interest_links SET status='published', match_method='exact_name_key+tr_census' WHERE link_key=?", - ); - const would = []; - if (!dryRun) db.exec('BEGIN'); - for (const l of held) { - const eiks = census.get(l.entity_key); - if (eiks && eiks.size === 1 && eiks.has(l.eik)) { - would.push(l.link_key); - if (!dryRun) upd.run(l.link_key); - } - } - if (!dryRun) db.exec('COMMIT'); - return { promoted: would.length, stillHeld: held.length - would.length, would }; -} - -// Resolve --dump: a single file, a comma-list, or --dump-dir (every *.json/*.jsonl inside). -function dumpFiles() { - const dir = arg('dump-dir'); - if (dir) - return fs - .readdirSync(dir) - .filter((f) => /\.(json|jsonl)$/.test(f)) - .map((f) => path.join(dir, f)); - return DUMP - ? DUMP.split(',') - .map((s) => s.trim()) - .filter(Boolean) - : []; -} - -// CLI entry (guarded so importing this module in tests has no side effects) -const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); -if (isMain) { - const files = dumpFiles(); - const dryRun = process.argv.includes('--dry-run'); - const forcePartial = process.argv.includes('--force-partial'); - const minEik = arg('min-eik') !== undefined ? Number(arg('min-eik')) : null; - if (!files.length) { - console.log( - 'no --dump/--dump-dir provided; census not run. Provide TR open-data files to promote tier-C links.', - ); - } else { - const db = new DatabaseSync(DB); - const census = buildCensus(files); - const eikTotal = [...census.values()].reduce((n, s) => n + s.size, 0); - console.log( - `census: ${files.length} file(s) → ${census.size} distinct name-keys / ${eikTotal} ЕИК`, - ); - const { promoted, stillHeld, would } = promote(db, census, { dryRun, minEik, forcePartial }); - console.log( - `${dryRun ? 'DRY-RUN would promote' : 'tier-C promotions'}: ${promoted} published, ${stillHeld} still held (non-unique or namesake mismatch)`, - ); - if (dryRun && would.length) - console.log(' would-promote link_keys:\n ' + would.join('\n ')); - db.close(); - } -} diff --git a/scripts/cacbg/tr-census.test.mjs b/scripts/cacbg/tr-census.test.mjs deleted file mode 100644 index 76d2fbb0..00000000 --- a/scripts/cacbg/tr-census.test.mjs +++ /dev/null @@ -1,121 +0,0 @@ -// TR census — real data.egov.bg deed shape + deterministic tier-C promotion. -// Run: node --import ./scripts/cacbg/register-ts.mjs --test scripts/cacbg/tr-census.test.mjs -import { test, before, after } from 'node:test'; -import assert from 'node:assert/strict'; -import { DatabaseSync } from 'node:sqlite'; -import fs from 'node:fs'; -import path from 'node:path'; -import os from 'node:os'; -import { extractEntity, buildCensus, promote } from './tr-census.mjs'; -const { companyNameKey } = await import('../../packages/shared/src/company-name-key.ts'); - -let dir, dump, dbPath; -const K = (s) => companyNameKey(s); - -before(() => { - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-census-')); - dump = path.join(dir, 'tr.json'); - // Real export shape: Message[].Body[].Deeds[].Deed[], each with its fields under `$`; CompanyName is - // bare and LegalForm is a code — the census must reconstruct «name + Bulgarian form» to match bidders. - const deeds = [ - { CompanyName: 'СИЙ', LegalForm: 'AD', UIC: '444444447' }, // globally unique → promote - { CompanyName: 'ОБЩА ФИРМА', LegalForm: 'OOD', UIC: '555555556' }, // one of two namesakes - { CompanyName: 'Обща Фирма', LegalForm: 'OOD', UIC: '666666663' }, // second namesake → NOT unique - { CompanyName: 'СИЙ', LegalForm: 'AD', UIC: '444444447' }, // same deed on another day → deduped - ]; - fs.writeFileSync( - dump, - JSON.stringify({ Message: [{ Body: [{ Deeds: [{ Deed: deeds.map((d) => ({ $: d })) }] }] }] }), - ); - - dbPath = path.join(dir, 'db.sqlite'); - const db = new DatabaseSync(dbPath); - db.exec(` - CREATE TABLE interest_links(link_key TEXT PRIMARY KEY, eik TEXT, entity_key TEXT, status TEXT, publish_tier TEXT, match_method TEXT); - INSERT INTO interest_links VALUES ('p1|444444447','444444447','${K('СИЙ АД')}','held','C_hold','exact_name_key'); - INSERT INTO interest_links VALUES ('p2|555555556','555555556','${K('ОБЩА ФИРМА ООД')}','held','C_hold','exact_name_key'); - `); - db.close(); -}); - -after(() => fs.rmSync(dir, { recursive: true, force: true })); - -test('extractEntity reconstructs «name + Bulgarian legal form» from a TR deed and reads UIC as ЕИК', () => { - assert.deepEqual(extractEntity({ CompanyName: 'СИЙ', LegalForm: 'AD', UIC: '444444447' }), { - eik: '444444447', - name: 'СИЙ АД', - }); - assert.deepEqual(extractEntity({ CompanyName: 'ТЕСТ', LegalForm: 'EOOD', UIC: '205057459' }), { - eik: '205057459', - name: 'ТЕСТ ЕООД', - }); - assert.equal(extractEntity({ CompanyName: 'X', LegalForm: 'AD', UIC: '2018060520' }).eik, null); // 10 digits ≠ ЕИК - assert.equal(extractEntity({ name: 'flat ООД', uic: '5555555560000' }).eik, '5555555560000'); // flat fallback + 13-digit -}); - -test('buildCensus indexes name-key → set of ЕИК over the nested export (dedup by ЕИК)', () => { - const c = buildCensus(dump); - assert.equal(c.get(K('СИЙ АД')).size, 1); // appears twice, one ЕИК → deduped - assert.equal(c.get(K('ОБЩА ФИРМА ООД')).size, 2); // two distinct ЕИК fold to one key → non-unique -}); - -test('promote publishes only globally-unique tier-C links; shared names stay held', () => { - const db = new DatabaseSync(dbPath); - // census covers 3 distinct ЕИК → assert coverage with --min-eik 3 (the partial-census gate). - const res = promote(db, buildCensus(dump), { minEik: 3 }); - assert.equal(res.promoted, 1); - const rows = new Map( - db - .prepare('SELECT link_key,status,match_method FROM interest_links') - .all() - .map((r) => [r.link_key, r]), - ); - assert.equal(rows.get('p1|444444447').status, 'published'); - assert.match(rows.get('p1|444444447').match_method, /tr_census/); - assert.equal(rows.get('p2|555555556').status, 'held'); // two namesakes → stays held - db.close(); -}); - -test('partial-census gate: a real promote refuses without --min-eik, and when coverage is below it', () => { - const db = new DatabaseSync(dbPath); - // isolate: p1 back to held (an earlier test may have published it — shared dbPath) - db.exec( - "UPDATE interest_links SET status='held', match_method='exact_name_key' WHERE link_key='p1|444444447'", - ); - const census = buildCensus(dump); // 3 distinct ЕИК - // no --min-eik → refuse (can't assert completeness → could fabricate a false-unique attribution) - assert.throws(() => promote(db, census, {}), /--min-eik/); - // --min-eik above the census's coverage → refuse (partial dump) - assert.throws(() => promote(db, census, { minEik: 4 }), /partial census|distinct ЕИК/); - // the refused runs published nothing — p1 is still held - assert.equal( - db.prepare("SELECT status FROM interest_links WHERE link_key='p1|444444447'").get().status, - 'held', - ); - db.close(); -}); - -test('partial-census gate: --force-partial is the explicit override', () => { - const db = new DatabaseSync(dbPath); - db.exec( - "UPDATE interest_links SET status='held', match_method='exact_name_key' WHERE link_key='p1|444444447'", - ); - const res = promote(db, buildCensus(dump), { forcePartial: true }); // no minEik, but forced - assert.equal(res.promoted, 1); - db.close(); -}); - -test('dry-run reports would-promote set without mutating the DB', () => { - const db = new DatabaseSync(dbPath); - // reset p1 back to held for an isolated dry-run check - db.exec( - "UPDATE interest_links SET status='held', match_method='exact_name_key' WHERE link_key='p1|444444447'", - ); - const res = promote(db, buildCensus(dump), { dryRun: true }); - assert.deepEqual(res.would, ['p1|444444447']); - assert.equal( - db.prepare("SELECT status FROM interest_links WHERE link_key='p1|444444447'").get().status, - 'held', - ); // untouched - db.close(); -}); diff --git a/scripts/cli-ts-imports.test.mjs b/scripts/cli-ts-imports.test.mjs new file mode 100644 index 00000000..3b59e5f1 --- /dev/null +++ b/scripts/cli-ts-imports.test.mjs @@ -0,0 +1,69 @@ +// The plain-Node CLI scripts import shared logic straight out of packages/ as raw TypeScript, which +// Node type-strips on the fly. That resolver is far stricter than the bundler one the vitest suites +// run under: vite happily resolves an extensionless relative specifier, plain Node does not. So an +// import added to a shared module can be green across every package suite and still make +// `node scripts/load-fx.mjs` die at startup with ERR_MODULE_NOT_FOUND — which is exactly what +// happened while writing the drain fix in this branch, and nothing in CI noticed. +// +// This test closes that gap the only way that proves anything: it resolves each specifier the way +// Node's ESM loader does (relative to the importing script) and imports it in a child process with +// no hooks, no register shim and no bundler, asserting the whole graph loads. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { readdirSync, readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const here = dirname(fileURLToPath(import.meta.url)); + +// Every `import … from ''` in a CLI script that reaches outside scripts/ — any extension or +// none, since an extensionless one is precisely the shape that breaks. Only the top-level scripts: +// scripts/cacbg/* run under the register-ts hook and are a different contract. +const IMPORT_RE = /\bfrom\s+'(\.\.\/[^']+)'/g; + +function crossPackageImports() { + const found = []; + for (const name of readdirSync(here).sort()) { + if (!name.endsWith('.mjs') || name.endsWith('.test.mjs')) continue; + const file = resolve(here, name); + const src = readFileSync(file, 'utf8'); + for (const [, specifier] of src.matchAll(IMPORT_RE)) found.push({ name, file, specifier }); + } + return found; +} + +const imports = crossPackageImports(); + +test('the scan actually finds the cross-package CLI imports', () => { + // Without this the whole file degrades to a no-op the moment the regex or the layout drifts — + // zero pairs would mean zero assertions and a green run. The three known importers today are + // import.mjs, load-eop.mjs and load-fx.mjs. + assert.ok(imports.length >= 3, `expected cross-package imports, found ${imports.length}`); + const importers = new Set(imports.map((i) => i.name)); + for (const expected of ['import.mjs', 'load-eop.mjs', 'load-fx.mjs']) { + assert.ok(importers.has(expected), `${expected} should import shared package code`); + } +}); + +for (const { name, file, specifier } of imports) { + test(`${name} → ${specifier} loads under plain node`, () => { + // ESM relative resolution IS new URL(specifier, parent) — no extension search, no index lookup. + // Doing it here reproduces the loader's answer for the specifier exactly as the script wrote it. + const target = new URL(specifier, pathToFileURL(file)).href; + try { + execFileSync( + 'node', + ['--input-type=module', '-e', `await import(${JSON.stringify(target)})`], + { + encoding: 'utf8', + stdio: 'pipe', + }, + ); + } catch (e) { + assert.fail( + `${name} cannot load ${specifier} under plain node — the CLI would die at startup:\n${e.stderr}`, + ); + } + }); +} diff --git a/scripts/derive-amendments.sql b/scripts/derive-amendments.sql index 2ab13245..217490ea 100644 --- a/scripts/derive-amendments.sql +++ b/scripts/derive-amendments.sql @@ -1,16 +1,134 @@ -- Sigma — roll raw_amendments up onto raw_contracts. -- Run AFTER scripts/load-eop.mjs (which stages the EOP base + in-bucket OCDS amendments). --- Re-runnable: resets the rollup, then matches amendments by (unp, contract_number). +-- Re-runnable. On the full-derive path scripts/resolve-amendment-contracts.sql (#306) has already rewritten +-- namespace-mismatched EOP annexes onto their target contract_number BEFORE this file runs. This file then +-- recovers the УНП for OCDS amendments via the tender.id bridge (#286), prefers the EOP annex over its OCDS +-- twin (dropping the twin), and matches amendments by (unp, contract_number). -- current_value = the after-value of the LATEST amendment; annex_count = how many. -- Contracts without amendments keep annex_count = 0 and current_value = NULL (the -- convention downstream is COALESCE(current_value, signing_value)). +-- #286: OCDS amendments stage the OCID in `unp` (the УНП is absent from the OCDS release), so they +-- match no contract. Recover the real УНП through the same bridge the OCDS-lots enrichment uses +-- (normalize-raw.sql): OCDS tender.id (staged as raw_amendments.tender_ext_id) → EOP tenderId → УНП. +-- The EOP tenderId lives in raw_tenders.tender_id for procedures in the поръчки feed AND in +-- raw_contracts.tender_ext_id for procedures that appear only as contracts (the "synthetic tenders" +-- of normalize-raw §2b) — so try raw_tenders first, then fall back to raw_contracts, else leave the +-- ocid untouched. The ocid stays only as a surrogate. Idempotent: a full run re-stages raw_amendments +-- and recomputes the same УНП. Being the first amendment step in the full pipeline, it leaves +-- raw_amendments corrected for promote-amendments.sql. +-- KEEP THE BRIDGE UPDATE BELOW IN LOCKSTEP with scripts/refresh-slice.sql: the UPDATE between the +-- @bridge-lockstep markers must stay byte-for-byte equivalent across both scripts (enforced by +-- packages/db/src/amendments-bridge-lockstep.test.ts). The prefer-EOP dedup DELETE deliberately does NOT +-- match — the slice path additionally reconciles against the cumulative served `amendments`, which this +-- full path never needs (promote-amendments.sql rebuilds it from scratch). raw_tenders(tender_id) is +-- indexed in work-staging-schema.sql, but raw_contracts(tender_ext_id) is not, so the fallback lookups +-- below would full-scan raw_contracts once per OCDS row — index it defensively (mirrors the lots bridge, +-- which indexes raw_tenders.tender_id before its join in normalize-raw.sql). The WHERE bridges only when +-- the chosen source maps the tender_ext_id to exactly ONE distinct УНП — see the refuse-on-ambiguity note +-- inside the block; the ocds_ambiguous_bridges diagnostic below surfaces any refusal. +CREATE INDEX IF NOT EXISTS idx_raw_contracts_tender_ext_id ON raw_contracts(tender_ext_id); +-- @bridge-lockstep start +UPDATE raw_amendments +SET unp = COALESCE( + (SELECT rt.unp FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL ORDER BY rt.unp LIMIT 1), + (SELECT rc.unp FROM raw_contracts rc + WHERE rc.tender_ext_id = raw_amendments.tender_ext_id AND rc.unp IS NOT NULL ORDER BY rc.unp LIMIT 1) +) +WHERE source LIKE 'ocds:%' + AND tender_ext_id IS NOT NULL + AND ( + -- raw_tenders wins when it resolves the procedure to exactly ONE УНП; else fall back to raw_contracts + -- when IT is unambiguous. Refuse to bridge (leave the OCID as an honest residual) when the chosen + -- source maps one tender_ext_id to more than one distinct УНП — the domain is 1-to-1, so this guards a + -- feed anomaly rather than silently mis-attributing every annex of the losing procedure (issue #286). + (SELECT COUNT(DISTINCT rt.unp) FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL) = 1 + OR ( + NOT EXISTS (SELECT 1 FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL) + AND (SELECT COUNT(DISTINCT rc.unp) FROM raw_contracts rc + WHERE rc.tender_ext_id = raw_amendments.tender_ext_id AND rc.unp IS NOT NULL) = 1 + ) + ); +-- @bridge-lockstep end + +-- #286 diagnostic (printed by wrangler, review nikimilenkov LOW 1): count OCDS amendments the bridge +-- REFUSED because their tender_ext_id resolves to more than one distinct УНП in the chosen source. The +-- domain is 1-to-1, so this must be 0 on a healthy feed; a non-zero value is a feed anomaly to investigate, +-- not a silent mis-attribution. Runs after the bridge, so refused rows still carry their OCID. +SELECT COUNT(*) AS ocds_ambiguous_bridges +FROM raw_amendments o +WHERE o.source LIKE 'ocds:%' + AND o.tender_ext_id IS NOT NULL + AND o.unp LIKE 'ocds-%' + AND ( + (SELECT COUNT(DISTINCT rt.unp) FROM raw_tenders rt + WHERE rt.tender_id = o.tender_ext_id AND rt.unp IS NOT NULL) > 1 + OR ( + NOT EXISTS (SELECT 1 FROM raw_tenders rt + WHERE rt.tender_id = o.tender_ext_id AND rt.unp IS NOT NULL) + AND (SELECT COUNT(DISTINCT rc.unp) FROM raw_contracts rc + WHERE rc.tender_ext_id = o.tender_ext_id AND rc.unp IS NOT NULL) > 1 + ) + ); + +-- #286 diagnostic (printed by wrangler, BEFORE the drop below): keep the residual OCDS under-count +-- OBSERVABLE. The prefer-EOP dedup is contract-level — it drops EVERY OCDS annex on a contract that +-- already has an EOP annex, so a genuinely OCDS-only *extra* amendment there is lost. Per-annex twin +-- matching can't rescue it: OCDS document_number is the release id (ocds-…) while EOP's is the АОП +-- document number — different id spaces that never align, so we cannot prove per-annex which drops are +-- true twins. Report bounds instead (run after the bridge above, so o.unp is the recovered УНП): +-- dropped = every OCDS annex removed by the dedup (UPPER bound on annexes lost) +-- excess_over_eop = Σ max(0, ocds_on_contract − eop_on_contract) (LOWER bound: cannot all be twins) +SELECT + COALESCE(SUM(g.ocds_n), 0) AS ocds_annexes_dropped, + COALESCE(SUM(CASE WHEN g.ocds_n > g.eop_n THEN g.ocds_n - g.eop_n ELSE 0 END), 0) + AS ocds_annexes_excess_over_eop +FROM ( + SELECT o.unp, o.contract_number, + COUNT(*) AS ocds_n, + (SELECT COUNT(*) FROM raw_amendments e + WHERE e.source LIKE 'eop:%' AND e.unp = o.unp AND e.contract_number = o.contract_number) AS eop_n + FROM raw_amendments o + WHERE o.source LIKE 'ocds:%' + AND EXISTS (SELECT 1 FROM raw_amendments e + WHERE e.source LIKE 'eop:%' AND e.unp = o.unp AND e.contract_number = o.contract_number) + GROUP BY o.unp, o.contract_number +) g; + +-- #286: prefer the EOP annex. ~99% of OCDS amendments duplicate an EOP annex for the same contract; +-- keeping both would double annex_count and duplicate the served timeline. Drop the OCDS twin when an +-- EOP annex already exists for the same (unp, contract_number). Genuinely OCDS-only annexes survive +-- (they carry value_after = NULL from ingest, so they never drive current_value — issue #286). +DELETE FROM raw_amendments +WHERE source LIKE 'ocds:%' + AND EXISTS ( + SELECT 1 FROM raw_amendments e + WHERE e.source LIKE 'eop:%' + AND e.unp = raw_amendments.unp + AND e.contract_number = raw_amendments.contract_number + ); + +-- #306: the value-anchor resolver that links namespace-mismatched EOP annexes (annex-side number ≠ the +-- contract's filing number) by rewriting raw_amendments.contract_number lives in +-- scripts/resolve-amendment-contracts.sql and runs BEFORE this file on the full-derive path only (see that +-- file's header for the ordering-blocker and slice-gating rationale — reviews todorkolev #1/#3, nikimilenkov +-- HIGH 1). By the time this rollup runs, resolved annexes already carry their target contract_number (and +-- keep the original annex number in contract_number_raw for the natural_key below), so the rollup links them +-- with no further change. + UPDATE raw_contracts SET annex_count = 0, current_value = NULL; WITH keyed AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + -- #306: key on the ORIGINAL annex-side number (contract_number_raw) when the value resolver rewrote a + -- row, so a resolved annex keeps a stable identity and never collides with a native annex that shares + -- document_number on the target contract (review nikimilenkov MEDIUM 3). NULL raw → the plain number, + -- so unresolved rows and the slice path (which never resolves) produce byte-identical keys. + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), @@ -39,8 +157,10 @@ SET AND a.contract_number = raw_contracts.contract_number AND a.rn = 1 ), + -- #305 Tier-2: a text-confirmed double-count carries the corrected total in value_after_restated; use + -- it as the effective after so current_value reflects the true total, not the raw doubled value_after. current_value = ( - SELECT a.value_after FROM dedup a + SELECT COALESCE(a.value_after_restated, a.value_after) FROM dedup a WHERE a.unp = raw_contracts.unp AND a.contract_number = raw_contracts.contract_number AND a.value_after IS NOT NULL diff --git a/scripts/import-catchup.test.mjs b/scripts/import-catchup.test.mjs new file mode 100644 index 00000000..dc434472 --- /dev/null +++ b/scripts/import-catchup.test.mjs @@ -0,0 +1,117 @@ +// safeD1's missing-table detection, exercised through the real scripts/import.mjs. +// +// safeD1 has to tell "this table does not exist yet" (recoverable — latestLoadedDate falls back to +// data_freshness) apart from every other failure (must propagate). Getting that wrong breaks +// `import --catchup --plan-only` in its NORMAL state: drop-transient-staging removes raw_contracts in +// a finally, and --plan-only exits before the main flow recreates it. #277 fixed it; nothing tested it, +// because import.mjs runs on import and cannot be imported. The subprocess harness from #270 can. +// +// Run: node --test scripts/import-catchup.test.mjs +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(HERE, '..'); +const SCRIPT = resolve(HERE, 'import.mjs'); + +// VERIFIED AGAINST THE REAL BINARY, not imagined — the whole point of this test is the stream layout, +// so modelling it from memory would test nothing. Observed from +// `wrangler d1 execute sigma --local --json --command "SELECT 1 FROM no_such_table_xyz"`: +// +// exit code : 1 +// stdout : {"error":{"text":"no such table: no_such_table_xyz: SQLITE_ERROR"}} +// stderr : ▲ [WARNING] Processing wrangler.jsonc configuration: ... +// err.message from execFileSync : "Command failed: wrangler d1 execute ..." +// +// So the SQLite error is on STDOUT while wrangler's notices are on STDERR, and the exception message +// carries neither. That asymmetry is exactly what the old `err.message`-only check missed. +const SQLITE_ERROR_ON_STDOUT = '{"error":{"text":"no such table: raw_contracts: SQLITE_ERROR"}}'; + +const FAKE_WRANGLER = `#!${process.execPath} +import { appendFileSync } from 'node:fs'; +const argv = process.argv.slice(2); +appendFileSync(process.env.CU_LOG, JSON.stringify(argv) + '\\n'); +const ci = argv.indexOf('--command'); +if (ci !== -1) { + const sql = argv[ci + 1]; + if (/raw_contracts/.test(sql)) { + if (process.env.CU_OTHER_ERROR) { + process.stdout.write(JSON.stringify({ error: { text: 'database is locked: SQLITE_BUSY' } })); + } else { + process.stdout.write(${JSON.stringify(SQLITE_ERROR_ON_STDOUT)}); + } + process.stderr.write('\\u25b2 [WARNING] Processing wrangler.jsonc configuration:\\n'); + process.exit(1); + } + if (/data_freshness/.test(sql)) { + process.stdout.write(JSON.stringify([{ results: [{ max_loaded_date: '2026-07-27' }], success: true }])); + process.exit(0); + } + process.stdout.write(JSON.stringify([{ results: [], success: true }])); +} +process.exit(0); +`; + +// Stands in for the child scripts import.mjs shells out to, so a run that gets past planning stops +// here rather than reaching the network. The parent runs under process.execPath, so it keeps the real +// node while the child's PATH lookup finds this. +const FAKE_NODE = `#!${process.execPath} +process.exit(0); +`; + +function runImport(args, env = {}) { + const dir = mkdtempSync(join(tmpdir(), 'catchup-')); + try { + const binDir = join(dir, 'bin'); + mkdirSync(binDir); + writeFileSync(join(binDir, 'package.json'), '{"type":"module"}'); + for (const [name, src] of [ + ['wrangler', FAKE_WRANGLER], + ['node', FAKE_NODE], + ]) { + writeFileSync(join(binDir, name), src); + chmodSync(join(binDir, name), 0o755); + } + const log = join(dir, 'calls.log'); + writeFileSync(log, ''); + const res = spawnSync(process.execPath, [SCRIPT, ...args], { + cwd: ROOT, + encoding: 'utf8', + timeout: 30_000, + env: { ...process.env, PATH: `${binDir}:${process.env.PATH}`, CU_LOG: log, ...env }, + }); + const calls = readFileSync(log, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse); + return { status: res.status, stdout: res.stdout ?? '', stderr: res.stderr ?? '', calls }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +test('a missing raw_contracts falls back to data_freshness instead of crashing', () => { + // The reported failure: --plan-only in its normal steady state, with the transient staging gone. + const r = runImport(['--catchup', '--plan-only']); + assert.equal(r.status, 0, `--plan-only should survive a missing raw_contracts:\n${r.stderr}`); + assert.match(r.stdout, /catchup plan/); + // The date can only have come from the data_freshness fallback — raw_contracts never answered. + assert.match(r.stdout, /maxLoadedDate=2026-07-27/); +}); + +test('the fallback query is actually reached, not skipped', () => { + // Guards against a future "fix" that returns a plan without asking data_freshness at all, which + // would pass the assertion above by accident. + const r = runImport(['--catchup', '--plan-only']); + const asked = r.calls.some((c) => /data_freshness/.test(String(c[c.length - 1]))); + assert.ok(asked, 'safeD1 swallowed the error but nothing consulted data_freshness'); +}); + +test('an error that is NOT a missing table still propagates', () => { + // safeD1 must not become a blanket catch. A locked database is not "no corpus yet", and treating it + // as one would let the catch-up plan be computed from a lie. + const r = runImport(['--catchup', '--plan-only'], { CU_OTHER_ERROR: '1' }); + assert.notEqual(r.status, 0, 'a SQLITE_BUSY must not be swallowed as an empty result'); +}); diff --git a/scripts/import-guard.test.mjs b/scripts/import-guard.test.mjs new file mode 100644 index 00000000..f82c8b8c --- /dev/null +++ b/scripts/import-guard.test.mjs @@ -0,0 +1,201 @@ +// The --derive=full window guard, exercised through the real scripts/import.mjs. +// +// Testing the predicate alone is what let the first version ship: fullDeriveIsSafe() was green while +// the call site asked `SELECT COUNT(*) FROM contracts` and the clear it was protecting emptied +// fourteen tables. So this drives the actual script as a subprocess, with a fake `wrangler` and a +// fake `node` first on PATH, and asserts on what the script DID: whether it refused, which tables it +// named, whether it cleaned up after itself, and whether the load ever started. +// +// Run: node --test scripts/import-guard.test.mjs +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(HERE, '..'); +const SCRIPT = resolve(HERE, 'import.mjs'); + +// `wrangler` answers the probe from GUARD_FAKE_POPULATED and records every call; `node` stands in for +// the child scripts import.mjs shells out to (load-eop and friends), so a run that gets PAST the guard +// stops here instead of hitting the network. Launching the script under test with process.execPath +// keeps the real node for the parent while the child's PATH lookup finds the stub. +// The shebangs name the real interpreter outright. `#!/usr/bin/env node` would resolve through the +// very PATH these stubs sit at the front of, so the fake `node` would end up interpreting the fake +// `wrangler` and every answer would come back empty. +const FAKE_WRANGLER = `#!${process.execPath} +import { appendFileSync } from 'node:fs'; +const argv = process.argv.slice(2); +appendFileSync(process.env.GUARD_FAKE_LOG, JSON.stringify(argv) + '\\n'); +const ci = argv.indexOf('--command'); +if (argv.includes('--json') && ci !== -1) { + const sql = argv[ci + 1]; + const populated = (process.env.GUARD_FAKE_POPULATED || '').split(',').filter(Boolean); + const missing = (process.env.GUARD_FAKE_MISSING || '').split(',').filter(Boolean); + const aliases = [...sql.matchAll(/AS "([^"]+)"/g)].map((m) => m[1]); + if (aliases.length) { + const row = {}; + for (const a of aliases) if (!missing.includes(a)) row[a] = populated.includes(a) ? 1 : 0; + process.stdout.write(JSON.stringify([{ results: [row], success: true }])); + process.exit(0); + } + process.stdout.write(JSON.stringify([{ results: [], success: true }])); +} +process.exit(0); +`; + +const FAKE_NODE = `#!${process.execPath} +import { appendFileSync } from 'node:fs'; +appendFileSync(process.env.GUARD_FAKE_LOG, JSON.stringify(['node', ...process.argv.slice(2)]) + '\\n'); +process.exit(0); +`; + +function bin(dir, name, source) { + const file = join(dir, name); + writeFileSync(file, source); + chmodSync(file, 0o755); +} + +/** Runs the real import.mjs with the fakes in front, and returns what it did. */ +function runImport(args, { populated = '', missing = '' } = {}) { + const dir = mkdtempSync(join(tmpdir(), 'guard-')); + try { + const binDir = join(dir, 'bin'); + mkdirSync(binDir); + writeFileSync(join(binDir, 'package.json'), '{"type":"module"}'); + bin(binDir, 'wrangler', FAKE_WRANGLER); + bin(binDir, 'node', FAKE_NODE); + const log = join(dir, 'calls.log'); + writeFileSync(log, ''); + const res = spawnSync(process.execPath, [SCRIPT, ...args], { + cwd: ROOT, + encoding: 'utf8', + timeout: 30_000, + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + GUARD_FAKE_LOG: log, + GUARD_FAKE_POPULATED: populated, + GUARD_FAKE_MISSING: missing, + SIGMA_D1_NAME: 'sigma-test-local', + }, + }); + const calls = readFileSync(log, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse); + return { + status: res.status, + stderr: res.stderr ?? '', + calls, + refused: /refusing --derive=full/.test(res.stderr ?? ''), + loadStarted: calls.some((c) => c[0] === 'node' && String(c[1]).includes('load-eop')), + // The guard's own query, not merely any --json call: the derive paths issue plenty of their own. + probe: calls.find((c) => /EXISTS\(SELECT 1 FROM contracts\)/.test(String(c[c.length - 1]))), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +const PARTIAL = ['--from=2026-06-01', '--derive=full']; + +test('refuses a partial-window full derive over a populated corpus, before the load', () => { + const r = runImport(PARTIAL, { populated: 'contracts,lots,tenders,bidders,authorities' }); + assert.equal(r.refused, true, r.stderr); + assert.equal(r.status, 1); + assert.equal(r.loadStarted, false, 'the load must not start after a refusal'); + assert.match(r.stderr, /contracts/); +}); + +test('a corpus with NO contracts but populated tenders still refuses', () => { + // The regression this guard exists for. The previous call site asked COUNT(*) FROM contracts, so + // exactly this state - the one a half-failed run leaves behind - was waved through, and the rebuild + // would then drop tenders, bidders and authorities with nothing to reload them from. + const r = runImport(PARTIAL, { populated: 'tenders,bidders,authorities' }); + assert.equal(r.refused, true, r.stderr); + assert.equal(r.status, 1); + assert.match(r.stderr, /tenders/); + assert.doesNotMatch(r.stderr, /including [^\n]*\bcontracts\b/); +}); + +test('refuses when the probe cannot answer for every cleared table', () => { + // A missing table makes safeD1 return nothing at all, which would otherwise read as "no corpus". + const r = runImport(PARTIAL, { populated: 'contracts', missing: 'facet_counts' }); + assert.equal(r.status, 1); + assert.match(r.stderr, /could not read the corpus/); +}); + +test('an empty corpus is the initial backfill and passes', () => { + const r = runImport(PARTIAL, { populated: '' }); + assert.equal(r.refused, false, r.stderr); + assert.equal(r.loadStarted, true, 'the load should start when there is nothing to lose'); +}); + +test('a window reaching the start of the feed passes even over a full corpus', () => { + const r = runImport(['--from=2020-01-01', '--derive=full'], { + populated: 'contracts,authorities', + }); + assert.equal(r.refused, false, r.stderr); + assert.equal(r.loadStarted, true); +}); + +test('a slice derive is never probed at all', () => { + const r = runImport(['--from=2026-06-01', '--derive=slice'], { populated: 'contracts' }); + assert.equal(r.refused, false, r.stderr); + assert.equal(r.probe, undefined, 'slice derives must not pay for the corpus probe'); +}); + +// Written out by hand ON PURPOSE. The first version of this test re-derived the list from +// normalize-raw.sql using its own copy of the parser's regex — so the oracle inherited the parser's +// blind spot, and rewriting one line as `DELETE FROM "search_index";` (valid SQLite, invisible in +// review) dropped that table out of the probe with both suites still green. A list that shares the +// implementation's assumptions cannot test them. +const CLEARED = [ + 'search_index', + 'flow_pairs', + 'company_totals', + 'authority_joint_participation', + 'authority_totals', + 'sector_totals', + 'facet_counts', + 'home_totals', + 'contract_co_authorities', + 'contracts', + 'lots', + 'tenders', + 'bidders', + 'authorities', +]; + +test('the probe covers every table the SQL clears, not a subset', () => { + const r = runImport(PARTIAL, { populated: 'contracts' }); + const sql = String(r.probe[r.probe.indexOf('--command') + 1]); + for (const table of CLEARED) { + assert.match(sql, new RegExp(`FROM ${table}\\)`), `probe is missing ${table}`); + } +}); + +test('the hand-written list still matches what the SQL clears', () => { + // The other half of the cross-check: the list above holds the parser to account, and this holds the + // list to account. Either one drifting is caught here instead of in production. The matcher is + // deliberately loose about quoting so that a re-quoted table shows up as a MISMATCH rather than + // vanishing the way it did from the first version. + const sql = readFileSync(resolve(ROOT, 'scripts/normalize-raw.sql'), 'utf8'); + const marker = sql.indexOf('-- @full-clear'); + assert.notEqual(marker, -1, 'normalize-raw.sql lost its @full-clear marker'); + const block = sql.slice(marker).split(/\r?\n\s*\r?\n/)[0]; + const deletes = [...block.matchAll(/DELETE\s+FROM\s+(.+?)\s*;/gi)].map((m) => + m[1].replace(/^["`[]/, '').replace(/["`\]]$/, ''), + ); + assert.deepEqual(deletes, CLEARED); +}); + +test('a refusal leaves no transient staging behind', () => { + const r = runImport(PARTIAL, { populated: 'contracts' }); + const files = r.calls.filter((c) => c.includes('--file')).map((c) => c[c.indexOf('--file') + 1]); + assert.ok( + files.some((f) => /drop-transient-staging\.sql$/.test(String(f))), + 'the guard should tear down the staging it found created', + ); +}); diff --git a/scripts/import.mjs b/scripts/import.mjs index 578a1132..fe2aba16 100644 --- a/scripts/import.mjs +++ b/scripts/import.mjs @@ -14,9 +14,14 @@ import { } from 'node:fs'; import { basename, dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { computeCatchupWindow, daysInWindow } from '../packages/ingest/src/ocds.ts'; +import { + computeCatchupWindow, + daysInWindow, + fullDeriveIsSafe, +} from '../packages/ingest/src/ocds.ts'; import { dropTransientStagingStatements, + fullClearTables, refreshSliceStatementGroups, } from '../packages/ingest/src/refresh.ts'; import { assertIntegrity } from './integrity-checks.mjs'; @@ -37,7 +42,6 @@ function reportAnomalies(runner, label) { const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const apiDir = resolve(root, 'apps/web'); const DEFAULT_FROM = '2020-01-01'; -const LARGE_GAP_DAYS = 14; const DEFAULT_LOOKBACK_DAYS = 3; const remote = process.argv.includes('--remote'); @@ -127,7 +131,8 @@ function safeD1(sql) { try { return d1(sql); } catch (err) { - const msg = String(err?.message ?? err); + // wrangler writes the SQLITE error to stdout, not the exception message. + const msg = `${err?.message ?? err} ${err?.stdout ?? ''} ${err?.stderr ?? ''}`; if (/no such table|does not exist/i.test(msg)) return []; throw err; } @@ -215,12 +220,11 @@ function resolveCatchupPlan() { const to = String(arg('to') || window.to); const gapDays = daysInWindow(from, to); const requestedDerive = arg('derive'); - const derive = - requestedDerive && requestedDerive !== true - ? String(requestedDerive) - : gapDays > LARGE_GAP_DAYS - ? 'full' - : 'slice'; + // The catch-up window is gap-aware, so it only ever covers the tail of the feed. A full derive + // rebuilds `contracts` from staging (normalize-raw.sql opens with DELETE FROM contracts), which + // would drop every contract older than the window — so catch-up always derives a slice, however + // wide the gap. An operator who really has loaded the whole feed can still pass --derive=full. + const derive = requestedDerive && requestedDerive !== true ? String(requestedDerive) : 'slice'; return { from, to, maxLoadedDate, gapDays, derive }; } @@ -229,7 +233,64 @@ function validateDeriveMode(mode) { throw new Error(`unknown --derive=${mode}; expected full|slice`); } +// Refuse the one combination that silently destroys data: a full derive (which rebuilds the domain +// from staging) driven by a window that does not reach back to the start of the feed. Anything the +// window misses is deleted and never reloaded. Checked before the load, and the refusal tears the +// transient staging back down: it cannot run any earlier, because the catch-up plan reads +// raw_contracts, but it must not leave a half-built schema behind for a run that never started. +// +// The question is asked of EVERY table the full clear empties, not of `contracts` alone: a corpus +// with no contracts but populated tenders, bidders or authorities is exactly the state a half-failed +// run leaves behind, and the narrow-window rebuild would then wipe those too while the guard waved +// it through. The list comes out of normalize-raw.sql itself (see @full-clear there). +function assertDeriveWindowSafe(mode, from) { + if (mode !== 'full') return; + const tables = fullClearTables(readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8')); + if (tables.length === 0) + throw new Error( + 'normalize-raw.sql has no @full-clear block — refusing to guess what it clears', + ); + // EXISTS per table, not COUNT(*) over the union: this runs on every full derive and must stay + // cheap on a corpus of hundreds of thousands of rows. + const probe = tables.map((t) => `(SELECT EXISTS(SELECT 1 FROM ${t})) AS "${t}"`).join(', '); + const row = safeD1(`SELECT ${probe}`)[0]; + // safeD1 turns a missing table into an empty result, which would otherwise read as "no corpus" and + // wave the destructive path through — one absent table blinding the guard about the other thirteen. + // A probe that could not answer is not an answer: fail closed. + if (!row || tables.some((table) => !(table in row))) { + console.error( + `!! refusing --derive=full: could not read the corpus. Every table normalize-raw.sql clears ` + + `(${tables.join(', ')}) must be answerable before a rebuild may drop it.`, + ); + execSqlStatements(dropTransientStagingStatements(), 'drop-transient-staging'); + process.exit(1); + } + const populated = Object.entries(row) + .filter(([, present]) => Number(present) > 0) + .map(([table]) => table); + if ( + fullDeriveIsSafe({ windowFrom: from, feedStart: DEFAULT_FROM, hasCorpus: populated.length > 0 }) + ) + return; + console.error( + `!! refusing --derive=full: the load window starts ${from}, but the corpus is already ` + + `populated back to ${DEFAULT_FROM}.\n` + + ` A full derive rebuilds the domain from staging, so everything before ${from} would be ` + + `dropped and not reloaded — including ${populated.join(', ')}.\n` + + ` Use --derive=slice for an incremental refresh, or reload the whole feed with ` + + `--from=${DEFAULT_FROM}.`, + ); + execSqlStatements(dropTransientStagingStatements(), 'drop-transient-staging'); + process.exit(1); +} + async function runFullDerive() { + // #306: link namespace-mismatched EOP annexes by value BEFORE derive-amendments.sql (its prefer-EOP + // dedup would otherwise resurrect OCDS twins — review todorkolev #1). This STANDALONE script is the + // full-derive form (candidates from the whole re-staged raw_contracts). The daily/slice + Worker path runs + // an equivalent value anchor inside refresh-slice.sql that draws candidates from the served `contracts` + // corpus instead of the window (so "unique on the procedure" is corpus-wide — review nikimilenkov HIGH 1). + execSql(resolve(root, 'scripts/resolve-amendment-contracts.sql')); execSql(resolve(root, 'scripts/derive-amendments.sql')); run('node', ['scripts/load-fx.mjs', '--apply', ...passthru]); execSql(resolve(root, 'scripts/load-nuts.sql')); @@ -307,6 +368,8 @@ async function runWorkBackfill() { `--out=${resolve(workDir, `${stem}.eop-load.sql`)}`, ...loadFlags, ]); + // #306: value-anchor resolver runs first on the full-derive path — see runFullDerive. + sqliteFile(workDb, resolve(root, 'scripts/resolve-amendment-contracts.sql')); sqliteFile(workDb, resolve(root, 'scripts/derive-amendments.sql')); run('node', [ 'scripts/load-fx.mjs', @@ -370,19 +433,24 @@ if (arg('work-db') !== undefined) { console.log(`==> Sigma import (${remote ? 'REMOTE' : 'local'})`); run('wrangler', ['d1', 'migrations', 'apply', d1Name, loc, ...d1PersistArgs], apiDir); execSqlStatements(dropTransientStagingStatements(), 'drop-stale-transient-staging'); +// Must precede resolveCatchupPlan(): latestLoadedDate() reads raw_contracts, which lives here. execSql(resolve(root, 'scripts/work-staging-schema.sql')); let deriveMode = String(arg('derive') || 'full'); let loadFlags = explicitRangeFlags(); +// Mirrors load-eop.mjs, which also falls back to DEFAULT_FROM when no --from is given. +let windowFrom = String(arg('from') || DEFAULT_FROM); if (catchup) { const plan = resolveCatchupPlan(); deriveMode = plan.derive; loadFlags = rangeFlags(plan.from, plan.to); + windowFrom = plan.from; console.log( `==> catchup window ${plan.from}..${plan.to} (${plan.gapDays} days, latest=${plan.maxLoadedDate || 'none'}, derive=${deriveMode})`, ); } validateDeriveMode(deriveMode); +assertDeriveWindowSafe(deriveMode, windowFrom); run('node', ['scripts/load-eop.mjs', '--apply', ...loadFlags, ...passthru]); if (deriveMode === 'slice') await runSliceDerive(); diff --git a/scripts/integrity-checks.d.mts b/scripts/integrity-checks.d.mts index d358d0af..82a6207e 100644 --- a/scripts/integrity-checks.d.mts +++ b/scripts/integrity-checks.d.mts @@ -30,6 +30,7 @@ export function checkNoNegativeValues(runner: IntegrityRunner): Promise; export function checkDateSanity(runner: IntegrityRunner): Promise; export function checkStagingReconciliation(runner: IntegrityRunner): Promise; +export function checkAmendmentTwins(runner: IntegrityRunner): Promise; export const CHECKS: Array<(runner: IntegrityRunner) => Promise>; export function runIntegrityChecks(runner: IntegrityRunner): Promise; diff --git a/scripts/integrity-checks.mjs b/scripts/integrity-checks.mjs index b48c4912..e87a57de 100644 --- a/scripts/integrity-checks.mjs +++ b/scripts/integrity-checks.mjs @@ -372,6 +372,38 @@ export async function checkStagingReconciliation(runner) { }; } +// 7) Amendment twin dedup (#286). The OCDS→EOP bridge lets an OCDS amendment reach the same contract as +// its EOP twin; the prefer-EOP dedup keeps only one per (unp, contract_number) by DELETE-ing OCDS rows +// before promotion — the SOLE guard, since promotion is unconditional. On the incremental path a twin +// can straddle windows (EOP served earlier, OCDS arriving later), so the slice dedup reconciles against +// the served table; this gate is the post-condition of that (and of the full path): no (unp, +// contract_number) may carry BOTH an EOP and an OCDS served amendment, or annex_count double-counts. +export async function checkAmendmentTwins(runner) { + const name = 'amendment-twin-dedup'; + if (!(await tableExists(runner, 'amendments'))) + return { name, ok: true, skipped: true, detail: 'amendments table absent' }; + const n = num( + await scalar( + runner, + 'SELECT COUNT(*) AS n FROM (' + + 'SELECT unp, contract_number FROM amendments ' + + 'WHERE unp IS NOT NULL AND contract_number IS NOT NULL ' + + 'GROUP BY unp, contract_number ' + + "HAVING SUM(source LIKE 'eop:%') > 0 AND SUM(source LIKE 'ocds:%') > 0)", + 'n', + ), + ); + return { + name, + ok: n === 0, + skipped: false, + detail: + n === 0 + ? 'no (unp, contract_number) carries both an EOP and an OCDS amendment (prefer-EOP dedup intact)' + : `${n} (unp, contract_number) carry both an EOP and an OCDS amendment — prefer-EOP dedup regressed and annex_count double-counts (#286)`, + }; +} + export const CHECKS = [ checkNonEmptyCorpus, checkRollupReconciliation, @@ -380,6 +412,7 @@ export const CHECKS = [ checkEikValidity, checkDateSanity, checkStagingReconciliation, + checkAmendmentTwins, ]; export async function runIntegrityChecks(runner) { diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index ebfef975..246e91ec 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -31,6 +31,10 @@ -- every contract has a parent. bids stays empty (the data has a bid COUNT, not bids). -- Full clear in child→parent order (D1 enforces FKs). +-- @full-clear — everything emptied between this marker and the blank line below is rebuilt from +-- staging alone, so a full derive driven by a partial window loses whatever the window misses. +-- scripts/import.mjs reads the list from here to decide whether refusing is warranted; keep new +-- DELETEs inside the block so the guard picks them up on its own. DROP TABLE IF EXISTS joint_tender_leads; DROP TABLE IF EXISTS unp_prefix_authorities; DROP TABLE IF EXISTS joint_authority_members; @@ -690,7 +694,8 @@ SET ownership_kind = ( -- 5) Contracts — awarded lines (1:1 with staging rows), linked to tender + winning bidder, -- with the data-quality verdict (see 0007_data_quality.sql): --- value_flag = 'value_suspect' effective value >2bn EUR, or >200× the procedure estimate +-- value_flag = 'value_suspect' effective value >2bn EUR, >200× the procedure estimate, or +-- inside the 95×–105× стотинки band (a dropped decimal point) -- when that estimate is at least 1000 EUR — repaired to the -- procedure estimate for sums/display -- | 'value_low' zero/negative, OR a tiny signed value (< 1000 EUR) that is also @@ -698,7 +703,9 @@ SET ownership_kind = ( -- LABELLED — large legitimate framework call-offs (a small share of -- a huge ceiling but big in absolute terms) are excluded by the -- < 1000 EUR floor, so they keep counting and stay unflagged --- | 'annex_suspect' amendment pushed current_value ≥100× signing, or negative → +-- | 'annex_suspect' amendment pushed current_value ≥100× signing, or negative, or +-- a single annex step jumped ≥10× while the aggregate ended ≥5× +-- over signing (a mis-keyed annex value) → -- fall back to signing_value, or current_value if signing is -- missing, so the contract still counts -- | 'review' ≥10× the procedure estimate (kept, but flagged) @@ -725,7 +732,10 @@ INSERT OR IGNORE INTO contracts -- travels alongside the value it minted — computed ONCE over raw_amendments, not per-row). WITH amendment_dedup AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + -- #306: key on the original annex-side number (contract_number_raw) when the value resolver rewrote a + -- row — must match derive-amendments.sql / promote-amendments.sql byte-for-byte (review nikimilenkov + -- MEDIUM 3). NULL raw → the plain number, so unresolved rows are unaffected. + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), @@ -812,6 +822,7 @@ FROM ( CASE y.value_flag WHEN 'value_suspect' THEN y.proc_est_native WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS display_native, -- value_suspect is repaired directly from proc_est_eur in the outer amount_eur CASE; value_low and @@ -819,6 +830,7 @@ FROM ( CASE y.value_flag WHEN 'value_suspect' THEN NULL WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, -- Keep the companion currency paired with the exact native value chosen above. In particular, @@ -830,6 +842,10 @@ FROM ( WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') ELSE COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') END + WHEN 'annex_total_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') + END ELSE CASE WHEN y.current_value IS NOT NULL THEN COALESCE(NULLIF(y.amendment_currency, ''), NULLIF(y.currency, ''), 'BGN') ELSE COALESCE(NULLIF(y.currency, ''), 'BGN') @@ -875,7 +891,20 @@ FROM ( CASE -- Over-valuation + absurd are checked FIRST and repaired to the procedure estimate. -- value_low is a labelled-but-counted flag (see the amount_eur CASE). - WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND c.eff_eur > 200 * c.proc_est_eur) THEN 'value_suspect' + WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND (c.eff_eur > 200 * c.proc_est_eur + -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly + -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with + -- nothing between 105x and 200x, so the band is narrow on purpose. + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) + -- The same dropped decimal point, but on a LOT of a multi-lot procedure: there the signature + -- is 100x the lot's OWN estimate, and the ratio to the whole procedure lands wherever the + -- lot's share puts it (issue #247 reports one at 89.8x). Measuring against the own-row + -- estimate alone would be unsafe - for framework and unit-price procedures it is a UNIT + -- price and a whole call-off legitimately dwarfs it - so it is paired with the procedure + -- level condition that already means "implausibly large for this procedure" (>= 10x, the + -- review threshold). A unit-price call-off sits at ~1x the procedure estimate and is spared. + OR (c.own_est_eur >= 1000 AND c.proc_est_eur >= 1000 AND c.eff_eur >= 10 * c.proc_est_eur + AND c.eff_eur >= 95 * c.own_est_eur AND c.eff_eur <= 105 * c.own_est_eur) THEN 'value_suspect' -- value_low: zero/negative, OR a tiny signed value (< 1000 EUR) that is also < 5% of the -- estimate. Large legitimate framework call-offs (small share of a huge ceiling but big in -- absolute terms) are NOT caught — the < 1000 EUR floor keeps them OUT of value_low. @@ -923,7 +952,95 @@ FROM ( ) END ), 0) < 0.05 THEN 'value_low' - WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100)) THEN 'annex_suspect' + WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 + -- Mis-keyed annex: a single step jumped ≥10× AND the aggregate ended ≥5× over signing. + -- The step alone is NOT enough — some chains have a huge step that a later annex pulls + -- back below signing, and flagging those would RAISE the shown value, not repair it. + OR (c.current_value / c.signing_value >= 5 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_before > 0 AND am.value_after >= 10 * am.value_before + ))))) THEN 'annex_suspect' + -- #305 value double-count: a driving annex reports a new TOTAL added to the old instead of + -- replacing it, so value_after ≈ 2× the OLD total. ЗОП чл.116 caps a single amendment at +50%, + -- so one step cannot legally more than double a contract — the ≥2× single step IS the defect + -- signal, wherever it sits in the chain. Scope: value_after in [2×,10×) a base that value_before + -- ties to a KNOWN prior total — signing_value OR a preceding annex's value_after (the multi-annex + -- case) — same currency. Slow legitimate climbs never reach ≥2× so stay untouched; the ≥10× + -- mis-key is #299's annex_suspect above; cross-currency doubles are an FX artefact ('review'); + -- and the ABS(... - current_value) tie binds this to the annex that DRIVES current_value, so a + -- doubled annex later superseded by a correct one is NOT flagged. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + -- #305 Tier-2: a text-treated annex (restated total or confirmed-genuine increment) is NOT an + -- arithmetic suspect — value_treatment IS NOT NULL means the основание text already resolved it. + AND am.value_treatment IS NULL + -- #305 multi-annex: the doubled step need not be the FIRST annex. value_before may be a + -- prior cumulative total (a preceding annex's value_after), not signing. Anchor to signing + -- OR a legitimately-grown prior total (a prior annex that was itself not a double), while a + -- single ≥2× step (ЗОП чл.116 caps one amendment at +50%) is the defect wherever it sits. + AND am.value_before > 0 AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× (value_after ≈ 2× value_before) on an ORPHAN + -- base — value_before ties neither signing NOR any prior annex's value_after (e.g. contract + -- 84818, whose annex base 76.77M is unrelated to the contract's values). A legal +100% in one + -- amendment is impossible (ЗОП чл.116), so flag → signing fallback (EXCLUDE); never REWRITES + -- (that stays gated, #307 HIGH-2). The orphan guard leaves a legitimate compounding-doubling + -- chain (each step's base ties the prior step) untouched, exactly as the legit-prior arm does. + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2: the double-count model presupposes a self-consistent row (value_after ≈ + -- value_before + value_delta). If value_delta is present and contradicts that, the model + -- provably does not apply — do NOT flag (mirrors the TS classifier's a≈b+d precondition). + AND (am.value_delta IS NULL + OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' + -- #305 NEW-HIGH-1 (multi-annex chain contamination): the double-count correction is per-row and + -- does NOT propagate down a chain. When a PRIOR annex was double-count corrected (value_after was + -- doubled, restated down), a LATER annex still arrives from the feed computed on the CONTAMINATED + -- (raw, doubled) base. Its own step ratio is legitimate (<2×) so the gate above misses it, and the + -- prior annex is text-treated so it is excluded too — yet current_value inherited the doubled + -- total. Detect the driving annex whose value_before ties to a prior annex's RAW value_after where + -- that prior was restated to a lower total, and flag → signing fallback (an honest exclusion beats + -- a served overstatement) until per-chain value_before propagation (a follow-up) recomputes it. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_treatment IS NULL + AND am.value_before > 0 + AND ABS(am.value_after - c.current_value) < 0.01 + AND EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after_restated IS NOT NULL + AND prev.value_after_restated < prev.value_after + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -976,6 +1093,24 @@ FROM ( LIMIT 1 ) END AS proc_est_eur, + -- Own-row (per-lot) estimate in EUR. The "too high" flags deliberately measure against the + -- PROCEDURE estimate, because for framework and unit-price procedures this per-row number is a + -- UNIT price and a whole call-off legitimately dwarfs it. It is computed here ONLY for the + -- стотинки band, which pairs it with a procedure-level condition for exactly that reason. + CASE + WHEN c.estimated_value IS NULL THEN NULL + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.estimated_value + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.estimated_value / 1.95583 + ELSE c.estimated_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END AS own_est_eur, t.estimated_value AS proc_est_native, aw.currency AS amendment_currency FROM raw_contracts c @@ -1173,7 +1308,10 @@ CREATE TABLE IF NOT EXISTS pipeline_stats ( DELETE FROM pipeline_stats; WITH amendment_dedup AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + -- #306: key on the original annex-side number (contract_number_raw) when the value resolver rewrote a + -- row — must match derive-amendments.sql / promote-amendments.sql byte-for-byte (review nikimilenkov + -- MEDIUM 3). NULL raw → the plain number, so unresolved rows are unaffected. + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), @@ -1206,7 +1344,20 @@ SELECT 1, SELECT c.*, CASE -- Mirrors the main derive CASE above; value_low is checked AFTER over-valuation + absurd. - WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND c.eff_eur > 200 * c.proc_est_eur) THEN 'value_suspect' + WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND (c.eff_eur > 200 * c.proc_est_eur + -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly + -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with + -- nothing between 105x and 200x, so the band is narrow on purpose. + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) + -- The same dropped decimal point, but on a LOT of a multi-lot procedure: there the signature + -- is 100x the lot's OWN estimate, and the ratio to the whole procedure lands wherever the + -- lot's share puts it (issue #247 reports one at 89.8x). Measuring against the own-row + -- estimate alone would be unsafe - for framework and unit-price procedures it is a UNIT + -- price and a whole call-off legitimately dwarfs it - so it is paired with the procedure + -- level condition that already means "implausibly large for this procedure" (>= 10x, the + -- review threshold). A unit-price call-off sits at ~1x the procedure estimate and is spared. + OR (c.own_est_eur >= 1000 AND c.proc_est_eur >= 1000 AND c.eff_eur >= 10 * c.proc_est_eur + AND c.eff_eur >= 95 * c.own_est_eur AND c.eff_eur <= 105 * c.own_est_eur) THEN 'value_suspect' WHEN COALESCE(c.current_value, c.signing_value) <= 0 THEN 'value_low' WHEN c.estimated_value > 0 AND c.signing_value IS NOT NULL AND ( CASE @@ -1251,7 +1402,95 @@ SELECT 1, ) END ), 0) < 0.05 THEN 'value_low' - WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100)) THEN 'annex_suspect' + WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 + -- Mis-keyed annex: a single step jumped ≥10× AND the aggregate ended ≥5× over signing. + -- The step alone is NOT enough — some chains have a huge step that a later annex pulls + -- back below signing, and flagging those would RAISE the shown value, not repair it. + OR (c.current_value / c.signing_value >= 5 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_before > 0 AND am.value_after >= 10 * am.value_before + ))))) THEN 'annex_suspect' + -- #305 value double-count: a driving annex reports a new TOTAL added to the old instead of + -- replacing it, so value_after ≈ 2× the OLD total. ЗОП чл.116 caps a single amendment at +50%, + -- so one step cannot legally more than double a contract — the ≥2× single step IS the defect + -- signal, wherever it sits in the chain. Scope: value_after in [2×,10×) a base that value_before + -- ties to a KNOWN prior total — signing_value OR a preceding annex's value_after (the multi-annex + -- case) — same currency. Slow legitimate climbs never reach ≥2× so stay untouched; the ≥10× + -- mis-key is #299's annex_suspect above; cross-currency doubles are an FX artefact ('review'); + -- and the ABS(... - current_value) tie binds this to the annex that DRIVES current_value, so a + -- doubled annex later superseded by a correct one is NOT flagged. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + -- #305 Tier-2: a text-treated annex (restated total or confirmed-genuine increment) is NOT an + -- arithmetic suspect — value_treatment IS NOT NULL means the основание text already resolved it. + AND am.value_treatment IS NULL + -- #305 multi-annex: the doubled step need not be the FIRST annex. value_before may be a + -- prior cumulative total (a preceding annex's value_after), not signing. Anchor to signing + -- OR a legitimately-grown prior total (a prior annex that was itself not a double), while a + -- single ≥2× step (ЗОП чл.116 caps one amendment at +50%) is the defect wherever it sits. + AND am.value_before > 0 AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× (value_after ≈ 2× value_before) on an ORPHAN + -- base — value_before ties neither signing NOR any prior annex's value_after (e.g. contract + -- 84818, whose annex base 76.77M is unrelated to the contract's values). A legal +100% in one + -- amendment is impossible (ЗОП чл.116), so flag → signing fallback (EXCLUDE); never REWRITES + -- (that stays gated, #307 HIGH-2). The orphan guard leaves a legitimate compounding-doubling + -- chain (each step's base ties the prior step) untouched, exactly as the legit-prior arm does. + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2: the double-count model presupposes a self-consistent row (value_after ≈ + -- value_before + value_delta). If value_delta is present and contradicts that, the model + -- provably does not apply — do NOT flag (mirrors the TS classifier's a≈b+d precondition). + AND (am.value_delta IS NULL + OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' + -- #305 NEW-HIGH-1 (multi-annex chain contamination): the double-count correction is per-row and + -- does NOT propagate down a chain. When a PRIOR annex was double-count corrected (value_after was + -- doubled, restated down), a LATER annex still arrives from the feed computed on the CONTAMINATED + -- (raw, doubled) base. Its own step ratio is legitimate (<2×) so the gate above misses it, and the + -- prior annex is text-treated so it is excluded too — yet current_value inherited the doubled + -- total. Detect the driving annex whose value_before ties to a prior annex's RAW value_after where + -- that prior was restated to a lower total, and flag → signing fallback (an honest exclusion beats + -- a served overstatement) until per-chain value_before propagation (a follow-up) recomputes it. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_treatment IS NULL + AND am.value_before > 0 + AND ABS(am.value_after - c.current_value) < 0.01 + AND EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after_restated IS NOT NULL + AND prev.value_after_restated < prev.value_after + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1297,6 +1536,24 @@ SELECT 1, LIMIT 1 ) END AS proc_est_eur, + -- Own-row (per-lot) estimate in EUR. The "too high" flags deliberately measure against the + -- PROCEDURE estimate, because for framework and unit-price procedures this per-row number is a + -- UNIT price and a whole call-off legitimately dwarfs it. It is computed here ONLY for the + -- стотинки band, which pairs it with a procedure-level condition for exactly that reason. + CASE + WHEN c.estimated_value IS NULL THEN NULL + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.estimated_value + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.estimated_value / 1.95583 + ELSE c.estimated_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END AS own_est_eur, t.estimated_value AS proc_est_native, aw.currency AS amendment_currency FROM raw_contracts c @@ -1318,6 +1575,7 @@ SELECT 1, ) c WHERE CASE c.value_flag WHEN 'annex_suspect' THEN COALESCE(c.signing_value, c.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(c.signing_value, c.current_value) ELSE COALESCE(c.current_value, c.signing_value) END IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || c.unp) @@ -1345,6 +1603,7 @@ SELECT (SELECT contract_candidates FROM pipeline_stats) AS contract_candidates, (SELECT COUNT(*) FROM contracts WHERE value_flag = 'value_suspect') AS value_suspect, (SELECT COUNT(*) FROM contracts WHERE value_flag = 'annex_suspect') AS annex_suspect, + (SELECT COUNT(*) FROM contracts WHERE value_flag = 'annex_total_suspect') AS annex_total_suspect, (SELECT COUNT(*) FROM contracts WHERE value_flag = 'review') AS review, (SELECT COUNT(*) FROM contracts WHERE fx_converted = 1) AS fx_converted, (SELECT ROUND(SUM(amount_eur) / 1e9, 2) FROM contracts) AS clean_total_eur_bn, diff --git a/scripts/precompute.sql b/scripts/precompute.sql index c79a26ef..f93a0a79 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -22,8 +22,9 @@ -- signing/current in EUR for the contract page's estimated→signing→current strip. -- BGN at the fixed peg (÷1.95583), EUR as-is, foreign at the row's stored fx_rate (eur_per_unit). -- Display rule: NULL where the figure is suspect, so the caller renders „данните се преглеждат", --- never a fabricated number. signing suppressed for value_suspect; current suppressed for value_ or --- annex_suspect (the suspect annex is the bad part). estimated_value_eur is derived per-request on +-- never a fabricated number. signing suppressed for value_suspect; current suppressed for value_, +-- annex_suspect or annex_total_suspect (#305; the suspect annex is the bad part). estimated_value_eur +-- is derived per-request on -- the contract detail loader from the tender (procurement-level, shared across a multi-lot prepiska). UPDATE contracts SET signing_value_eur = CASE @@ -33,7 +34,7 @@ UPDATE contracts SET WHEN fx_rate IS NOT NULL THEN signing_value * fx_rate ELSE NULL END, current_value_eur = CASE - WHEN value_flag IN ('value_suspect','annex_suspect') OR current_value IS NULL THEN NULL + WHEN value_flag IN ('value_suspect','annex_suspect','annex_total_suspect') OR current_value IS NULL THEN NULL WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'EUR' THEN current_value WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'BGN' THEN current_value / 1.95583 WHEN fx_rate IS NOT NULL THEN current_value * fx_rate @@ -245,6 +246,13 @@ FROM interest_links il JOIN persons p ON p.id = il.person_id -- rendering both re-identifies the relative via a ТР owner lookup, and the company is already surfaced -- by the self row. WHERE il.status = 'published' AND il.interest_class IN ('private_ownership', 'family_ownership') + -- …and the identity rests on a Trade Register fact (#279, ADR-0033). This predicate is the THIRD copy + -- of the surface gate — the other two are SURFACED_OWNERSHIP in packages/db/src/queries/related-persons.ts + -- and the sibling block in the other of precompute.sql / refresh-slice.sql. All three must move + -- together: this one feeds the officials search index, so omitting it would keep officials findable + -- whose links no longer surface. + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed')) AND EXISTS (SELECT 1 FROM contracts cc JOIN bidders bb ON bb.id = cc.bidder_id WHERE bb.eik_normalized = il.eik) AND NOT (il.interest_class = 'family_ownership' AND EXISTS ( diff --git a/scripts/promote-amendments.sql b/scripts/promote-amendments.sql index 5d72e367..c5e70557 100644 --- a/scripts/promote-amendments.sql +++ b/scripts/promote-amendments.sql @@ -7,13 +7,18 @@ DELETE FROM amendments; INSERT OR REPLACE INTO amendments ( - id, natural_key, contract_number, unp, value_before, value_after, value_delta, currency, - published_at, document_number, description, source + id, natural_key, contract_number, contract_number_raw, link_method, unp, + value_before, value_after, value_delta, currency, + published_at, document_number, description, source, + value_restated, value_treatment, value_suspect ) WITH keyed AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + -- #306: key on the original annex-side number (contract_number_raw) when the value resolver rewrote a + -- row — MUST match derive-amendments.sql's rollup key byte-for-byte, else annex_count and the served + -- timeline disagree (review nikimilenkov MEDIUM 3). NULL raw → the plain number. + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), @@ -38,15 +43,66 @@ SELECT natural_key, natural_key, contract_number, + contract_number_raw, -- #306 provenance: the annex-side number before the value resolver rewrote it + link_method, -- #306 provenance: 'value_anchor' for value-linked rows, else NULL unp, value_before, - value_after, - value_delta, + -- #305 Tier-2: serve the effective (text-corrected) after and a self-consistent delta; a restated + -- annex carries the true total, an untreated one is unchanged. + COALESCE(value_after_restated, value_after), + COALESCE(value_after_restated, value_after) - value_before, currency, published_at, document_number, description, - source + source, + CASE WHEN value_after_restated IS NOT NULL THEN 1 ELSE 0 END, + value_treatment, + -- #305 residual: mark a suspected double-count that is NOT already text-treated so the UI suppresses + -- the untrusted value_after. Mirrors normalize-raw.sql's annex_total_suspect arithmetic gate, but + -- joined to raw_contracts for the contract's signing_value/currency (this served INSERT has no + -- contract row to read). value_treatment IS NULL keeps a restated/genuine row out (value_restated + -- already owns those). No current_value tie here: the tie in normalize-raw only decides whether the + -- CONTRACT is flagged; the per-row marker suppresses any row whose after is an unbridgeable double. + CASE WHEN value_treatment IS NULL + AND value_before > 0 + AND value_after >= 2 * value_before AND value_after < 10 * value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (value_delta IS NULL OR ABS(value_after - (value_before + value_delta)) < 0.01 * value_after) + AND EXISTS ( + SELECT 1 FROM raw_contracts rc + WHERE rc.unp = dedup.unp AND rc.contract_number = dedup.contract_number + AND rc.signing_value > 0 + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing. Anchor to signing OR a legitimately-grown prior total (prev not + -- itself a double); a single ≥2× step violates ЗОП чл.116 wherever it sits (see normalize-raw.sql). + AND ( + ABS(dedup.value_before - rc.signing_value) < 0.01 * rc.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = dedup.unp AND prev.contract_number = dedup.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - dedup.value_before) < 0.01 * dedup.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double). + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: EXACT single-step 2× on an ORPHAN base (value_before ties neither signing + -- nor any prior annex) — mark the row suspect; never rewrites (see normalize-raw.sql). The + -- orphan guard leaves compounding chains untouched. + OR ( + ABS(dedup.value_after - 2 * dedup.value_before) < 0.005 * dedup.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = dedup.unp AND prev.contract_number = dedup.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - dedup.value_before) < 0.01 * dedup.value_before + ) + ) + ) + AND COALESCE(NULLIF(dedup.currency, ''), COALESCE(NULLIF(rc.currency, ''), 'BGN')) + = COALESCE(NULLIF(rc.currency, ''), 'BGN') + ) + THEN 1 ELSE 0 END FROM dedup WHERE rn = 1; diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 3f4544be..60854cf5 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -21,13 +21,226 @@ DROP TABLE IF EXISTS refresh_unp_prefix_authorities; DROP TABLE IF EXISTS refresh_joint_authority_members; DROP TABLE IF EXISTS refresh_joint_tender_sources; DROP TABLE IF EXISTS refresh_amendment_winners; +DROP TABLE IF EXISTS refresh_amendment_contract_resolve; CREATE TABLE refresh_touched_contracts (id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_bidders (bidder_id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_authorities (authority_id TEXT PRIMARY KEY); + +-- #286: recover the УНП for OCDS amendments via the tender.id bridge before any raw_amendments read +-- below, mirroring derive-amendments.sql (raw_tenders first, then the raw_contracts synthetic-tender +-- fallback). In the slice path the raw tables hold only the loaded window, so an OCDS amendment whose +-- procedure was staged outside the window stays unbridged until the next full derive — best-effort by +-- design; the full pipeline is authoritative. +-- KEEP THE BRIDGE UPDATE BELOW IN LOCKSTEP with scripts/derive-amendments.sql: the UPDATE between the +-- @bridge-lockstep markers must stay byte-for-byte equivalent to the full path (packages/db/src/ +-- amendments-bridge-lockstep.test.ts enforces it). The prefer-EOP dedup INTENTIONALLY DIVERGES from the +-- full path — the slice path also reconciles against the cumulative served `amendments` table (below and +-- before the promotion), which the full path never needs because promote-amendments.sql rebuilds it from +-- scratch. Index raw_contracts(tender_ext_id) so the fallback lookups don't full-scan raw_contracts per +-- OCDS row (raw_tenders(tender_id) is already indexed); ORDER BY unp keeps the recovered УНП deterministic. +CREATE INDEX IF NOT EXISTS idx_raw_contracts_tender_ext_id ON raw_contracts(tender_ext_id); +-- @bridge-lockstep start +UPDATE raw_amendments +SET unp = COALESCE( + (SELECT rt.unp FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL ORDER BY rt.unp LIMIT 1), + (SELECT rc.unp FROM raw_contracts rc + WHERE rc.tender_ext_id = raw_amendments.tender_ext_id AND rc.unp IS NOT NULL ORDER BY rc.unp LIMIT 1) +) +WHERE source LIKE 'ocds:%' + AND tender_ext_id IS NOT NULL + AND ( + -- raw_tenders wins when it resolves the procedure to exactly ONE УНП; else fall back to raw_contracts + -- when IT is unambiguous. Refuse to bridge (leave the OCID as an honest residual) when the chosen + -- source maps one tender_ext_id to more than one distinct УНП — the domain is 1-to-1, so this guards a + -- feed anomaly rather than silently mis-attributing every annex of the losing procedure (issue #286). + (SELECT COUNT(DISTINCT rt.unp) FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL) = 1 + OR ( + NOT EXISTS (SELECT 1 FROM raw_tenders rt + WHERE rt.tender_id = raw_amendments.tender_ext_id AND rt.unp IS NOT NULL) + AND (SELECT COUNT(DISTINCT rc.unp) FROM raw_contracts rc + WHERE rc.tender_ext_id = raw_amendments.tender_ext_id AND rc.unp IS NOT NULL) = 1 + ) + ); +-- @bridge-lockstep end + +-- #306: slice-safe value-anchor resolver. Links EOP annexes whose annex-side number is in a different +-- namespace than the contract's filing number (annex carries an internal number like 148846; the contract +-- carries Д-226), by the exact value_before → signing_value anchor — the same 99.99%-precision link the +-- full path applies in scripts/resolve-amendment-contracts.sql. It runs HERE, before the prefer-EOP dedup +-- DELETE below and the amendment promotion further down, for the SAME reason the full path runs it before +-- derive-amendments.sql's dedup: rewriting an EOP annex onto a contract that already kept an OCDS twin would +-- resurrect the twin and trip amendment-twin-dedup (#303, review todorkolev #1). +-- +-- The full path is gated OFF the slice because its candidates came from the WINDOWED raw_contracts, where +-- "unique on the procedure" means "unique in the window" and a corpus-ambiguous annex would mislink (review +-- nikimilenkov HIGH 1). This slice version closes that gap the way the deferred plan §4 prescribes: candidate +-- contracts are drawn from the CUMULATIVE served `contracts` (the whole corpus) UNIONed with this window's +-- raw_contracts, so uniqueness is asked over the corpus, not the window — the measured precision carries. It +-- is intentionally NOT under a byte-identical lockstep marker with the full-path script: the candidate source +-- differs by construction. Scans are bounded to the procedures that actually have an EOP annex in this window +-- (window_unps), so the served-corpus read stays a keyed lookup, not a full scan (idx_contracts_tender_id). +-- Resolved prior-window targets need no extra touch-wiring: the rewrite below lands the target contract_number +-- on raw_amendments, and the existing `@refresh-batch amendments` touch join (raw_amendments → contracts on +-- contract_number) then scopes those targets into refresh_touched_contracts for free (plan §4). +DROP TABLE IF EXISTS refresh_amendment_contract_resolve; +CREATE TABLE refresh_amendment_contract_resolve AS +WITH window_unps AS ( + SELECT DISTINCT unp FROM raw_amendments + WHERE source LIKE 'eop:%' AND unp IS NOT NULL AND contract_number IS NOT NULL +), +-- Every contract NUMBER that exists on the affected procedures — from BOTH this window's raw_contracts and the +-- served corpus, REGARDLESS of signing_value. `grp` below asks this to decide "is the annex's number already a +-- real contract on the procedure". It MUST be value-agnostic: the full path asks only "does such a contract +-- exist" (NOT EXISTS over raw_contracts), so keying the slice question off contract_candidates (which requires +-- signing_value > 0) would diverge — an annex that points to a ZERO-value contract by number would look +-- namespace-mismatched and get value-linked to a neighbour, contradicting "links by number, not value" and +-- making the two paths disagree (review todorkolev). +all_contract_numbers AS ( + SELECT unp, contract_number FROM raw_contracts + WHERE contract_number IS NOT NULL AND unp IN (SELECT unp FROM window_unps) + UNION + SELECT substr(tender_id, 3) AS unp, contract_number FROM contracts + WHERE contract_number IS NOT NULL AND tender_id IN (SELECT 't:' || unp FROM window_unps) +), +-- One row per LOGICAL contract on the affected procedures, for VALUE matching (signing_value > 0). Two sources, +-- deduped: (0) this window's raw_contracts — cumulative EOP buckets repeat a contract across days, so collapse +-- to the latest source-day then highest id, mirroring normalize-raw; (1) the served `contracts` corpus (unp = +-- the tender_id suffix, contractor ЕИК via the winning bidder). A contract present in BOTH is one logical +-- contract — src_rank keeps the window row (freshest signing_value) and the COUNT below never double-counts it +-- into a false ambiguity. +contract_candidates AS ( + SELECT unp, contract_number, signing_value, currency, contractor_eik FROM ( + SELECT unp, contract_number, signing_value, currency, contractor_eik, + ROW_NUMBER() OVER ( + PARTITION BY unp, contract_number ORDER BY src_rank, cand_source DESC, ord DESC + ) AS rn + FROM ( + SELECT c.unp, c.contract_number, c.signing_value, c.currency, + TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) + AS contractor_eik, + 0 AS src_rank, c.source AS cand_source, c.id AS ord + FROM raw_contracts c + WHERE c.contract_number IS NOT NULL + AND c.signing_value IS NOT NULL AND c.signing_value > 0 + AND c.unp IN (SELECT unp FROM window_unps) + UNION ALL + SELECT substr(c.tender_id, 3) AS unp, c.contract_number, c.signing_value, c.currency, + b.eik_normalized AS contractor_eik, + 1 AS src_rank, '' AS cand_source, '' AS ord + FROM contracts c + LEFT JOIN bidders b ON b.id = c.bidder_id + WHERE c.contract_number IS NOT NULL + AND c.signing_value IS NOT NULL AND c.signing_value > 0 + AND c.tender_id IN (SELECT 't:' || unp FROM window_unps) + ) + ) WHERE rn = 1 +), +-- EOP annexes on the affected procedures that match NO REAL contract by (unp, contract_number) — the +-- namespace-mismatch group. Value-less members (admin/term steps mid-chain) are included so they can inherit +-- the chain's target (review nikimilenkov MEDIUM 2). The NOT EXISTS is over all_contract_numbers (every real +-- contract number on the procedure, value-agnostic), so an annex that DOES match a corpus contract directly is +-- correctly excluded — it links by number, not value — even when that target has signing_value <= 0, matching +-- the full path exactly (review todorkolev). +grp AS ( + SELECT a.id AS amendment_id, a.unp, a.contract_number AS annex_cnum, + a.value_before, a.currency, + TRIM(CASE WHEN a.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(a.contractor_eik, 5) ELSE a.contractor_eik END) + AS contractor_eik + FROM raw_amendments a + WHERE a.source LIKE 'eop:%' + AND a.unp IS NOT NULL AND a.contract_number IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM all_contract_numbers c + WHERE c.unp = a.unp AND c.contract_number = a.contract_number + ) +), +-- Exact, currency- and contractor-matched value anchor (< 0.5 стотинка). Currency must be EXPLICIT on both +-- sides (no blank-vs-blank agreement via a default). The EIK guard is null-tolerant and refuses a value +-- collision onto a different contractor's contract for free (review nikimilenkov LOW 1 / MEDIUM 5). +vmatch AS ( + SELECT g.amendment_id, g.unp, g.annex_cnum, c.contract_number AS resolved_cnum, + COUNT(*) OVER (PARTITION BY g.amendment_id) AS n_match + FROM grp g + JOIN contract_candidates c + ON c.unp = g.unp + AND ABS(c.signing_value - g.value_before) < 0.005 + AND NULLIF(c.currency, '') IS NOT NULL + AND NULLIF(g.currency, '') IS NOT NULL + AND c.currency = g.currency + AND (g.contractor_eik IS NULL OR c.contractor_eik IS NULL OR g.contractor_eik = c.contractor_eik) + WHERE g.value_before IS NOT NULL AND g.value_before > 0 +), +-- A member's OWN unique (n_match = 1) exact match — trustworthy on its own, so it always applies and is NOT +-- voided when annex-number siblings point elsewhere (a lot-base number shared across contracts; review +-- nikimilenkov MEDIUM 1). Propagation is withheld on disagreement, never the direct hits. +direct AS ( + SELECT amendment_id, unp, annex_cnum, resolved_cnum FROM vmatch WHERE n_match = 1 +), +-- Propagate one AGREED target across a (unp, annex-number) chain to value-less members, only when the direct +-- members agree on a single contract (review MEDIUM 2). +group_target AS ( + SELECT unp, annex_cnum, MIN(resolved_cnum) AS resolved_cnum + FROM direct GROUP BY unp, annex_cnum HAVING COUNT(DISTINCT resolved_cnum) = 1 +) +-- Link to the own unique match, else the agreed group target — UNLESS the member is itself value-ambiguous +-- (n_match >= 2), which never links directly or by inheritance (review todorkolev #2). +SELECT g.amendment_id, + COALESCE( + (SELECT d.resolved_cnum FROM direct d WHERE d.amendment_id = g.amendment_id), + (SELECT gt.resolved_cnum FROM group_target gt WHERE gt.unp = g.unp AND gt.annex_cnum = g.annex_cnum) + ) AS resolved_cnum +FROM grp g +WHERE NOT EXISTS ( + SELECT 1 FROM vmatch v WHERE v.amendment_id = g.amendment_id AND v.n_match >= 2 +); + +CREATE INDEX idx_refresh_amendment_contract_resolve_id + ON refresh_amendment_contract_resolve(amendment_id); + +-- Rewrite in place, preserving provenance: keep the annex-side number in contract_number_raw and stamp +-- link_method so value-linked rows stay enumerable through promotion into served `amendments`. The raw number +-- also keeps the amendment natural_key stable (see the promotion below and derive/promote on the full path), +-- so a resolved row never collides with a native annex sharing document_number on the target (MEDIUM 3). +UPDATE raw_amendments +SET + contract_number_raw = contract_number, + link_method = 'value_anchor', + contract_number = ( + SELECT r.resolved_cnum FROM refresh_amendment_contract_resolve r WHERE r.amendment_id = raw_amendments.id + ) +WHERE id IN (SELECT amendment_id FROM refresh_amendment_contract_resolve WHERE resolved_cnum IS NOT NULL); + +DROP TABLE IF EXISTS refresh_amendment_contract_resolve; + +-- #286: prefer the EOP annex — drop OCDS twins so annex_count and the served timeline aren't doubled. +-- The full path (derive-amendments.sql) only checks raw_amendments because it re-stages the whole corpus +-- every run. The slice path must ALSO consult the cumulative served `amendments`: an EOP annex promoted by +-- an EARLIER window is not in this window's raw_amendments, yet its OCDS twin must still be dropped, or the +-- INSERT OR REPLACE promotion below (keyed by a natural_key that never matches across sources) would leave +-- both rows and double annex_count (issue #286, review nikimilenkov HIGH 1). +DELETE FROM raw_amendments +WHERE source LIKE 'ocds:%' + AND ( + EXISTS ( + SELECT 1 FROM raw_amendments e + WHERE e.source LIKE 'eop:%' + AND e.unp = raw_amendments.unp + AND e.contract_number = raw_amendments.contract_number + ) + OR EXISTS ( + SELECT 1 FROM amendments s + WHERE s.source LIKE 'eop:%' + AND s.unp = raw_amendments.unp + AND s.contract_number = raw_amendments.contract_number + ) + ); + CREATE TABLE refresh_amendment_winners AS WITH keyed AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), NULLIF(seq_no, ''), 'content:' || COALESCE(published_at, '') || ':' || @@ -1000,7 +1213,7 @@ FROM ( ELSE q.signing_value * q.fx_rate END AS signing_value_eur, CASE - WHEN q.value_flag IN ('value_suspect', 'annex_suspect') OR q.current_value IS NULL THEN NULL + WHEN q.value_flag IN ('value_suspect', 'annex_suspect', 'annex_total_suspect') OR q.current_value IS NULL THEN NULL WHEN q.current_value_currency = 'EUR' THEN q.current_value WHEN q.current_value_currency = 'BGN' THEN q.current_value / 1.95583 ELSE q.current_value * q.fx_rate @@ -1010,11 +1223,13 @@ FROM ( CASE y.value_flag WHEN 'value_suspect' THEN y.proc_est_native WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS display_native, CASE y.value_flag WHEN 'value_suspect' THEN NULL WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, CASE y.value_flag @@ -1024,6 +1239,11 @@ FROM ( ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') END + WHEN 'annex_total_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + END ELSE CASE WHEN y.current_value IS NOT NULL THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') @@ -1059,7 +1279,19 @@ FROM ( -- Over-valuation + absurd FIRST and repaired to the procedure estimate. -- value_low is labelled-but-counted (see the amount_eur CASE). Keep in sync with -- normalize-raw.sql and the EOP block below. - WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND c.eff_eur > 200 * c.proc_est_eur) THEN 'value_suspect' + WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND (c.eff_eur > 200 * c.proc_est_eur + -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly + -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with + -- nothing between 105x and 200x, so the band is narrow on purpose. + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) + -- The same dropped decimal point on a LOT of a multi-lot procedure: 100x the lot's OWN + -- estimate, while the ratio to the whole procedure lands wherever the lot's share puts it + -- (issue #247 reports one at 89.8x). The own-row estimate alone is unsafe - for framework + -- and unit-price procedures it is a UNIT price and a whole call-off legitimately dwarfs it - + -- so it is paired with the procedure-level condition that already means "implausibly large + -- for this procedure" (>= 10x, the review threshold). + OR (c.own_est_eur >= 1000 AND c.proc_est_eur >= 1000 AND c.eff_eur >= 10 * c.proc_est_eur + AND c.eff_eur >= 95 * c.own_est_eur AND c.eff_eur <= 105 * c.own_est_eur) THEN 'value_suspect' -- value_low: zero/negative, OR a tiny signed value (< 1000 EUR) that is also < 5% of the -- estimate. The < 1000 EUR floor keeps large legitimate framework call-offs OUT. WHEN COALESCE(c.current_value, c.signing_value) <= 0 THEN 'value_low' @@ -1106,7 +1338,64 @@ FROM ( ) END ), 0) < 0.05 THEN 'value_low' - WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100)) THEN 'annex_suspect' + WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 + -- Mis-keyed annex: a single step jumped ≥10× AND the aggregate ended ≥5× over signing. + -- The step alone is NOT enough — some chains have a huge step that a later annex pulls + -- back below signing, and flagging those would RAISE the shown value, not repair it. + OR (c.current_value / c.signing_value >= 5 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_before > 0 AND am.value_after >= 10 * am.value_before + ))))) THEN 'annex_suspect' + -- #305 value double-count: a driving annex reports a new TOTAL added to the old instead of + -- replacing it, so value_after ≈ 2× the OLD total. ЗОП чл.116 caps a single amendment at +50%, + -- so one step cannot legally more than double a contract — the ≥2× single step IS the defect + -- signal, wherever it sits in the chain. Scope: value_after in [2×,10×) a base that value_before + -- ties to a KNOWN prior total — signing_value OR a preceding annex's value_after (the multi-annex + -- case) — same currency. Slow legitimate climbs never reach ≥2× so stay untouched; the ≥10× + -- mis-key is #299's annex_suspect above; cross-currency doubles are an FX artefact ('review'); + -- and the ABS(... - current_value) tie binds this to the annex that DRIVES current_value, so a + -- doubled annex later superseded by a correct one is NOT flagged. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + -- #305 Tier-2: skip text-treated annexes (restated total or confirmed-genuine increment). + AND am.value_treatment IS NULL + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing. Anchor to signing OR a legitimately-grown prior total (prev + -- not itself a double); a single ≥2× step violates ЗОП чл.116 wherever it sits (see normalize-raw.sql). + AND am.value_before > 0 AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× on an ORPHAN base (value_before ties neither + -- signing nor any prior annex) is the ЗОП чл.116 defect signature — flag (→ signing + -- fallback, EXCLUDE); never rewrites. The orphan guard leaves compounding chains untouched + -- (see normalize-raw.sql). + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (am.value_delta IS NULL OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1162,6 +1451,23 @@ FROM ( LIMIT 1 ) END AS proc_est_eur, + -- Own-row (per-lot) estimate in EUR - see the note at the стотинки band below: used ONLY + -- there, paired with a procedure-level condition, because for framework and unit-price + -- procedures this number is a UNIT price. + CASE + WHEN c.estimated_value IS NULL THEN NULL + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.estimated_value + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.estimated_value / 1.95583 + ELSE c.estimated_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END AS own_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c JOIN contractor_identity ci @@ -1276,7 +1582,7 @@ FROM ( ELSE q.signing_value * q.fx_rate END AS signing_value_eur, CASE - WHEN q.value_flag IN ('value_suspect', 'annex_suspect') OR q.current_value IS NULL THEN NULL + WHEN q.value_flag IN ('value_suspect', 'annex_suspect', 'annex_total_suspect') OR q.current_value IS NULL THEN NULL WHEN q.current_value_currency = 'EUR' THEN q.current_value WHEN q.current_value_currency = 'BGN' THEN q.current_value / 1.95583 ELSE q.current_value * q.fx_rate @@ -1286,11 +1592,13 @@ FROM ( CASE y.value_flag WHEN 'value_suspect' THEN y.proc_est_native WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS display_native, CASE y.value_flag WHEN 'value_suspect' THEN NULL WHEN 'annex_suspect' THEN COALESCE(y.signing_value, y.current_value) + WHEN 'annex_total_suspect' THEN COALESCE(y.signing_value, y.current_value) ELSE COALESCE(y.current_value, y.signing_value) END AS trusted_native, CASE y.value_flag @@ -1300,6 +1608,11 @@ FROM ( ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') END + WHEN 'annex_total_suspect' THEN CASE + WHEN y.signing_value IS NOT NULL THEN COALESCE(NULLIF(y.currency, ''), 'BGN') + ELSE COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w + WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') + END ELSE CASE WHEN y.current_value IS NOT NULL THEN COALESCE((SELECT NULLIF(w.currency, '') FROM refresh_amendment_winners w WHERE w.unp = y.unp AND w.contract_number = y.contract_number), NULLIF(y.currency, ''), 'BGN') @@ -1339,7 +1652,19 @@ FROM ( -- Over-valuation + absurd FIRST and repaired to the procedure estimate. -- value_low is labelled-but-counted (see the amount_eur CASE). Keep in sync with -- normalize-raw.sql and the OCDS block above. - WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND c.eff_eur > 200 * c.proc_est_eur) THEN 'value_suspect' + WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND (c.eff_eur > 200 * c.proc_est_eur + -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly + -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with + -- nothing between 105x and 200x, so the band is narrow on purpose. + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) + -- The same dropped decimal point on a LOT of a multi-lot procedure: 100x the lot's OWN + -- estimate, while the ratio to the whole procedure lands wherever the lot's share puts it + -- (issue #247 reports one at 89.8x). The own-row estimate alone is unsafe - for framework + -- and unit-price procedures it is a UNIT price and a whole call-off legitimately dwarfs it - + -- so it is paired with the procedure-level condition that already means "implausibly large + -- for this procedure" (>= 10x, the review threshold). + OR (c.own_est_eur >= 1000 AND c.proc_est_eur >= 1000 AND c.eff_eur >= 10 * c.proc_est_eur + AND c.eff_eur >= 95 * c.own_est_eur AND c.eff_eur <= 105 * c.own_est_eur) THEN 'value_suspect' -- value_low: zero/negative, OR a tiny signed value (< 1000 EUR) that is also < 5% of the -- estimate. The < 1000 EUR floor keeps large legitimate framework call-offs OUT. WHEN COALESCE(c.current_value, c.signing_value) <= 0 THEN 'value_low' @@ -1386,7 +1711,64 @@ FROM ( ) END ), 0) < 0.05 THEN 'value_low' - WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100)) THEN 'annex_suspect' + WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 + -- Mis-keyed annex: a single step jumped ≥10× AND the aggregate ended ≥5× over signing. + -- The step alone is NOT enough — some chains have a huge step that a later annex pulls + -- back below signing, and flagging those would RAISE the shown value, not repair it. + OR (c.current_value / c.signing_value >= 5 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + AND am.value_before > 0 AND am.value_after >= 10 * am.value_before + ))))) THEN 'annex_suspect' + -- #305 value double-count: a driving annex reports a new TOTAL added to the old instead of + -- replacing it, so value_after ≈ 2× the OLD total. ЗОП чл.116 caps a single amendment at +50%, + -- so one step cannot legally more than double a contract — the ≥2× single step IS the defect + -- signal, wherever it sits in the chain. Scope: value_after in [2×,10×) a base that value_before + -- ties to a KNOWN prior total — signing_value OR a preceding annex's value_after (the multi-annex + -- case) — same currency. Slow legitimate climbs never reach ≥2× so stay untouched; the ≥10× + -- mis-key is #299's annex_suspect above; cross-currency doubles are an FX artefact ('review'); + -- and the ABS(... - current_value) tie binds this to the annex that DRIVES current_value, so a + -- doubled annex later superseded by a correct one is NOT flagged. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND EXISTS ( + SELECT 1 FROM raw_amendments am + WHERE am.unp = c.unp AND am.contract_number = c.contract_number + -- #305 Tier-2: skip text-treated annexes (restated total or confirmed-genuine increment). + AND am.value_treatment IS NULL + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing. Anchor to signing OR a legitimately-grown prior total (prev + -- not itself a double); a single ≥2× step violates ЗОП чл.116 wherever it sits (see normalize-raw.sql). + AND am.value_before > 0 AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× on an ORPHAN base (value_before ties neither + -- signing nor any prior annex) is the ЗОП чл.116 defect signature — flag (→ signing + -- fallback, EXCLUDE); never rewrites. The orphan guard leaves compounding chains untouched + -- (see normalize-raw.sql). + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (am.value_delta IS NULL OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, @@ -1442,6 +1824,23 @@ FROM ( LIMIT 1 ) END AS proc_est_eur, + -- Own-row (per-lot) estimate in EUR - see the note at the стотинки band below: used ONLY + -- there, paired with a procedure-level condition, because for framework and unit-price + -- procedures this number is a UNIT price. + CASE + WHEN c.estimated_value IS NULL THEN NULL + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN c.estimated_value + WHEN COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN c.estimated_value / 1.95583 + ELSE c.estimated_value * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = COALESCE(NULLIF(c.procurement_currency, ''), NULLIF(c.currency, '')) + AND f.rate_date <= c.contract_date + AND f.rate_date >= date(c.contract_date, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END AS own_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c JOIN contractor_identity ci @@ -1503,14 +1902,29 @@ WHERE status <> 'awarded' -- 5) Promote window amendments into served domain history and roll touched contracts. -- @refresh-batch amendments +-- #286 convergence (review nikimilenkov HIGH 1), the other direction of the served-table reconciliation +-- above: an EOP annex arriving in THIS window supersedes an OCDS twin a PRIOR slice already served for the +-- same (unp, contract_number). Drop the stale served OCDS row before promotion so the rollup never counts +-- both. The full path needs no equivalent — promote-amendments.sql rebuilds `amendments` wholesale. +DELETE FROM amendments +WHERE source LIKE 'ocds:%' + AND EXISTS ( + SELECT 1 FROM raw_amendments e + WHERE e.source LIKE 'eop:%' + AND e.unp = amendments.unp + AND e.contract_number = amendments.contract_number + ); + INSERT OR REPLACE INTO amendments ( - id, natural_key, contract_number, unp, value_before, value_after, value_delta, currency, - published_at, document_number, description, source + id, natural_key, contract_number, contract_number_raw, link_method, unp, + value_before, value_after, value_delta, currency, + published_at, document_number, description, source, + value_restated, value_treatment, value_suspect ) WITH keyed AS ( SELECT *, - 'am:' || COALESCE(unp, '') || ':' || COALESCE(contract_number, '') || ':' || + 'am:' || COALESCE(unp, '') || ':' || COALESCE(NULLIF(contract_number_raw, ''), contract_number, '') || ':' || COALESCE( NULLIF(document_number, ''), NULLIF(correction_number, ''), @@ -1535,15 +1949,66 @@ SELECT natural_key, natural_key, contract_number, + contract_number_raw, -- #306 provenance: the annex-side number before the value resolver rewrote it + link_method, -- #306 provenance: 'value_anchor' for value-linked rows, else NULL unp, value_before, - value_after, - value_delta, + -- #305 Tier-2: serve the effective (text-corrected) after and a self-consistent delta; the current_value + -- rollup below reads this served value_after, so a restated annex drives current_value with the true total. + COALESCE(value_after_restated, value_after), + COALESCE(value_after_restated, value_after) - value_before, currency, published_at, document_number, description, - source + source, + CASE WHEN value_after_restated IS NOT NULL THEN 1 ELSE 0 END, + value_treatment, + -- #305 residual: mark a suspected double-count that is NOT already text-treated so the UI suppresses + -- the untrusted value_after. Mirrors normalize-raw.sql's annex_total_suspect arithmetic gate, but + -- joined to raw_contracts for the contract's signing_value/currency (this served INSERT has no + -- contract row to read). value_treatment IS NULL keeps a restated/genuine row out (value_restated + -- already owns those). No current_value tie here: the tie in normalize-raw only decides whether the + -- CONTRACT is flagged; the per-row marker suppresses any row whose after is an unbridgeable double. + CASE WHEN value_treatment IS NULL + AND value_before > 0 + AND value_after >= 2 * value_before AND value_after < 10 * value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (value_delta IS NULL OR ABS(value_after - (value_before + value_delta)) < 0.01 * value_after) + AND EXISTS ( + SELECT 1 FROM raw_contracts rc + WHERE rc.unp = dedup.unp AND rc.contract_number = dedup.contract_number + AND rc.signing_value > 0 + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing. Anchor to signing OR a legitimately-grown prior total (prev not + -- itself a double); a single ≥2× step violates ЗОП чл.116 wherever it sits (see normalize-raw.sql). + AND ( + ABS(dedup.value_before - rc.signing_value) < 0.01 * rc.signing_value + OR EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = dedup.unp AND prev.contract_number = dedup.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - dedup.value_before) < 0.01 * dedup.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double). + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: EXACT single-step 2× on an ORPHAN base (value_before ties neither signing + -- nor any prior annex) — mark the row suspect; never rewrites (see normalize-raw.sql). The + -- orphan guard leaves compounding chains untouched. + OR ( + ABS(dedup.value_after - 2 * dedup.value_before) < 0.005 * dedup.value_before + AND NOT EXISTS ( + SELECT 1 FROM raw_amendments prev + WHERE prev.unp = dedup.unp AND prev.contract_number = dedup.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - dedup.value_before) < 0.01 * dedup.value_before + ) + ) + ) + AND COALESCE(NULLIF(dedup.currency, ''), COALESCE(NULLIF(rc.currency, ''), 'BGN')) + = COALESCE(NULLIF(rc.currency, ''), 'BGN') + ) + THEN 1 ELSE 0 END FROM dedup WHERE rn = 1; @@ -1559,7 +2024,10 @@ SET WHERE a.unp = substr(contracts.tender_id, 3) AND a.contract_number = contracts.contract_number AND a.value_after IS NOT NULL - ORDER BY a.published_at DESC, a.id DESC + -- #305: tie-break on natural_key to match the full-rebuild path (derive-amendments.sql), so the + -- driving amendment picked for current_value is identical across full and slice when two annexes + -- share published_at. `id` (row insertion order) diverged from natural_key and could pick a different row. + ORDER BY a.published_at DESC, a.natural_key DESC LIMIT 1 ), current_value_currency = ( @@ -1567,7 +2035,9 @@ SET WHERE a.unp = substr(contracts.tender_id, 3) AND a.contract_number = contracts.contract_number AND a.value_after IS NOT NULL - ORDER BY a.published_at DESC, a.id DESC + -- #305: same natural_key tie-break as current_value above, so the currency comes from the same + -- driving amendment the value does. + ORDER BY a.published_at DESC, a.natural_key DESC LIMIT 1 ) WHERE (id GLOB 'c:[eo]:*' AND EXISTS ( @@ -1626,7 +2096,131 @@ WITH contract_base AS ( ) ORDER BY rc.source DESC, rc.id DESC LIMIT 1 - ), te.estimated_value) AS classifier_estimated_value + ), te.estimated_value) AS classifier_estimated_value, + c.signed_at, + -- The contract row's OWN estimate and the currency it is denominated in, for the стотинки band's + -- own-row arm (#247). Deliberately WITHOUT the procedure fallback classifier_estimated_value carries: + -- the arm asks "is this 100× the row's own estimate", and a row with no own estimate has no answer. + -- Falling back would make the arm fire exactly where the procedure band already fires, i.e. mean + -- something other than its name. The four INSERT sites read raw_contracts.estimated_value the same way. + ( + SELECT rc.estimated_value + FROM raw_contracts rc + WHERE rc.unp = substr(c.tender_id, 3) + AND rc.contract_number = c.contract_number + AND ( + (c.id LIKE 'c:e:%' AND rc.source LIKE 'eop:%') + OR (c.id LIKE 'c:o:%' AND rc.source LIKE 'ocds:%') + ) + ORDER BY rc.source DESC, rc.id DESC + LIMIT 1 + ) AS own_est_native, + -- Same row, same order — the estimate's OWN currency chain, byte-for-byte the one the four INSERT + -- sites use (procurement_currency → currency → BGN). Reading it off contracts.currency instead would + -- convert a foreign-currency estimate at the contract's currency (review cefothe #4). + ( + SELECT COALESCE(NULLIF(rc.procurement_currency, ''), NULLIF(rc.currency, ''), 'BGN') + FROM raw_contracts rc + WHERE rc.unp = substr(c.tender_id, 3) + AND rc.contract_number = c.contract_number + AND ( + (c.id LIKE 'c:e:%' AND rc.source LIKE 'eop:%') + OR (c.id LIKE 'c:o:%' AND rc.source LIKE 'ocds:%') + ) + ORDER BY rc.source DESC, rc.id DESC + LIMIT 1 + ) AS own_est_currency, + -- One annex step jumped ≥10× — half of the mis-keyed-annex conjunction below. Checked against + -- the CUMULATIVE domain amendments, matching where this pass re-rolls current_value from. + EXISTS ( + SELECT 1 FROM amendments am + WHERE am.unp = substr(c.tender_id, 3) + AND am.contract_number = c.contract_number + AND am.value_before > 0 AND am.value_after >= 10 * am.value_before + ) AS has_step10, + -- #305 value double-count: a driving annex whose value_before ≈ a KNOWN prior total — the contract's + -- signing_value OR a preceding annex's value_after (the multi-annex case) — with value_after in + -- [2×,10×) that base, same currency, and matching current_value. Checked against the CUMULATIVE domain + -- amendments, matching where this pass re-rolls current_value from. ЗОП чл.116 caps a single amendment + -- at +50%, so the ≥2× step is the defect signal wherever it sits; slow climbs never reach ≥2×, and the + -- ≥10× mis-key and cross-currency cases are handled elsewhere. + EXISTS ( + SELECT 1 FROM amendments am + WHERE am.unp = substr(c.tender_id, 3) + AND am.contract_number = c.contract_number + -- #305 Tier-2: skip text-treated annexes. Restated totals already stop matching (served value_after + -- is the corrected total, no longer ≈2× before), but the explicit guard also covers confirmed-genuine + -- increments, whose value_after is legitimately ≥2× and must NOT be arithmetic-flagged. + AND am.value_treatment IS NULL + AND am.value_before > 0 AND c.signing_value > 0 + -- #305 multi-annex: value_before may be a prior cumulative total (a preceding annex's + -- value_after), not signing — anchor to signing OR a legitimately-grown prior total, prev not + -- itself a double (see normalize-raw.sql). + AND ( + ABS(am.value_before - c.signing_value) < 0.01 * c.signing_value + OR EXISTS ( + SELECT 1 FROM amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + -- #305 NEW-HIGH-2: this reconciliation reads the CUMULATIVE served `amendments` (a prior-window + -- annex is not in this window's raw_amendments), but the full path anchors on RAW values. For a + -- #305-restated prev the served value_after is the CORRECTED (lower) total, not the raw one, so + -- `value_after < 2*value_before` flips true and the gate would disagree with the full rebuild + -- (flag flips between the daily slice and the next full derive). Restrict the anchor to + -- non-restated prevs, whose served value_after == raw value_after — reproducing the full-path + -- (raw) decision without losing cross-window history. + AND prev.value_restated = 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + -- ...and that prior total was itself reached legitimately (prev not a ≥2× double), + -- so a compounding chain where every step doubles is left untouched, not restated. + AND prev.value_before > 0 AND prev.value_after < 2 * prev.value_before + ) + -- #305 84818-class: an EXACT single-step 2× on an ORPHAN base (value_before ties neither signing + -- nor any prior served annex) is the ЗОП чл.116 defect signature — flag (→ signing fallback, + -- EXCLUDE); never rewrites. The orphan guard leaves compounding chains untouched (see + -- normalize-raw.sql). + OR ( + ABS(am.value_after - 2 * am.value_before) < 0.005 * am.value_before + AND NOT EXISTS ( + SELECT 1 FROM amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_after > 0 + AND ABS(prev.value_after - am.value_before) < 0.01 * am.value_before + ) + ) + ) + AND am.value_after >= 2 * am.value_before AND am.value_after < 10 * am.value_before + -- #305 M2 self-consistency: skip when value_delta is present and a ≉ b + d (model N/A). + AND (am.value_delta IS NULL OR ABS(am.value_after - (am.value_before + am.value_delta)) < 0.01 * am.value_after) + AND ABS(am.value_after - c.current_value) < 0.01 + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) AS has_double, + -- #305 NEW-HIGH-1 (multi-annex chain contamination), slice mirror. The full path (normalize-raw.sql) + -- detects this on RAW values (prev.value_after_restated < prev.value_after AND am.value_before ≈ raw + -- prev.value_after). The slice reads the CUMULATIVE served `amendments`, which does NOT retain the raw + -- value_after of a restated prev — so this is a CONSERVATIVE approximation: a driving annex whose + -- value_before sits ABOVE a restated prior annex's CORRECTED total (it rode the raw, doubled base) but + -- within a contamination band (< 2× the corrected total, i.e. not a fresh legitimate double). Flags → + -- signing fallback, matching the full path's honest exclusion. Exactness is restored on the next full + -- rebuild; a follow-up value_before-propagation PR removes the approximation entirely. + EXISTS ( + SELECT 1 FROM amendments am + WHERE am.unp = substr(c.tender_id, 3) + AND am.contract_number = c.contract_number + AND am.value_treatment IS NULL + AND am.value_before > 0 + AND ABS(am.value_after - c.current_value) < 0.01 + AND EXISTS ( + SELECT 1 FROM amendments prev + WHERE prev.unp = am.unp AND prev.contract_number = am.contract_number + AND prev.value_restated = 1 + AND am.value_before > prev.value_after + AND am.value_before < 2 * prev.value_after + ) + AND COALESCE(NULLIF(am.currency, ''), COALESCE(NULLIF(c.currency, ''), 'BGN')) + = COALESCE(NULLIF(c.currency, ''), 'BGN') + ) AS has_contaminated_base FROM contracts c JOIN tenders te ON te.id = c.tender_id WHERE ( @@ -1649,25 +2243,65 @@ WITH contract_base AS ( ), base AS ( SELECT id, currency, signing_value, current_value, current_value_currency, fx_rate, proc_est_eur, proc_est_native, CASE - WHEN c.value_flag <> 'annex_suspect' - AND NOT (c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100))) + WHEN c.value_flag NOT IN ('annex_suspect', 'annex_total_suspect') + AND NOT (c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 + OR (c.current_value / c.signing_value >= 5 AND c.has_step10))))) + AND NOT (c.current_value IS NOT NULL AND c.signing_value > 0 AND (c.has_double OR c.has_contaminated_base)) THEN c.value_flag - WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND c.eff_eur > 200 * c.proc_est_eur) THEN 'value_suspect' - WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND c.current_value / c.signing_value >= 100)) THEN 'annex_suspect' + WHEN c.eff_eur > 2000000000 OR (c.proc_est_eur >= 1000 AND (c.eff_eur > 200 * c.proc_est_eur + -- Dropped decimal point: the value was entered in стотинки, so it lands at almost exactly + -- 100x the procedure estimate. Real overruns spread out; this is an isolated cluster with + -- nothing between 105x and 200x, so the band is narrow on purpose. + OR (c.eff_eur >= 95 * c.proc_est_eur AND c.eff_eur <= 105 * c.proc_est_eur))) + -- Own-row (per-lot) arm, mirroring the two INSERT-time copies above. + OR (c.own_est_eur >= 1000 AND c.proc_est_eur >= 1000 AND c.eff_eur >= 10 * c.proc_est_eur + AND c.eff_eur >= 95 * c.own_est_eur AND c.eff_eur <= 105 * c.own_est_eur) THEN 'value_suspect' + WHEN c.current_value IS NOT NULL AND (c.current_value < 0 OR (c.signing_value > 0 AND (c.current_value / c.signing_value >= 100 + -- Mis-keyed annex: a single step jumped ≥10× AND the aggregate ended ≥5× over signing. + -- The step alone is NOT enough — some chains have a huge step that a later annex pulls + -- back below signing, and flagging those would RAISE the shown value, not repair it. + OR (c.current_value / c.signing_value >= 5 AND c.has_step10)))) THEN 'annex_suspect' + -- #305 single-annex value double-count: the driving annex more than doubled the contract in one + -- step (ЗОП чл.116 caps a single amendment at +50%). has_double already ties to current_value. + -- has_contaminated_base additionally catches a legitimate-looking later annex riding a doubled base + -- (#305 NEW-HIGH-1) — both fall back to signing. + WHEN c.current_value IS NOT NULL AND c.signing_value > 0 AND (c.has_double OR c.has_contaminated_base) THEN 'annex_total_suspect' WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS new_value_flag - FROM contract_base c + FROM ( + -- Per-lot estimate in EUR for the стотинки band's own-row arm (see the note at the band). Mirrors + -- the four INSERT sites: the row's OWN estimate (NULL when it has none), converted through the + -- estimate's own currency with a dated fx lookup — not through the contract's currency or fx_rate. + SELECT cb.*, + CASE + WHEN cb.own_est_native IS NULL THEN NULL + WHEN cb.own_est_currency = 'EUR' THEN cb.own_est_native + WHEN cb.own_est_currency = 'BGN' THEN cb.own_est_native / 1.95583 + ELSE cb.own_est_native * ( + SELECT f.eur_per_unit + FROM fx_rates f + WHERE f.base_currency = cb.own_est_currency + AND f.rate_date <= cb.signed_at + AND f.rate_date >= date(cb.signed_at, '-10 days') + ORDER BY f.rate_date DESC + LIMIT 1 + ) + END AS own_est_eur + FROM contract_base cb + ) c ), calc AS ( SELECT id, new_value_flag, proc_est_eur, CASE new_value_flag WHEN 'value_suspect' THEN proc_est_native WHEN 'annex_suspect' THEN COALESCE(signing_value, current_value) + WHEN 'annex_total_suspect' THEN COALESCE(signing_value, current_value) ELSE COALESCE(current_value, signing_value) END AS display_native, CASE new_value_flag WHEN 'value_suspect' THEN NULL WHEN 'annex_suspect' THEN COALESCE(signing_value, current_value) + WHEN 'annex_total_suspect' THEN COALESCE(signing_value, current_value) ELSE COALESCE(current_value, signing_value) END AS trusted_native, CASE new_value_flag @@ -1676,13 +2310,17 @@ WITH contract_base AS ( WHEN signing_value IS NOT NULL THEN COALESCE(NULLIF(currency, ''), 'BGN') ELSE COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') END + WHEN 'annex_total_suspect' THEN CASE + WHEN signing_value IS NOT NULL THEN COALESCE(NULLIF(currency, ''), 'BGN') + ELSE COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') + END ELSE CASE WHEN current_value IS NOT NULL THEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') ELSE COALESCE(NULLIF(currency, ''), 'BGN') END END AS trusted_currency, CASE - WHEN new_value_flag IN ('value_suspect', 'annex_suspect') OR current_value IS NULL THEN NULL + WHEN new_value_flag IN ('value_suspect', 'annex_suspect', 'annex_total_suspect') OR current_value IS NULL THEN NULL WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'EUR' THEN current_value WHEN COALESCE(NULLIF(current_value_currency, ''), NULLIF(currency, ''), 'BGN') = 'BGN' THEN current_value / 1.95583 WHEN fx_rate IS NOT NULL THEN current_value * fx_rate @@ -1863,6 +2501,13 @@ FROM interest_links il JOIN persons p ON p.id = il.person_id -- rendering both re-identifies the relative via a ТР owner lookup, and the company is already surfaced -- by the self row. WHERE il.status = 'published' AND il.interest_class IN ('private_ownership', 'family_ownership') + -- …and the identity rests on a Trade Register fact (#279, ADR-0033). This predicate is the THIRD copy + -- of the surface gate — the other two are SURFACED_OWNERSHIP in packages/db/src/queries/related-persons.ts + -- and the sibling block in the other of precompute.sql / refresh-slice.sql. All three must move + -- together: this one feeds the officials search index, so omitting it would keep officials findable + -- whose links no longer surface. + AND EXISTS (SELECT 1 FROM interest_link_evidence e + WHERE e.link_key = il.link_key AND e.evidence_kind IN ('document','confirmed')) AND EXISTS (SELECT 1 FROM contracts cc JOIN bidders bb ON bb.id = cc.bidder_id WHERE bb.eik_normalized = il.eik) AND NOT (il.interest_class = 'family_ownership' AND EXISTS ( diff --git a/scripts/resolve-amendment-contracts.sql b/scripts/resolve-amendment-contracts.sql new file mode 100644 index 00000000..3498505d --- /dev/null +++ b/scripts/resolve-amendment-contracts.sql @@ -0,0 +1,150 @@ +-- Sigma — #306: link EOP annexes whose annex-side number is in a different namespace than the contract +-- number. The annex carries an internal number (e.g. 148846) while the contract on the same procedure +-- carries the buyer's filing number (e.g. Д-226), so the (unp, contract_number) join drops ~7% of EOP +-- annexes out of every annex→contract→company/authority rollup. String normalisation recovers almost none +-- of them (measured), because the two numbers are genuinely unrelated identifiers. Resolve by VALUE +-- instead: an annex's value_before is the contract's value at amendment time, so it equals the target +-- contract's signing_value. Link only when value_before matches EXACTLY (< 0.5 стотинка), currency- and +-- contractor-matched, uniquely one contract on the procedure — measured 99.99% precision on the already- +-- linked corpus (9348/9349). Rows that match 2+ contracts (value-ambiguous) or none (target not yet +-- ingested — the #249 class) are LEFT unlinked: an honest gap beats a wrong contract on a transparency site. +-- +-- FULL-PATH FORM (review nikimilenkov HIGH 1 + todorkolev #3). This file runs from runFullDerive / +-- runWorkBackfill in scripts/import.mjs, BEFORE derive-amendments.sql, with candidates drawn from the whole +-- re-staged raw_contracts. It is deliberately NOT run on the slice path (runSliceDerive): the slice's +-- raw_contracts holds only the current window, so "unique on the procedure" would mean "unique in the window", +-- not in the corpus — a corpus-ambiguous annex would look unique in a narrow window and mislink. The daily/ +-- slice + Worker path instead runs a corpus-safe equivalent INSIDE scripts/refresh-slice.sql (search "#306: +-- slice-safe value-anchor resolver"), whose candidates come from the served `contracts` table plus this +-- window's raw_contracts, so uniqueness is asked corpus-wide and the measured precision carries. Both forms +-- share the same value/currency/EIK anchor, chain rules, and provenance stamping; see +-- docs/implementation-plans/306-amendment-contract-namespace-link.md §4. +-- +-- ORDER (review todorkolev #1, the blocker): this runs BEFORE the #286 prefer-EOP dedup DELETE in +-- derive-amendments.sql. Rewriting an EOP annex onto a contract that already kept an OCDS twin would +-- resurrect that twin (annex_count = 2 on a one-annex contract) and trip the amendment-twin-dedup integrity +-- gate (#303), failing the whole derive. Running first — and above the #286 diagnostics, so their dropped/ +-- excess counts stay honest bounds on what the dedup actually removes — keeps the dedup the sole twin guard. +-- The resolver reads only source LIKE 'eop:%' rows + raw_contracts, so it has no dependency on the OCDS +-- bridge and is safe to run first. + +-- Candidate contracts, deduped to one row per LOGICAL contract. raw_contracts is CUMULATIVE — the EOP daily +-- open-data buckets repeat the same contract across consecutive days, and the collapse to one row per +-- (unp, contract_number) happens later in normalize-raw.sql, NOT in staging (review nikimilenkov HIGH 2). +-- Without this dedup, COUNT(*) OVER below would count staging ROWS, not contracts: a contract present in N +-- daily buckets would read as n_match = N and fail-close every real link, and a superseded row could match +-- an annex to a stale value. Mirror normalize-raw's rule (latest source-day, then highest id, wins). +DROP TABLE IF EXISTS amendment_contract_resolve; +CREATE TABLE amendment_contract_resolve AS +WITH contract_candidates AS ( + SELECT unp, contract_number, signing_value, currency, contractor_eik + FROM ( + SELECT c.unp, c.contract_number, c.signing_value, c.currency, c.contractor_eik, + ROW_NUMBER() OVER ( + PARTITION BY c.unp, c.contract_number + ORDER BY c.source DESC, c.id DESC + ) AS rn + FROM raw_contracts c + WHERE c.contract_number IS NOT NULL + AND c.signing_value IS NOT NULL AND c.signing_value > 0 + ) + WHERE rn = 1 +), +-- Every EOP annex whose (unp, contract_number) matches no contract — the namespace-mismatch group. Includes +-- value-less members (value_before NULL/≤0): an admin/term annex mid-chain carries no signing value of its +-- own but must still inherit the chain's target (review nikimilenkov MEDIUM 2), else the chain breaks and +-- current_value stops early. Grouped by the shared annex-side number (unp, annex_cnum) = one contract's chain. +grp AS ( + SELECT a.id AS amendment_id, a.unp, a.contract_number AS annex_cnum, + a.value_before, a.currency, a.contractor_eik + FROM raw_amendments a + WHERE a.source LIKE 'eop:%' + AND a.unp IS NOT NULL AND a.contract_number IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM raw_contracts c + WHERE c.unp = a.unp AND c.contract_number = a.contract_number + ) +), +-- Exact, currency- and contractor-matched value anchor. Only members that carry a usable value_before +-- participate in matching. The currency guard requires an EXPLICIT currency on both sides (review +-- nikimilenkov LOW 1): a blank-vs-blank pair must not silently agree via a 'BGN' default under so tight a +-- gate. The EIK guard (review nikimilenkov MEDIUM 5) is null-tolerant — the annex already carries the +-- contractor, so a value collision onto a DIFFERENT contractor's contract is refused for free. +vmatch AS ( + SELECT g.amendment_id, g.unp, g.annex_cnum, c.contract_number AS resolved_cnum, + COUNT(*) OVER (PARTITION BY g.amendment_id) AS n_match + FROM grp g + JOIN contract_candidates c + ON c.unp = g.unp + AND ABS(c.signing_value - g.value_before) < 0.005 + AND NULLIF(c.currency, '') IS NOT NULL + AND NULLIF(g.currency, '') IS NOT NULL + AND c.currency = g.currency + AND (g.contractor_eik IS NULL OR c.contractor_eik IS NULL OR g.contractor_eik = c.contractor_eik) + WHERE g.value_before IS NOT NULL AND g.value_before > 0 +), +-- `direct` — each member's OWN unique (n_match = 1) exact value match. This is trustworthy on its own +-- (exact cent + unique on the whole procedure = the measured 99.99% precision), so a direct hit always +-- applies. Crucially it is NOT voided when its annex-number siblings point elsewhere: an annex-side number +-- can be a LOT-BASE shared across contracts (real corpus: `20РП-У50А015` → …-Л01 @ 22569.98 AND …-Л03 @ +-- 28557.50, each annex exactly-uniquely matching its own lot). The one disagreement group in the whole live +-- corpus is exactly this benign multi-lot case — refusing it (an earlier revision, review nikimilenkov +-- MEDIUM 1) dropped 2 confirmed-correct links and prevented zero wrong ones, so disagreement withholds +-- PROPAGATION only, never the direct hits themselves. +direct AS ( + SELECT amendment_id, unp, annex_cnum, resolved_cnum FROM vmatch WHERE n_match = 1 +), +-- `group_target` — propagate ONE agreed target across a `(unp, annex-number)` chain to members that have no +-- own unique match (value-less admin steps, or later steps whose cumulative value matches no signing_value), +-- but only when the direct members AGREE (a lot-base spread disagrees → no propagation, and the direct hits +-- still stand). So later chain steps link and `current_value` reflects the last step (review MEDIUM 2). +group_target AS ( + SELECT unp, annex_cnum, MIN(resolved_cnum) AS resolved_cnum + FROM direct GROUP BY unp, annex_cnum HAVING COUNT(DISTINCT resolved_cnum) = 1 +) +-- A member links to its OWN unique match if it has one; else to the agreed group target — UNLESS the member +-- is itself value-ambiguous (n_match >= 2), which carries its own contradicting evidence and never links, +-- directly or by inheritance (review todorkolev #2). Members with neither a direct hit nor an agreed group +-- target keep resolved_cnum NULL and stay unlinked. +SELECT g.amendment_id, + COALESCE( + (SELECT d.resolved_cnum FROM direct d WHERE d.amendment_id = g.amendment_id), + (SELECT gt.resolved_cnum FROM group_target gt WHERE gt.unp = g.unp AND gt.annex_cnum = g.annex_cnum) + ) AS resolved_cnum +FROM grp g +WHERE NOT EXISTS ( + SELECT 1 FROM vmatch v WHERE v.amendment_id = g.amendment_id AND v.n_match >= 2 +); + +-- The rewrite UPDATE below correlates raw_amendments.id to amendment_contract_resolve.amendment_id once per +-- unlinked row; index the resolve table so that is a lookup, not a scan (review nikimilenkov LOW 4). +CREATE INDEX IF NOT EXISTS idx_amendment_contract_resolve_id + ON amendment_contract_resolve(amendment_id); + +-- Rewrite contract_number in place — exactly like the #286 УНП bridge — and PRESERVE PROVENANCE (review +-- nikimilenkov MEDIUM 4): keep the original annex-side number in contract_number_raw and stamp link_method +-- so the value-linked rows stay enumerable in staging, through promote, and in the served `amendments` +-- table. contract_number_raw also keeps the annex number in the amendment natural_key (see derive- +-- amendments.sql / promote-amendments.sql), so a resolved row never collides with a native annex that +-- happens to share document_number on the target contract (review nikimilenkov MEDIUM 3). +UPDATE raw_amendments +SET + contract_number_raw = contract_number, + link_method = 'value_anchor', + contract_number = ( + SELECT r.resolved_cnum FROM amendment_contract_resolve r WHERE r.amendment_id = raw_amendments.id + ) +WHERE id IN (SELECT amendment_id FROM amendment_contract_resolve WHERE resolved_cnum IS NOT NULL); + +-- #306 diagnostic (printed by wrangler): annexes linked by the value anchor, and those still unlinked. The +-- two predicates are complementary — both count over the SAME namespace-mismatch population (review +-- nikimilenkov LOW 3) — so linked + still_unlinked = the original mismatch count on a healthy run. +SELECT + (SELECT COUNT(*) FROM raw_amendments WHERE link_method = 'value_anchor') AS annexes_value_linked, + (SELECT COUNT(*) FROM raw_amendments a + WHERE a.source LIKE 'eop:%' AND a.unp IS NOT NULL AND a.contract_number IS NOT NULL + AND a.link_method IS NULL + AND NOT EXISTS (SELECT 1 FROM raw_contracts c + WHERE c.unp = a.unp AND c.contract_number = a.contract_number)) AS eop_annexes_still_unlinked; + +DROP TABLE IF EXISTS amendment_contract_resolve; diff --git a/scripts/seed.sql b/scripts/seed.sql index 2ff25add..3d26a0da 100644 --- a/scripts/seed.sql +++ b/scripts/seed.sql @@ -16,3 +16,103 @@ INSERT OR IGNORE INTO bidders (id, name, bulstat) VALUES ('bidder-a', 'Алфа ЕООД', '111111111'), ('bidder-b', 'Бета АД', '222222222'), ('bidder-c', 'Гама ООД', '333333333'); + +-- ── свързани лица (conflict-of-interest) smoke fixture ──────────────────────────────────────────── +-- Without these the /conflicts routes render an empty surface locally, so the whole feature is +-- unverifiable in a browser. Every person and company below is INVENTED — no real declarant, no real +-- winner. The ЕИК pass the real 9-digit checksum (scripts/normalize-raw.sql's eik_valid), so the +-- fixture cannot teach a wrong ЕИК shape to anyone reading it. +-- +-- The rows are shaped by what the read gate actually requires (packages/db/src/queries/related-persons.ts): +-- • status='published' AND interest_class IN (private_ownership, family_ownership) — SURFACED_OWNERSHIP +-- • a LIVE contract for that ЕИК — the read-time zero-contract gate +-- • signed_at inside [first_declared_year, last_declared_year] — IN_WINDOW ⇒ the „в периода" chip +-- • no published SELF stake on the same (official, ЕИК) for a family row — NOT_REDUNDANT_FAMILY +-- • a Trade Register evidence seal of a PUBLISHING rung — the #279 seal gate +-- One link per outcome, including the non-surfaced ones, so a change to the publishing rule is +-- visible as a before/after rather than as an empty page either way. + +INSERT OR IGNORE INTO authorities (id, name, bulstat, region) VALUES + ('auth-plovdiv', 'Община Пример', '131223340', 'Пловдив'); + +INSERT OR IGNORE INTO tenders + (id, source_id, title, authority_id, cpv_code, estimated_value, currency, procedure_type, status, published_at, deadline_at) +VALUES + ('t-conflict-01', 'AOP-2022-0101', 'Строителен надзор на общински обекти', 'auth-plovdiv', '71520000', 900000, 'BGN', 'открита процедура', 'closed', '2022-01-10', '2022-02-10'), + ('t-conflict-02', 'AOP-2022-0102', 'Консултантски услуги по проект', 'auth-sofia', '79400000', 400000, 'BGN', 'открита процедура', 'closed', '2022-04-05', '2022-05-05'), + ('t-conflict-03', 'AOP-2022-0103', 'Доставка на офис оборудване', 'auth-mrrb', '30190000', 250000, 'BGN', 'открита процедура', 'closed', '2022-06-01', '2022-07-01'); + +-- eik_valid=1 so these behave like resolved winners; settlement feeds the seat-confirmation rung. +INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, legal_form, settlement) VALUES + ('eik:201122335', 'АЛФА СТРОЙ ООД', '201122335', '201122335', 1, 'ООД', 'Пловдив'), + ('eik:203445566', 'БЕТА КОНСУЛТ ЕООД', '203445566', '203445566', 1, 'ЕООД', 'София'), + ('eik:204556676', 'ГАМА ИНВЕСТ АД', '204556676', '204556676', 1, 'АД', 'Варна'); + +INSERT OR IGNORE INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, amount_eur) VALUES + ('c-conflict-01', 't-conflict-01', 'eik:201122335', 840000, 'BGN', '2022-03-14', 'Д-2022-101', 429000), + ('c-conflict-02', 't-conflict-02', 'eik:203445566', 372000, 'BGN', '2022-06-20', 'Д-2022-102', 190000), + ('c-conflict-03', 't-conflict-03', 'eik:204556676', 236000, 'BGN', '2022-08-02', 'Д-2022-103', 120000); + +-- person id = 'person:' || key(name) || '|' || key(institution) — (name, institution), never a bare +-- name (ADR-0026), so two namesakes at different bodies stay distinct. +INSERT OR IGNORE INTO persons (id, name) VALUES + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'Иван Петров Тестов'), + ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ', 'Мария Георгиева Образцова'); + +INSERT OR IGNORE INTO declarations + (id, person_id, xml_file, control_hash, folder_year, declared_year, template, category, institution, position, source_url) +VALUES + ('decl:2023:demo-ivan.xml', 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'demo-ivan.xml', 'demo-hash-ivan', '2023', '2023', 'assets', 'Местна власт', 'Община Пример', 'Кмет', 'https://register.cacbg.bg/2023/demo-ivan.xml'), + ('decl:2023:demo-maria.xml', 'person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ', 'demo-maria.xml', 'demo-hash-maria', '2023', '2023', 'assets', 'Местна власт', 'Община София', 'Директор дирекция', 'https://register.cacbg.bg/2023/demo-maria.xml'); + +INSERT OR IGNORE INTO declared_interests (id, declaration_id, entity_raw, entity_key, kind, detail, timing, seat) VALUES + ('di:decl:2023:demo-ivan.xml:1', 'decl:2023:demo-ivan.xml', 'АЛФА СТРОЙ ООД', 'АЛФА СТРОЙ ООД', 'shares', '50%', 'annual', 'Пловдив'), + ('di:decl:2023:demo-ivan.xml:2', 'decl:2023:demo-ivan.xml', 'ГАМА ИНВЕСТ АД', 'ГАМА ИНВЕСТ АД', 'shares', '0,5%', 'annual', 'Варна'), + ('di:decl:2023:demo-maria.xml:1', 'decl:2023:demo-maria.xml', 'БЕТА КОНСУЛТ ЕООД', 'БЕТА КОНСУЛТ ЕООД', 'shares', '100%', 'annual', 'София'); + +-- Three links, three outcomes: +-- published private_ownership — the official's own stake, own_institution='exact' (the strongest signal) +-- published family_ownership — a relative's declared stake; the relative is never named (ADR-0032) +-- held (АД, C_hold) — a joint-stock parcel: collected, never surfaced (ADR-0022 materiality) +INSERT OR IGNORE INTO interest_links + (id, link_key, person_id, bidder_id, eik, entity_key, match_method, matcher_version, publish_tier, + relation, interest_class, contemporaneous, own_institution, evidence_count, + first_declared_year, last_declared_year, contract_count, contract_value_eur, + first_contract_year, last_contract_year, status) +VALUES + ('il:person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', + 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'eik:201122335', '201122335', 'АЛФА СТРОЙ ООД', + 'exact_name_key', 'seed-demo', 'B_distinctive', 'owns', 'private_ownership', 1, 'exact', 1, + '2021', '2023', 1, 429000, '2022', '2022', 'published'), + ('il:person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', + 'person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ', 'eik:203445566', '203445566', 'БЕТА КОНСУЛТ ЕООД', + 'exact_name_key', 'seed-demo', 'A_seat', 'related', 'family_ownership', 1, 'exact', 1, + '2021', '2023', 1, 190000, '2022', '2022', 'published'), + ('il:person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|204556676', 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|204556676', + 'person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР', 'eik:204556676', '204556676', 'ГАМА ИНВЕСТ АД', + 'exact_name_key', 'seed-demo', 'C_hold', 'owns', 'management_role', 0, 'none', 1, + '2021', '2023', 1, 120000, '2022', '2022', 'held'); + +INSERT OR IGNORE INTO interest_link_authorities (link_key, authority_id, authority_name, contract_count, value_eur, own) VALUES + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'auth-plovdiv', 'Община Пример', 1, 429000, 'exact'), + ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'auth-sofia', 'Община София', 1, 190000, 'exact'); + +-- Trade Register evidence seals (#279, ADR-0033, migration 0006). NOT optional decoration: since the +-- evidence ladder landed, SURFACED_OWNERSHIP requires a publishing seal, so a seeded link without one +-- renders NOTHING — which is the exact failure this fixture exists to prevent. A seal per link, one per +-- rung, so the dev surface shows the same three outcomes as production: +-- document — the register names this person in this company (the strongest rung) +-- confirmed — identity confirmed by declared data (seat), no person found in the act itself +-- bar_joint_stock — an АД: a declared parcel of shares is not a material ownership conflict +-- matched_fact stays inside the closed vocabulary (evidence.mjs isSealedFact) — never a name. +INSERT OR IGNORE INTO interest_link_evidence + (link_key, evidence_kind, registry_role, matched_fact, entry_number, entry_date, lookup_date, rules_version, live_status) +VALUES + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|201122335', 'document', 'owner', 'role:owner:CR_F_19_L', + '20220314150210', '2022-03-14', '2026-08-05', 'tr-rules-1', 'live'), + ('person:МАРИЯ ГЕОРГИЕВА ОБРАЗЦОВА|ОБЩИНА СОФИЯ|203445566|family', 'confirmed', NULL, 'seat:СОФИЯ', + '20210902110455', '2021-09-02', '2026-08-05', 'tr-rules-1', 'live'), + -- The held link is sealed too — seals exist for held links so the review queue can explain itself, + -- and this one doubles as the fixture's negative case for the seal gate. + ('person:ИВАН ПЕТРОВ ТЕСТОВ|ОБЩИНА ПРИМЕР|204556676', 'bar_joint_stock', NULL, NULL, + NULL, NULL, '2026-08-05', 'tr-rules-1', 'live'); diff --git a/scripts/ship-e2e.test.mjs b/scripts/ship-e2e.test.mjs new file mode 100644 index 00000000..1d150914 --- /dev/null +++ b/scripts/ship-e2e.test.mjs @@ -0,0 +1,379 @@ +// End-to-end coverage of the SHIP PATH ITSELF — the one thing unit tests of the helpers cannot give. +// +// Three times a refactor of this script silently dropped a guarantee while the suite stayed green: +// when main() called applyTableChunks, again after that moved into runShip(), and again when a first +// cut of THIS file asserted only on request filenames and counts — so mutations that shipped an empty +// payload, a wipe that deleted nothing, or every request to a PRODUCTION slot all passed. +// +// The fix is to stop trusting a stub: the fake `wrangler` here APPLIES each --file to a real sqlite +// target and answers each --command FROM that target. The assertions are then about what the target +// actually holds, which no amount of correct-looking request plumbing can fake. +// +// Run: node --test scripts/ship-e2e.test.mjs +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + MAX_BATCH_ROWS, + MAX_STATEMENTS_PER_REQUEST, + PACE_MS, + TABLES, +} from './ship-related-persons.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(HERE, '..'); +const SCRIPT = resolve(HERE, 'ship-related-persons.mjs'); +const MIG = resolve(ROOT, 'packages/db/migrations/0003_related_persons_foundation.sql'); +// 0006 too: the evidence seal is one of the shipped tables (#279, ADR-0033), so it appears in the ship's +// TABLES and WIPE_ORDER. A target built from 0003 alone makes the generated wipe abort on „no such table" +// before a single row moves — the schema under test has to be the schema the ship writes. +const MIG_EVIDENCE = resolve(ROOT, 'packages/db/migrations/0009_interest_link_evidence.sql'); +const D1_NAME = 'sigma-test-local'; + +// Derived from BOTH production constants, and the run below does NOT override either: one row past +// what a single default request can carry, so chunking is exercised at the shipped settings. Forcing +// it with --max-statements-per-request=1 (the first cut of this file) left the real constant free to +// be retuned to infinity with the suite still green. +const LINKS = MAX_STATEMENTS_PER_REQUEST * MAX_BATCH_ROWS + 1; +const EXPECTED_CHUNKS = Math.ceil(LINKS / MAX_BATCH_ROWS / MAX_STATEMENTS_PER_REQUEST); +// The failure-mode tests below do not need the full-size corpus — they force chunking with a flag and +// keep the fixture small, so only the one test that constrains the defaults pays for 10k rows. +const LINKS_SMALL = MAX_BATCH_ROWS * 2 + 1; + +const sqlite = (db, input) => execFileSync('sqlite3', ['-bail', db], { input, stdio: 'pipe' }); + +// Unlike the other scripts/*.test.mjs this one needs the `sqlite3` BINARY (present on ubuntu-latest +// and in the devcontainer): the fake wrangler applies real SQL to a real database, which is the whole +// reason these assertions mean anything. Say so up front — a missing binary would otherwise surface as +// an opaque ENOENT from whichever test happened to run first. Failing, never skipping: a skip here +// silently returns the suite to the state where mutations walked through it. +if (spawnSync('sqlite3', ['-version'], { stdio: 'ignore' }).error) { + throw new Error('scripts/ship-e2e.test.mjs requires the sqlite3 binary on PATH'); +} + +/** The served свързани-лица tables plus the two FK parents they reference. */ +const SCHEMA = `PRAGMA foreign_keys=ON; +CREATE TABLE bidders(id TEXT PRIMARY KEY); +CREATE TABLE authorities(id TEXT PRIMARY KEY); +.read ${MIG} +.read ${MIG_EVIDENCE} +INSERT INTO bidders(id) VALUES('eik:1'); +INSERT INTO authorities(id) VALUES('auth:1');`; + +const corpus = (links) => ` +INSERT INTO persons(id,name) VALUES('p1','П Тест'); +INSERT INTO declarations(id,person_id,xml_file,folder_year,template,source_url) VALUES('d1','p1','x.xml','2024','assets','u'); +INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VALUES('di1','d1','E','e','shares'); +${Array.from( + { length: links }, + (_, i) => + `INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('il${i}','p1|${i}','p1','eik:1','1','e','v1','B_distinctive','owns','published');\n` + + // One seal per link. Since #279 a published link without one is exactly the state the audit fails + // the run on (C_no_evidence) and the read gate refuses to surface, so a corpus without seals is not + // a shape the ship should ever be asked to carry. + `INSERT INTO interest_link_evidence(link_key,evidence_kind,lookup_date,rules_version,live_status) VALUES('p1|${i}','document','2026-08-13','tr-rules-1','live');`, +).join('\n')} +`; +// interest_link_authorities is deliberately left EMPTY above: its expected count is 0, which is the +// only shape where `Number(null) === 0` would let an unanswered read-back pass for "table is empty". + +const STALE = ` +INSERT INTO persons(id,name) VALUES('stale','Стар запис'); +INSERT INTO declarations(id,person_id,xml_file,folder_year,template,source_url) VALUES('sd','stale','s.xml','2019','assets','u'); +INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VALUES('sdi','sd','S','s','shares'); +INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('sil','stale|0','stale','eik:1','1','s','v0','B_distinctive','owns','published'); +INSERT INTO interest_link_authorities(link_key,authority_id,authority_name) VALUES('stale|0','auth:1','A'); +INSERT INTO interest_link_evidence(link_key,evidence_kind,lookup_date,rules_version,live_status) VALUES('stale|0','confirmed','2019-01-01','tr-rules-0','live');`; + +const EXPECTED_ROWS = { + persons: 1, + declarations: 1, + declared_interests: 1, + interest_links: LINKS, + interest_link_evidence: LINKS, // one seal per link (#279) + interest_link_authorities: 0, +}; + +/** + * A `wrangler` that touches no network but is otherwise faithful: it records the FULL argv, applies + * every --file to the target sqlite DB with foreign keys ON (so a wrong wipe order fails exactly as + * D1 would), and answers every --command from that same DB. Notices go to stderr and pure JSON to + * stdout, mirroring real `wrangler --json`. + * + * SHIP_FAKE_SKIP — drop one --file by name, to simulate a request that never landed. + * SHIP_FAKE_NULLN — answer the read-back with a non-numeric count, to exercise the fail-closed guard. + * SHIP_FAKE_NOISE — emit a `[WARNING]`-shaped line on stdout before the JSON. + * SHIP_FAKE_READFAIL — make the read-back call itself fail, so the catch path is exercised. + */ +function fakeWrangler(dir) { + const bin = join(dir, 'bin'); + mkdirSync(bin, { recursive: true }); + const log = join(dir, 'calls.jsonl'); + const target = join(dir, 'target.sqlite'); + // Seeded with STALE rows on purpose: against an empty target a wipe that deletes nothing is + // indistinguishable from a correct one, and that mutation escaped the first cut of this test. + sqlite(target, SCHEMA + STALE); + const exe = join(bin, 'wrangler'); + writeFileSync( + exe, + `#!/usr/bin/env node +import { appendFileSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +const argv = process.argv.slice(2); +const at = (f) => (argv.indexOf(f) >= 0 ? argv[argv.indexOf(f) + 1] : null); +const file = at('--file'); +const command = at('--command'); +const TARGET = ${JSON.stringify(target)}; +appendFileSync(${JSON.stringify(log)}, JSON.stringify({ argv, file: file && file.split('/').pop() }) + '\\n'); +const run = (input) => + execFileSync('sqlite3', ['-bail', TARGET], { input: 'PRAGMA foreign_keys=ON;\\n' + input, encoding: 'utf8' }); +try { + if (file) { + const skip = process.env.SHIP_FAKE_SKIP; + if (!skip || !file.endsWith(skip)) run('.read ' + file); + } else if (command) { + if (process.env.SHIP_FAKE_READFAIL) { process.stderr.write('read-back exploded'); process.exit(1); } + const rows = JSON.parse(run('.mode json\\n' + command) || '[]'); + const shaped = process.env.SHIP_FAKE_NULLN + ? rows.map((r) => (r.t === process.env.SHIP_FAKE_NULLN ? { ...r, n: null } : r)) + : rows; + if (process.env.SHIP_FAKE_NOISE) process.stdout.write('▲ [WARNING] Processing wrangler.jsonc\\n'); + process.stdout.write(JSON.stringify([{ results: shaped, success: true }])); + } +} catch (err) { + process.stderr.write(String(err.stderr || err.message)); + process.exit(1); +} +`, + { mode: 0o755 }, + ); + chmodSync(exe, 0o755); + // package.json so the extensionless fake is unambiguously ESM wherever os.tmpdir() lives. + writeFileSync(join(bin, 'package.json'), '{"type":"module"}'); + return { + bin, + target, + calls: () => + readFileSync(log, 'utf8') + .trim() + .split('\n') + .filter(Boolean) + .map((l) => JSON.parse(l)), + count: (t) => + Number(execFileSync('sqlite3', [target, `SELECT COUNT(*) FROM ${t};`]).toString()), + }; +} + +function runShip( + dir, + { + env = {}, + links = LINKS_SMALL, + forceChunks = true, + minLinks = 1, + remote = false, + yes = false, + emit = null, + } = {}, +) { + const work = join(dir, 'work.sqlite'); + sqlite(work, SCHEMA + corpus(links)); + const fake = fakeWrangler(dir); + const res = spawnSync( + process.execPath, + [ + SCRIPT, + `--work-db=${work}`, + ...(remote ? ['--remote'] : ['--local']), + ...(yes ? ['--yes'] : []), + ...(emit ? [`--emit=${emit}`] : []), + `--min-links=${minLinks}`, + // The pacing delay is always zeroed to keep the suite quick — it is covered by the runShip unit + // tests. Whether the REQUEST SIZE is overridden matters: the defaults-constraining test leaves + // it alone on purpose. + '--pace-ms=0', + ...(forceChunks ? ['--max-statements-per-request=1'] : []), + ], + { + cwd: ROOT, + encoding: 'utf8', + env: { + ...process.env, + ...env, + SIGMA_D1_NAME: D1_NAME, + PATH: `${fake.bin}:${process.env.PATH}`, + }, + }, + ); + return { res, fake }; +} + +test('a real ship run leaves the target holding exactly what the work DB held', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + const { res, fake } = runShip(dir, { links: LINKS, forceChunks: false }); + assert.equal(res.status, 0, `ship failed:\n${res.stderr}`); + + // The assertion that no amount of correct-looking plumbing can fake: the SQL really applied. + for (const [table, n] of Object.entries({ ...EXPECTED_ROWS, interest_links: LINKS })) + assert.equal(fake.count(table), n, `${table} did not land`); + + const calls = fake.calls(); + // Every request must name the declared DB and stay local — a mutation that retargets a production + // slot is the single highest-consequence regression this script can suffer. + for (const c of calls) { + assert.deepEqual(c.argv.slice(0, 3), ['d1', 'execute', D1_NAME]); + assert.ok(c.argv.includes('--local'), `request escaped --local: ${c.argv.join(' ')}`); + assert.ok(!c.argv.includes('--remote'), `request went remote: ${c.argv.join(' ')}`); + } + + const applies = calls.filter((c) => c.file); + assert.match(applies[0].file, /^0_wipe\./, 'the wipe must be the first request'); + + // Chunking: a table past the batch budget must arrive as several CONTIGUOUSLY numbered requests. + const nums = applies + .map((c) => /^interest_links\.(\d+)\./.exec(c.file)) + .filter(Boolean) + .map((m) => Number(m[1])); + assert.ok( + nums.length >= 2, + `interest_links must be chunked, got ${JSON.stringify(applies.map((c) => c.file))}`, + ); + assert.equal(nums.length, EXPECTED_CHUNKS); + assert.deepEqual( + nums, + Array.from({ length: nums.length }, (_, i) => i + 1), + ); + + // The read-back must be the LAST thing the run does, and must count each table from that table. + const last = calls.at(-1); + assert.ok(last.argv.includes('--command'), 'the read-back must come after the inserts'); + const sql = last.argv[last.argv.indexOf('--command') + 1]; + for (const table of TABLES) + assert.match( + sql, + new RegExp(`COUNT\\(\\*\\) AS n FROM "${table}"`), + `${table} not really counted`, + ); +}); + +test('a request that never landed fails the run', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-short-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + // The skip targets a LEAF table on purpose. Since #279 the ship also carries interest_link_evidence, + // which has an FK to interest_links — so dropping an interest_links chunk now fails on the FOREIGN KEY + // as the orphaned seals land, before the read-back ever runs. That is a stronger guard, but it would + // leave the read-back gate itself unexercised, which is what this test is for. A seal chunk has no + // dependents, so its loss is invisible until the counts are compared. + const { res } = runShip(dir, { env: { SHIP_FAKE_SKIP: 'interest_link_evidence.2.sql' } }); + assert.notEqual(res.status, 0, 'a short target must fail the run'); + assert.match(res.stderr, /ship verification FAILED/); + assert.match(res.stderr, /interest_link_evidence: shipped \d+, target has \d+/); +}); + +test('a read-back that answers with a non-number fails closed', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-nan-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + // `Number(null)` is 0, which would read as "the table is empty" and quietly pass. + const { res } = runShip(dir, { env: { SHIP_FAKE_NULLN: 'interest_link_authorities' } }); + assert.notEqual(res.status, 0, 'an unanswered count must fail the run'); + assert.match(res.stderr, /ship verification FAILED/); +}); + +test('a bracketed notice on stdout does not corrupt the read-back', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-noise-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + // This is a HYPOTHETICAL, and saying so is the point. `wrangler d1 execute --json` was run against + // the real tool afterwards: the notice goes to stderr, stdout is clean JSON, and execFileSync + // returns stdout alone. So this models a stream layout wrangler does not currently produce. + // + // It stays as a cheap guard against a future release moving notices onto stdout — but the PR that + // added it billed the scan as fixing an observed failure, which it never was. A fake is a claim + // about the world; this one went unchecked, the suite was green, and the false claim shipped. + // Anything modelled here that has not been confirmed against the real binary gets labelled as such. + const { res } = runShip(dir, { env: { SHIP_FAKE_NOISE: '1' } }); + assert.equal(res.status, 0, `a stdout notice broke the read-back:\n${res.stderr}`); +}); + +// The guards below all sit at UNPROTECTED call sites: deleting each one, or moving it after the +// destructive run, left the whole suite green. They are the last thing standing between a mistyped +// flag and a wiped production surface, so each gets an end-to-end test that also proves NO request +// was issued before the refusal. +// `d1 info` (the id resolution the authorization guard itself needs) is a read and is fine; what must +// never happen before a refusal is an `execute`, which is what carries the wipe. +const noWrites = (fake) => { + try { + return !fake.calls().some((c) => c.argv[0] === 'd1' && c.argv[1] === 'execute'); + } catch { + return true; // the log file is only created by the first invocation + } +}; + +test('an under-floor corpus refuses to wipe, before any request', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-floor-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + const { res, fake } = runShip(dir, { minLinks: LINKS_SMALL + 1 }); + assert.notEqual(res.status, 0); + assert.match(res.stderr, /refusing to ship/i); + assert.ok(noWrites(fake), 'the refusal must come before the wipe'); +}); + +test('a bare --remote refuses without --yes, before any request', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-remote-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + const { res, fake } = runShip(dir, { remote: true }); + assert.notEqual(res.status, 0); + assert.match(res.stderr, /--remote requires --yes/); + assert.ok(noWrites(fake), 'the refusal must come before the wipe'); +}); + +test('a --remote ship with no declared environment refuses, before any request', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-env-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + const { res, fake } = runShip(dir, { remote: true, yes: true, env: { SIGMA_SHIP_ENV: '' } }); + assert.notEqual(res.status, 0); + assert.match(res.stderr, /SIGMA_SHIP_ENV/); + assert.ok(noWrites(fake), 'the refusal must come before the wipe'); +}); + +test('a read-back that cannot answer at all fails the run', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-readfail-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + // „a verification step that cannot verify must not pass" — the catch returning {} is what makes + // that true, and returning the expectation instead would silently pass. + const { res } = runShip(dir, { env: { SHIP_FAKE_READFAIL: '1' } }); + assert.notEqual(res.status, 0); + assert.match(res.stderr, /ship verification FAILED/); + assert.match(res.stderr, /no answer/); +}); + +test('--emit writes a guarded wipe plus one file per table, and touches no database', (t) => { + const dir = mkdtempSync(join(tmpdir(), 'ship-e2e-emit-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + + const out = join(dir, 'emitted'); + const { res, fake } = runShip(dir, { emit: out }); + assert.equal(res.status, 0, `emit failed:\n${res.stderr}`); + assert.ok(noWrites(fake), '--emit must not touch a database'); + + const wipe = readFileSync(join(out, '0_wipe.sql'), 'utf8'); + assert.match(wipe, /DESTRUCTIVE, UNGUARDED/, 'the emitted wipe must carry its warning header'); + for (const table of TABLES) assert.match(wipe, new RegExp(`DELETE FROM "${table}"`)); + for (const table of TABLES) { + const body = readFileSync(join(out, `${table}.sql`), 'utf8'); + if (table === 'interest_link_authorities') assert.equal(body, '', 'empty table, empty file'); + else assert.match(body, new RegExp(`INSERT INTO "${table}"`)); + } +}); diff --git a/scripts/ship-related-persons.mjs b/scripts/ship-related-persons.mjs index 54da33de..543ce2b5 100644 --- a/scripts/ship-related-persons.mjs +++ b/scripts/ship-related-persons.mjs @@ -28,6 +28,8 @@ export const TABLES = [ 'declarations', 'declared_interests', 'interest_links', + // AFTER interest_links: a seal references its link, so inserting it first fails the FK (#279). + 'interest_link_evidence', 'interest_link_authorities', ]; // DELETE order for the pre-insert wipe — children before parents. related_persons_internal (PII, never @@ -35,6 +37,8 @@ export const TABLES = [ // carrying internal rows would block DELETE FROM declarations. export const WIPE_ORDER = [ 'interest_link_authorities', + // BEFORE interest_links, for the mirror reason: deleting a link whose seal survives fails the FK. + 'interest_link_evidence', 'related_persons_internal', 'interest_links', 'declared_interests', @@ -45,7 +49,81 @@ export function wipeSql() { return WIPE_ORDER.map((t) => `DELETE FROM ${sqlIdent(t)};`).join('\n') + '\n'; } const MAX_BATCH_BYTES = 90_000; -const MAX_BATCH_ROWS = 400; +export const MAX_BATCH_ROWS = 400; + +// One `d1 execute --file` per TABLE meant the whole table went up as a single bulk import: on the first +// full-corpus ship that was 516k rows written in one hour with a p90 batch time of 18.2s, two orders of +// magnitude above a normal cron hour (2.9-4.5k rows, 19-450ms) — after which the database returned +// „internal error" for hours, including on its own metadata endpoint, before recovering with all data +// intact. The Cloudflare-side mechanism is not provable from outside, so this is a deliberate defensive +// bound rather than a proven fix: cap how much one request carries and leave a gap between requests, so a +// re-seed is a series of ordinary writes instead of one shock. 25 × MAX_BATCH_ROWS = 10 000 rows/request. +export const MAX_STATEMENTS_PER_REQUEST = 25; +export const PACE_MS = 500; + +/** Group per-table INSERT statements into request-sized chunks. Pure — unit-tested. */ +export function chunkStatements(statements, maxPerRequest = MAX_STATEMENTS_PER_REQUEST) { + if (!Number.isInteger(maxPerRequest) || maxPerRequest < 1) + throw new Error( + `maxPerRequest must be a positive integer, got ${JSON.stringify(maxPerRequest)}`, + ); + const chunks = []; + for (let i = 0; i < statements.length; i += maxPerRequest) + chunks.push(statements.slice(i, i + maxPerRequest)); + return chunks; +} + +/** Block the (synchronous) ship loop without burning CPU. */ +const sleepSync = (ms) => { + if (ms > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +}; + +/** + * The whole destructive live path: wipe, then paced request-sized inserts per table, then the read-back + * check. Both guarantees live HERE, behind injected I/O (`apply`, `sleep`, `readCounts`), because both are + * one refactor away from silently vanishing — „one request per table" is exactly the shape this drifts back + * to, and a `if (!emit) assert…` line at a call site is exactly the kind of line that gets dropped. + * Testing the pure helpers alone did NOT catch either: reverting the call site and deleting the + * verification each left the whole suite green. Keep the orchestration itself covered. + * @returns {Record} rows shipped per table + */ +export function runShip({ + tables, + readTable, + wipeSql, + apply, + sleep, + readCounts, + maxStatements, + paceMs, +}) { + // ONE counter for the whole run, not one per table. Pacing per table left every table boundary + // unpaced — including wipe → first insert, which is the single most destructive transition here. + let requests = 0; + const applyPaced = (label, sql) => { + if (requests++) sleep(paceMs); // between requests only — never before the first + apply(label, sql); + }; + + applyPaced('0_wipe', wipeSql); + + const summary = {}; + for (const table of tables) { + const read = readTable(table); + if (!read) { + summary[table] = 'absent (skipped)'; + continue; + } + summary[table] = read.rowCount; + const chunks = chunkStatements(read.statements, maxStatements); + chunks.forEach((chunk, i) => + applyPaced(chunks.length > 1 ? `${table}.${i + 1}` : table, chunk.join('')), + ); + } + + assertShippedCounts(summary, readCounts(summary)); + return summary; +} // Supports --name=value, --name value, and bare --name (boolean). A --name whose next token is another // --flag (or absent) is a boolean; otherwise it consumes the next token as its value. @@ -107,6 +185,17 @@ export function parseMinLinks(raw) { return n; } +/** Shared shape check for the pacing flags: a bare `--flag` must not silently mean 1 (or 0). */ +function parseIntFlag(raw, name, min) { + if (raw === true) throw new Error(`--${name} requires a value, e.g. --${name}=25`); + const n = Number(raw); + if (!Number.isInteger(n) || n < min) + throw new Error(`--${name} must be an integer >= ${min}, got ${JSON.stringify(raw)}.`); + return n; +} +const parsePositiveInt = (raw, name) => parseIntFlag(raw, name, 1); +const parseNonNegativeInt = (raw, name) => parseIntFlag(raw, name, 0); + /** * The D1 name to ship to. A --remote write MUST name its target explicitly: this path DELETEs every * свързани-лица table before re-inserting, so a silent fallback on a remote run is unacceptable — an unset @@ -187,6 +276,56 @@ export function assertD1TargetAuthorized({ remote, shipEnv, d1Name, expectedId, /** Live lookup of the uuid Cloudflare maps `d1Name` to, via `wrangler d1 info --json`. Returns '' on any * failure (unknown name, network, parse) so assertD1TargetConsistent turns that into an explicit refusal. */ +/** + * Read the shipped tables' row counts back off the target, in one query. Returns {} when the read itself + * fails — `assertShippedCounts` then reports every table as unanswered and fails the run, which is the + * right way round: a verification step that cannot verify must not pass. + */ +/** First bracket that actually parses — a notice on the same stream could contain one of its own. + * Belt-and-braces: today wrangler keeps notices on stderr (see the call site). Exported for the test. */ +export function parseWranglerJson(out) { + for (let i = out.indexOf('['); i >= 0; i = out.indexOf('[', i + 1)) { + try { + return JSON.parse(out.slice(i)); + } catch { + // not the payload — keep looking + } + } + return JSON.parse(out); // no usable bracket: let the original parse error surface +} + +function readShippedCounts(d1Name, remote, expected) { + const tables = Object.entries(expected) + .filter(([, n]) => typeof n === 'number') + .map(([t]) => t); + if (!tables.length) return {}; + const sql = tables + .map((t) => `SELECT ${sqlLiteral(t)} AS t, COUNT(*) AS n FROM ${sqlIdent(t)}`) + .join(' UNION ALL '); + try { + const out = execFileSync( + 'wrangler', + ['d1', 'execute', d1Name, remote ? '--remote' : '--local', '--json', '--command', sql], + { cwd: resolve('apps/web'), encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }, + ); + // Defensive, NOT a fix for an observed failure — the earlier wording here claimed otherwise and + // was wrong. Checked against the real tool: `wrangler d1 execute --json` writes its notices + // („▲ [WARNING] Processing wrangler.jsonc") to STDERR and leaves stdout as clean JSON, and + // execFileSync returns stdout alone, so slicing from the first '[' is in fact safe today. The + // scan below survives a future release that changes that, and costs one failed parse if it does. + const parsed = parseWranglerJson(out); + const rows = (Array.isArray(parsed) ? parsed[0]?.results : parsed?.results) ?? []; + // Only a real number counts as an answer. `Number(null)` is 0, which would let a null-valued cell pass + // for „the table is empty"; anything non-numeric must land as NaN so assertShippedCounts fails closed. + return Object.fromEntries(rows.map((r) => [r.t, typeof r.n === 'number' ? r.n : Number.NaN])); + } catch (err) { + console.error( + `ship: could not read back row counts — ${err instanceof Error ? err.message : err}`, + ); + return {}; + } +} + function resolveD1Id(d1Name) { try { const out = execFileSync('wrangler', ['d1', 'info', d1Name, '--json'], { @@ -200,6 +339,34 @@ function resolveD1Id(d1Name) { } } +/** + * Compare what we meant to ship against what the target actually holds. Pure — unit-tested; the live read + * is injected as `readCounts`. + * + * The ship is a wipe followed by SEVERAL independent requests (no cross-request transaction), so a failure + * part-way leaves the target holding some tables and not others — and today nothing notices: the run exits 0 + * and the surface renders as a smaller corpus. That asymmetry is glaring next to the READ path, where the + * EOP hydrate already refuses to proceed on a local↔remote row-count mismatch. Close it on the destructive + * side too: any drift fails the run, loudly and with the numbers. + */ +export function assertShippedCounts(expected, actual) { + const drift = Object.entries(expected) + .filter(([, n]) => typeof n === 'number') + .map(([table, n]) => ({ table, expected: n, actual: actual[table] })) + .filter(({ expected: e, actual: a }) => a !== e); + if (drift.length) + throw new Error( + 'ship verification FAILED — the target does not hold what was shipped:\n' + + drift + .map( + ({ table, expected: e, actual: a }) => + ` ${table}: shipped ${e}, target has ${a === undefined ? 'no answer' : a}`, + ) + .join('\n') + + '\nThe wipe already ran, so the surface is now partial. Re-run the ship.', + ); +} + /** Batched multi-row INSERTs for one table, bounded by D1's statement size. Pure — unit-tested. */ export function insertStatements(table, cols, rows) { if (!cols.length || !rows.length) return []; @@ -238,6 +405,11 @@ function main() { const remote = Boolean(arg('remote', false)); const d1Name = resolveD1Name({ remote, envName: process.env.SIGMA_D1_NAME }); const minLinks = parseMinLinks(arg('min-links', 50)); + const maxStatements = parsePositiveInt( + arg('max-statements-per-request', MAX_STATEMENTS_PER_REQUEST), + 'max-statements-per-request', + ); + const paceMs = parseNonNegativeInt(arg('pace-ms', PACE_MS), 'pace-ms'); if (remote && !arg('yes', false)) throw new Error('--remote requires --yes (guards against an accidental prod write)'); @@ -298,26 +470,46 @@ function main() { '-- ⚠ DESTRUCTIVE, UNGUARDED: this wipe was emitted with --emit and did NOT pass the live D1\n' + '-- target-authorization check (SIGMA_SHIP_ENV allowlist + name↔SIGMA_D1_ID). If you apply it by hand,\n' + '-- YOU are responsible for confirming the target D1 is the intended one before running it.\n'; - if (emit) writeFileSync(resolve(emit, '0_wipe.sql'), EMIT_WIPE_HEADER + wipeSql()); - else applyFile('0_wipe', wipeSql()); + // One read of a source table: null when the table is absent from the work DB. + const readTable = (table) => { + const cols = sqliteJson(`PRAGMA table_info(${sqlIdent(table)})`).map((r) => r.name); + if (!cols.length) return null; + const rows = sqliteJson(`SELECT * FROM ${sqlIdent(table)}`); + return { rowCount: rows.length, statements: insertStatements(table, cols, rows) }; + }; - const summary = {}; + let summary = {}; try { - for (const table of TABLES) { - const cols = sqliteJson(`PRAGMA table_info(${sqlIdent(table)})`).map((r) => r.name); - if (!cols.length) { - summary[table] = 'absent (skipped)'; - continue; + if (emit) { + // --emit keeps ONE file per table: those are applied by hand, and numbered fragments would only add + // ordering rope to a manual run. Nothing is written to a DB, so there is nothing to pace or verify — + // the header on 0_wipe.sql puts the target check on whoever applies them. + writeFileSync(resolve(emit, '0_wipe.sql'), EMIT_WIPE_HEADER + wipeSql()); + for (const table of TABLES) { + const read = readTable(table); + if (!read) { + summary[table] = 'absent (skipped)'; + continue; + } + summary[table] = read.rowCount; + writeFileSync(resolve(emit, `${table}.sql`), read.statements.join('')); } - const rows = sqliteJson(`SELECT * FROM ${sqlIdent(table)}`); - const inserts = insertStatements(table, cols, rows).join(''); - summary[table] = rows.length; - if (emit) writeFileSync(resolve(emit, `${table}.sql`), inserts); - else if (inserts) applyFile(table, inserts); // wipe already cleared it; skip an empty INSERT batch + } else { + summary = runShip({ + tables: TABLES, + readTable, + wipeSql: wipeSql(), + apply: applyFile, + sleep: sleepSync, + readCounts: (expected) => readShippedCounts(d1Name, remote, expected), + maxStatements, + paceMs, + }); } } finally { if (tmp) rmSync(tmp, { recursive: true, force: true }); } + console.log( JSON.stringify( { workDb, target: emit ? `emit:${emit}` : remote ? 'D1:remote' : 'D1:local', rows: summary }, diff --git a/scripts/ship-related-persons.test.mjs b/scripts/ship-related-persons.test.mjs index b15612d1..39e99d17 100644 --- a/scripts/ship-related-persons.test.mjs +++ b/scripts/ship-related-persons.test.mjs @@ -7,9 +7,13 @@ import { parseMinLinks, resolveD1Name, insertStatements, + chunkStatements, + runShip, + assertShippedCounts, sqlLiteral, sqlIdent, TABLES, + WIPE_ORDER, } from './ship-related-persons.mjs'; test('sqlLiteral escapes quotes, strips NUL, and NULLs non-finite/absent', () => { @@ -70,11 +74,31 @@ test('TABLES ships parents before children and covers the served related-persons 'declared_interests', 'interest_links', 'interest_link_authorities', + 'interest_link_evidence', ]) { assert.ok(TABLES.includes(t), `missing ${t}`); } }); +test('the evidence seal ships AFTER the links it references, and is wiped BEFORE them', () => { + // D1 enforces foreign keys, so ordering is not cosmetic: inserting a seal before its link fails, + // and deleting a link while a seal still references it fails the re-seed at interest_links. + assert.ok( + TABLES.indexOf('interest_link_evidence') > TABLES.indexOf('interest_links'), + 'a seal inserted before its link violates the FK', + ); + assert.ok( + WIPE_ORDER.indexOf('interest_link_evidence') < WIPE_ORDER.indexOf('interest_links'), + 'a link deleted while its seal survives violates the FK', + ); +}); + +test('every shipped table is wiped, and every wiped table is real', () => { + // A table added to TABLES but forgotten in WIPE_ORDER accumulates stale rows on every re-ship — + // the surface would then carry evidence for links that no longer exist. + for (const t of TABLES) assert.ok(WIPE_ORDER.includes(t), `${t} ships but is never wiped`); +}); + test('assertD1TargetAuthorized: declared env + allowlisted name + (name↔id) before a remote wipe (T48)', () => { const ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'; const ok = { @@ -181,3 +205,126 @@ test('related_persons_internal (relative-name PII) is NOT shipped to the served // build/work DB only. If a real read path is ever added, ship it deliberately and revisit anonymization. assert.ok(!TABLES.includes('related_persons_internal')); }); + +// The first full-corpus ship put 516k rows through in one hour, p90 batch 18.2s, and the database then +// returned „internal error" for hours. Chunking bounds what a single request carries; these lock the +// boundaries so a refactor cannot quietly restore the one-request-per-table shape. +test('chunkStatements splits into request-sized groups and preserves order', () => { + const stmts = Array.from({ length: 7 }, (_, i) => `S${i};`); + assert.deepEqual(chunkStatements(stmts, 3), [ + ['S0;', 'S1;', 'S2;'], + ['S3;', 'S4;', 'S5;'], + ['S6;'], + ]); + assert.deepEqual(chunkStatements(stmts, 100), [stmts], 'fits in one request'); + assert.deepEqual(chunkStatements([], 3), [], 'nothing to ship, nothing to send'); +}); + +test('chunkStatements refuses a non-positive size rather than looping forever', () => { + assert.throws(() => chunkStatements(['a'], 0), /positive integer/); + assert.throws(() => chunkStatements(['a'], -1), /positive integer/); + assert.throws(() => chunkStatements(['a'], 1.5), /positive integer/); +}); + +// The ship wipes, then writes over SEVERAL requests with no cross-request transaction. Before this, a +// failure part-way exited 0 and the surface silently served a partial corpus — while the READ path +// (EOP hydrate) already refused on a row-count mismatch. Same standard on the destructive side. +test('assertShippedCounts passes when the target holds exactly what was shipped', () => { + assert.doesNotThrow(() => + assertShippedCounts({ persons: 3, interest_links: 2 }, { persons: 3, interest_links: 2 }), + ); +}); + +test('assertShippedCounts fails on a short table and names the numbers', () => { + assert.throws( + () => assertShippedCounts({ persons: 3, interest_links: 2 }, { persons: 3, interest_links: 1 }), + (e) => + /interest_links: shipped 2, target has 1/.test(e.message) && + /verification FAILED/.test(e.message), + ); +}); + +test('assertShippedCounts fails closed when the read-back returned nothing', () => { + assert.throws( + () => assertShippedCounts({ persons: 3 }, {}), + /persons: shipped 3, target has no answer/, + ); +}); + +test('assertShippedCounts ignores tables the ship skipped as absent', () => { + assert.doesNotThrow(() => + assertShippedCounts({ persons: 1, gone: 'absent (skipped)' }, { persons: 1 }), + ); +}); + +// These drive the REAL ship path, not the helpers in isolation. That distinction is the whole point: +// with the previous shape — helpers unit-tested, main() calling them — reverting the call site to one +// request per table, and deleting the verification line outright, BOTH left the suite fully green. +const shipHarness = (over = {}) => { + const calls = []; + const naps = []; + const source = over.source ?? { + persons: { rowCount: 5, statements: ['A;', 'B;', 'C;', 'D;', 'E;'] }, + declarations: { rowCount: 1, statements: ['F;'] }, + }; + const opts = { + tables: over.tables ?? ['persons', 'declarations'], + readTable: (t) => source[t] ?? null, + wipeSql: 'DELETE FROM persons;', + apply: (name, sql) => calls.push([name, sql]), + sleep: (ms) => naps.push(ms), + readCounts: over.readCounts ?? ((expected) => ({ ...expected })), + maxStatements: over.maxStatements ?? 2, + paceMs: 500, + }; + return { calls, naps, run: () => runShip(opts) }; +}; + +test('runShip wipes, then ships every table in request-sized chunks, in order', () => { + const h = shipHarness(); + const summary = h.run(); + + assert.deepEqual( + h.calls.map(([name]) => name), + ['0_wipe', 'persons.1', 'persons.2', 'persons.3', 'declarations'], + 'wipe first, chunks numbered so a failed request is identifiable, single-chunk table stays bare', + ); + assert.deepEqual( + h.calls.map(([, sql]) => sql), + ['DELETE FROM persons;', 'A;B;', 'C;D;', 'E;', 'F;'], + ); + assert.deepEqual(summary, { persons: 5, declarations: 1 }); +}); + +// THE regression this exists to prevent: the counter used to restart per table, so every table +// boundary — including wipe → first insert, the most destructive transition in the run — was unpaced. +test('runShip paces every request boundary, including wipe → first insert', () => { + const h = shipHarness(); + h.run(); + assert.equal(h.calls.length, 5); + assert.deepEqual( + h.naps, + [500, 500, 500, 500], + 'one gap between each pair of requests: none before the first, none after the last, and none skipped at a table boundary', + ); +}); + +test('runShip verifies what landed — a short table fails the run', () => { + const h = shipHarness({ readCounts: (expected) => ({ ...expected, persons: 4 }) }); + assert.throws(() => h.run(), /ship verification FAILED[\s\S]*persons: shipped 5, target has 4/); +}); + +test('runShip fails the run when the read-back itself could not answer', () => { + const h = shipHarness({ readCounts: () => ({}) }); + assert.throws(() => h.run(), /no answer/); +}); + +test('runShip skips a table absent from the work DB without shipping or verifying it', () => { + const h = shipHarness({ tables: ['persons', 'declarations', 'ghost'] }); + const summary = h.run(); + assert.equal(summary.ghost, 'absent (skipped)'); + assert.ok( + !h.calls.some(([name]) => name.startsWith('ghost')), + 'an absent table must issue no request', + ); +}); diff --git a/scripts/ship-reseed.test.mjs b/scripts/ship-reseed.test.mjs index 4864029d..44020c5c 100644 --- a/scripts/ship-reseed.test.mjs +++ b/scripts/ship-reseed.test.mjs @@ -15,6 +15,9 @@ import { wipeSql, TABLES } from './ship-related-persons.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); const MIG = resolve(HERE, '..', 'packages/db/migrations/0003_related_persons_foundation.sql'); +// 0006 adds interest_link_evidence, which references interest_links — so it is part of the FK graph +// this test exists to police, and the wipe must clear it before its parent (#279, ADR-0033). +const MIG_EVIDENCE = resolve(HERE, '..', 'packages/db/migrations/0009_interest_link_evidence.sql'); // Stub only the two 0000 tables 0002's FKs reference (bidders, authorities) — a PK column is all a FK needs — // then apply the real 0002 and seed one row down every FK chain so a parent delete has live children. @@ -25,6 +28,7 @@ function newPopulatedDb() { CREATE TABLE bidders(id TEXT PRIMARY KEY); CREATE TABLE authorities(id TEXT PRIMARY KEY); .read ${MIG} +.read ${MIG_EVIDENCE} INSERT INTO bidders(id) VALUES('eik:1'); INSERT INTO authorities(id) VALUES('auth:1'); INSERT INTO persons(id,name) VALUES('p1','П Тест'); @@ -33,6 +37,7 @@ INSERT INTO declared_interests(id,declaration_id,entity_raw,entity_key,kind) VAL INSERT INTO interest_links(id,link_key,person_id,bidder_id,eik,entity_key,matcher_version,publish_tier,relation,status) VALUES('il1','p1|1','p1','eik:1','1','e','v1','B_distinctive','owns','published'); INSERT INTO interest_link_authorities(link_key,authority_id,authority_name) VALUES('p1|1','auth:1','A'); INSERT INTO related_persons_internal(id,declaration_id,related_name,related_kind) VALUES('rp1','d1','X','related_person'); +INSERT INTO interest_link_evidence(link_key,evidence_kind,matched_fact,lookup_date,rules_version,live_status) VALUES('p1|1','document','role:owner:CR_F_19_L','2026-08-05','tr-rules-1','live'); `; execFileSync('sqlite3', ['-bail', db], { input: setup, stdio: 'pipe' }); return { dir, db }; diff --git a/scripts/tr/cache.mjs b/scripts/tr/cache.mjs new file mode 100644 index 00000000..764c16f2 --- /dev/null +++ b/scripts/tr/cache.mjs @@ -0,0 +1,278 @@ +// The deed cache — resumability, and the PII rail (issue #279, ADR-0033 decision 5). +// +// A registry deed contains third-party personal data: the names of owners and managers who hold no +// public office, and the company's street address. Two rails follow from that, and both live here: +// +// 1. The INDEX stores no name at all — ЕИК, dates, codes, verdicts, and a hash of the body. Names +// exist only in the raw JSON under git-ignored scratch/, are read only to produce a boolean, and +// never enter a public table, a response or a log. A hash rather than an excerpt, because an +// excerpt of a deed IS third-party personal data. +// 2. Nothing may carry a STANDALONE ten-digit run. That is the ЕГН shape, and the check is sound +// precisely because an ЕИК is 9 or 13 digits — never 10 — so it cannot reject a legitimate +// identifier. „Standalone" is load-bearing: a 13-digit ЕИК contains ten-digit substrings, so an +// unanchored match would refuse every клон. ЕГН was absent from every payload examined; this is +// the rail for the day one leaks. +// +// Resumability is the other job: the register throttles hard and a 429 ends the run (client.mjs), so a +// crawl must be able to pick up exactly where it stopped without re-requesting what it already has. + +import { DatabaseSync } from 'node:sqlite'; +import fs from 'node:fs'; +import path from 'node:path'; +import { safeEik } from './paths.mjs'; + +const SCHEMA = ` +CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, + status TEXT NOT NULL, -- fetched | outside_tr + http_status INTEGER, + fetched_at TEXT NOT NULL, + raw_path TEXT, -- relative to the raw dir; the ONLY place names live + body_sha256 TEXT, -- integrity + change detection, never an excerpt + legal_form_code INTEGER, + legal_form_verdict TEXT, -- closely_held | joint_stock | unknown (unknown WITHHOLDS) + seat_normalized TEXT, -- settlement only; never a street address (ADR-0010 item 3) + seat_entry_date TEXT, + latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, + outside_reason TEXT +); +CREATE INDEX IF NOT EXISTS idx_deeds_status ON deeds(status); +CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); +`; + +/** Open (creating if absent) the cache at `file`. Idempotent — never wipes an existing cache. */ +export function openCache(file) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const db = new DatabaseSync(file); + db.exec('PRAGMA journal_mode = WAL'); + db.exec(SCHEMA); + return db; +} + +// ── the ЕГН rail ────────────────────────────────────────────────────────────── +// ANCHORED, and that is the whole correctness of the rail. An ЕИК is 9 or 13 digits — never 10 — so +// a ten-digit run cannot be a legitimate identifier here. But that reasoning only holds when the run +// is matched as a WHOLE: an unanchored /\d{10}/ matches INSIDE the 13-digit ЕИК of a клон, and +// rawPath on the fetched path is `.json`, so every branch office would be refused and the crawl +// would abort on the first one. +const EGN_SHAPE = /(? `${r.eik}.json`), + ); + for (const name of names) { + if (!name.endsWith('.json') || known.has(name)) continue; + // The same ENOENT tolerance the expired loop above has, and for a sharper reason here: this loop + // runs AFTER the DB DELETE has committed, so an unguarded throw half-purges — rows gone, files + // still on disk — and reports the whole run as failed. The listing is a snapshot, so a name can + // legitimately be gone by the time we reach it (a concurrent purge, an operator clearing scratch). + // Already gone is the goal state. Anything else still surfaces: a purge that cannot delete has + // left third-party names on disk, and reporting success would be the failure it exists to prevent. + try { + unlink(path.join(rawDir, name)); + orphans++; + } catch (e) { + if (e.code !== 'ENOENT') throw e; + } + } + return { rows: expired.length, files, orphans }; +} diff --git a/scripts/tr/cache.test.mjs b/scripts/tr/cache.test.mjs new file mode 100644 index 00000000..a7357ec8 --- /dev/null +++ b/scripts/tr/cache.test.mjs @@ -0,0 +1,293 @@ +// node:test — the deed cache. Its job is to make the crawl resumable and to hold the PII rail. +// +// The rail (ADR-0033 decision 5): the INDEX stores no name at all — only ЕИК, dates, codes, verdicts +// and a body hash. Names exist solely in the raw JSON under git-ignored scratch/, are read only to +// produce a boolean, and never enter a public table, a response or a log. The ten-digit refusal below +// is the ЕГН guard, and it is sound precisely because an ЕИК is 9 or 13 digits — never 10. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + openCache, + upsertDeed, + markOutsideTr, + pendingEiks, + readDeed, + coverage, + purgeExpired, + RETENTION_DAYS, +} from './cache.mjs'; + +function tmpDb() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-cache-')); + return { dir, file: path.join(dir, 'tr-cache.sqlite') }; +} +const withCache = (fn) => { + const { dir, file } = tmpDb(); + const db = openCache(file); + try { + return fn(db); + } finally { + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } +}; + +const deed = (over = {}) => ({ + eik: '115536179', + httpStatus: 200, + fetchedAt: '2026-08-05T10:00:00Z', + rawPath: 'deeds/115536179.json', + bodySha256: 'a'.repeat(64), + legalFormCode: 4, + legalFormVerdict: 'closely_held', + seatNormalized: 'ПЛОВДИВ', + seatEntryDate: '2014-01-23', + latestOwnEntryDate: '2013-07-16', + ...over, +}); + +test('openCache is idempotent — re-opening an existing cache preserves rows', () => { + const { dir, file } = tmpDb(); + let db = openCache(file); + upsertDeed(db, deed()); + db.close(); + db = openCache(file); // must not wipe + assert.equal(readDeed(db, '115536179')?.eik, '115536179'); + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); +}); + +test('upsertDeed replaces on re-fetch rather than duplicating', () => + withCache((db) => { + upsertDeed(db, deed()); + upsertDeed(db, deed({ seatNormalized: 'СОФИЯ', fetchedAt: '2026-09-01T10:00:00Z' })); + assert.equal(coverage(db, ['115536179']).fetched, 1); + assert.equal(readDeed(db, '115536179').seatNormalized, 'СОФИЯ'); + })); + +test('pendingEiks returns only what is not yet cached — this is what makes a run resumable', () => + withCache((db) => { + upsertDeed(db, deed({ eik: '115536179' })); + markOutsideTr(db, '204556676', 'BULSTAT association'); + const want = ['115536179', '204556676', '201122335', '203445566']; + assert.deepEqual(pendingEiks(db, want).sort(), ['201122335', '203445566']); + })); + +test('a stale deed becomes pending again past the TTL, a fresh one does not', () => + withCache((db) => { + upsertDeed(db, deed({ fetchedAt: '2026-01-01T00:00:00Z' })); // long past + upsertDeed(db, deed({ eik: '201122335', fetchedAt: '2026-08-05T00:00:00Z' })); + const now = new Date('2026-08-05T12:00:00Z'); + assert.deepEqual(pendingEiks(db, ['115536179', '201122335'], { maxAgeDays: 35, now }), [ + '115536179', + ]); + })); + +test('coverage reports the fraction cached — the input to the fail-closed load gate', () => + withCache((db) => { + upsertDeed(db, deed({ eik: '115536179' })); + upsertDeed(db, deed({ eik: '201122335' })); + markOutsideTr(db, '204556676', 'ДЗЗД'); + const c = coverage(db, ['115536179', '201122335', '204556676', '203445566']); + assert.equal(c.wanted, 4); + assert.equal(c.fetched, 2); + assert.equal(c.outsideTr, 1); + assert.equal(c.missing, 1); + // „outside ТР" is a RESOLVED outcome, not a gap: it is known and permanent, so it counts as covered. + assert.equal(c.covered, 3); + })); + +// ── the PII rail ────────────────────────────────────────────────────────────── +test('the index REFUSES a value carrying a ten-digit run (the ЕГН guard)', () => + withCache((db) => { + // Sound because an ЕИК is 9 or 13 digits, never 10 — so this can never reject a legitimate code. + assert.throws( + () => upsertDeed(db, deed({ seatNormalized: 'СОФИЯ 8001014567' })), + /ten-digit|ЕГН/i, + ); + assert.throws(() => markOutsideTr(db, '204556676', 'подадено от 8001014567'), /ten-digit|ЕГН/i); + })); + +test('valid 9- and 13-digit codes are NOT caught by the ЕГН guard', () => + withCache((db) => { + assert.doesNotThrow(() => upsertDeed(db, deed({ outsideReason: null }))); + assert.doesNotThrow(() => markOutsideTr(db, '1155361790001', 'клон')); + })); + +// A 13-digit ЕИК (клон/подразделение) CONTAINS ten-digit substrings, so an unanchored /\d{10}/ +// rejects it — and rawPath on the fetched path is `.json`, derived from that very ЕИК. This is +// the exact shape fetch-deeds.mjs writes (path.relative(rawDir, deedPath(eik))), and upsertDeed sits +// past its JSON.parse/assertUicEcho try-catch, so a throw here aborts the whole crawl on the first +// branch office that returns a deed. The guard's own stated soundness ("an ЕИК is 9 or 13 digits, +// never 10") only holds if the run is matched as a WHOLE, which is why the pattern is anchored. +test('a 13-digit ЕИК does not trip the ЕГН guard through its own derived rawPath', () => + withCache((db) => { + assert.doesNotThrow(() => + upsertDeed(db, deed({ eik: '1155361790001', rawPath: '1155361790001.json' })), + ); + assert.equal(readDeed(db, '1155361790001').eik, '1155361790001'); + })); + +// The rail must not be a hand-maintained allowlist of four field names: upsertDeed binds thirteen +// values, and the next one added would bypass the check silently. Every bound value is screened. +test('the ЕГН guard screens EVERY bound value, not a hand-picked subset', () => + withCache((db) => { + for (const field of ['seatEntryDate', 'latestOwnEntryDate', 'fetchedAt']) + assert.throws(() => upsertDeed(db, deed({ [field]: '8001014567' })), /ten-digit|ЕГН/i, field); + })); + +// ...with one exemption, and it is measured rather than assumed: a sha256 hex digest is 64 chars of +// [0-9a-f], so a standalone ten-digit run occurs in ~7% of hashes (18% unanchored). Screening it +// would refuse roughly one deed in fourteen for no privacy gain — a hash is not an ЕГН, and it is a +// hash precisely so that no deed content reaches the index. +test('the body hash is exempt from the ЕГН guard — a digit run there is arithmetic, not an ЕГН', () => + withCache((db) => { + assert.doesNotThrow(() => upsertDeed(db, deed({ bodySha256: `8001014567${'a'.repeat(54)}` }))); + })); + +test('the schema exposes no column that could hold a person name', () => + withCache((db) => { + const cols = db + .prepare(`SELECT name FROM pragma_table_info('deeds')`) + .all() + .map((r) => r.name); + for (const forbidden of ['name', 'person', 'owner', 'manager', 'holder', 'full_name']) + assert.ok(!cols.includes(forbidden), `deeds.${forbidden} must not exist (PII rail)`); + // A hash, never an excerpt — an excerpt of a deed is third-party personal data. + assert.ok(cols.includes('body_sha256')); + })); + +test('markOutsideTr is permanent-by-intent and records WHY', () => + withCache((db) => { + markOutsideTr(db, '204556676', 'ДЗЗД — BULSTAT, not TR'); + const row = readDeed(db, '204556676'); + assert.equal(row.status, 'outside_tr'); + assert.match(row.outsideReason, /ДЗЗД/); + })); + +// ── retention (ADR-0033 decision 5) ─────────────────────────────────────────── +// The ADR promises „a 35-day retention and a purge step in the same job". Freshness (pendingEiks' +// maxAgeDays) only makes a row pending again — it re-REQUESTS, it never deletes. Only this purge +// removes the third-party names in the raw deed, so these tests are the difference between a stated +// TTL and an enforced one. +function withRaw(fn) { + const { dir, file } = tmpDb(); + const rawDir = path.join(dir, 'deeds'); + fs.mkdirSync(rawDir, { recursive: true }); + const db = openCache(file); + try { + return fn(db, rawDir); + } finally { + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } +} +const writeRaw = (rawDir, eik) => + fs.writeFileSync(path.join(rawDir, `${eik}.json`), '{"names":"third-party PII"}'); + +test('purgeExpired deletes the raw deed AND its row past the retention window', () => + withRaw((db, rawDir) => { + const now = new Date('2026-08-05T00:00:00Z'); + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-05-01T00:00:00Z' })); // ~96 days old + upsertDeed(db, deed({ eik: '201122335', fetchedAt: '2026-08-01T00:00:00Z' })); // 4 days old + writeRaw(rawDir, '115536179'); + writeRaw(rawDir, '201122335'); + + const res = purgeExpired(db, rawDir, { retentionDays: 35, now }); + assert.equal(res.rows, 1); + assert.equal(res.files, 1); + assert.equal(fs.existsSync(path.join(rawDir, '115536179.json')), false, 'PII must be gone'); + assert.equal(readDeed(db, '115536179'), null); + // The in-window deed is untouched — a purge that also evicted live cache would force a re-crawl, + // which is the one thing the pacing exists to avoid. + assert.equal(fs.existsSync(path.join(rawDir, '201122335.json')), true); + assert.ok(readDeed(db, '201122335')); + })); + +test('purgeExpired removes orphaned raw deeds — unreachable data is pure retained PII', () => + withRaw((db, rawDir) => { + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-08-01T00:00:00Z' })); + writeRaw(rawDir, '115536179'); + writeRaw(rawDir, '204556676'); // no index row: no read path can ever reach it + const res = purgeExpired(db, rawDir, { now: new Date('2026-08-05T00:00:00Z') }); + assert.equal(res.orphans, 1); + assert.equal(fs.existsSync(path.join(rawDir, '204556676.json')), false); + assert.equal(fs.existsSync(path.join(rawDir, '115536179.json')), true); + })); + +test('purgeExpired defaults to the 35-day window and tolerates an already-missing file', () => + withRaw((db, rawDir) => { + assert.equal(RETENTION_DAYS, 35); + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-06-25T00:00:00Z' })); // 41 days + upsertDeed(db, deed({ eik: '201122335', fetchedAt: '2026-07-15T00:00:00Z' })); // 21 days + // No raw file on disk for the expired row: already gone is the goal state, not an error. + const res = purgeExpired(db, rawDir, { now: new Date('2026-08-05T00:00:00Z') }); + assert.equal(res.rows, 1); + assert.equal(res.files, 0); + assert.ok(readDeed(db, '201122335'), 'the 21-day-old deed is inside the window'); + })); + +test('purgeExpired tolerates an orphan that vanished under it, and keeps sweeping', () => { + // The benign race: readdirSync lists a name, and it is gone by the time unlink runs (a concurrent + // purge, an operator clearing scratch/). The expired loop right above has always tolerated this; + // the orphan loop did not, and it throws AFTER the DB DELETE has committed — so the run half-purges, + // reports "purge failed", and leaves the operator unable to tell "already gone" from "an orphan + // still holding third-party names". Injected rather than staged, because the race cannot be timed + // from a test; the injection seam matches the one httpGet/sleep/now already use in this codebase. + return withRaw((db, rawDir) => { + upsertDeed(db, deed({ eik: '115536179', fetchedAt: '2026-08-01T00:00:00Z' })); + writeRaw(rawDir, '115536179'); + writeRaw(rawDir, '204556676'); // orphan 1 — disappears under us + writeRaw(rawDir, '831391124'); // orphan 2 — must still be swept + + const seen = []; + const res = purgeExpired(db, rawDir, { + now: new Date('2026-08-05T00:00:00Z'), + unlink: (p) => { + seen.push(path.basename(p)); + if (p.endsWith('204556676.json')) { + const err = new Error('ENOENT: no such file or directory'); + err.code = 'ENOENT'; + throw err; + } + fs.unlinkSync(p); + }, + }); + + assert.deepEqual(seen.sort(), ['204556676.json', '831391124.json'], 'both orphans attempted'); + assert.equal( + res.orphans, + 1, + 'a file that was already gone was not deleted BY US — do not count it', + ); + assert.equal(fs.existsSync(path.join(rawDir, '831391124.json')), false, 'the sweep continued'); + assert.equal( + fs.existsSync(path.join(rawDir, '115536179.json')), + true, + 'the live deed is untouched', + ); + }); +}); + +test('purgeExpired still refuses loudly on an orphan it could not delete for a REAL reason', () => { + // The other half, and the reason the guard is ENOENT-only. A permission error means retained PII is + // still on disk; reporting success would be the failure mode the purge exists to prevent. + return withRaw((db, rawDir) => { + writeRaw(rawDir, '204556676'); + assert.throws( + () => + purgeExpired(db, rawDir, { + now: new Date('2026-08-05T00:00:00Z'), + unlink: () => { + const err = new Error('EACCES: permission denied'); + err.code = 'EACCES'; + throw err; + }, + }), + /EACCES/, + ); + }); +}); diff --git a/scripts/tr/client.mjs b/scripts/tr/client.mjs new file mode 100644 index 00000000..473f5e30 --- /dev/null +++ b/scripts/tr/client.mjs @@ -0,0 +1,171 @@ +// HTTP client for the Търговски регистър public API (issue #279, ADR-0033). +// +// TLS: PLAIN HTTPS, verified against the system roots at Node's secure default. Do NOT reach for +// scripts/cacbg/tls.mjs here. That module exists because register.cacbg.bg serves an incomplete chain, +// so we pin ITS leaf key (ADR-0011) — `getPinned` hard-refuses every other host by design. This host +// chains correctly, so ordinary verification is available and is strictly stronger than a pin we would +// have to hand-maintain. Copying the pinning across "for consistency" would weaken this leg. +// +// Rate limiting: the register throttles, and when it does the block is SUSTAINED. An earlier spike saw +// HTTP 429 at roughly 50 cumulative requests ending in a burst, and thereafter 429 to every subsequent +// request — including simple /Deeds/{eik} calls that had worked seconds before. There is no +// `Retry-After` and no `X-RateLimit-*` header, so a client cannot pace against a published budget; it +// can only avoid tripping one. That makes a 429 an instruction to STOP, not a transient to retry +// through: `politeTrGet` retries 5xx and network faults with growing backoff, and never retries a 429. +// (#279 §3's "5 retries with growing backoff" and the decision that a 429 ends the run are consistent +// only if the retry set excludes 429.) The limiter is the operator's only way to express a rate +// preference, and tuning around it empirically is what spec §3.3's "NEVER bulk-scrape" forbids. + +import https from 'node:https'; +import { safeEik } from './paths.mjs'; + +export const TR_HOST = 'portal.registryagency.bg'; + +// Identify the crawler honestly. The operator's only lever on us is the rate limiter, so at minimum +// they should be able to see who is calling and where to complain. +export const TR_USER_AGENT = + 'sigma-bot/1.0 (+https://github.com/midt-bg/sigma) contact: via repo issues'; + +/** Thrown on HTTP 429. Distinguishable so the crawler can end the run instead of marking anything. */ +export class RateLimitError extends Error { + constructor(url) { + super(`REFUSE TO CONTINUE: ${TR_HOST} returned 429 for ${url} — the block is sustained; stop`); + this.name = 'RateLimitError'; + this.url = url; + } +} + +/** Refuse any URL that is not an https request to the register. Call before touching the network. */ +export function assertTrHost(url) { + let u; + try { + u = new URL(String(url)); + } catch { + throw new Error(`bad URL: ${url}`); + } + if (u.protocol !== 'https:' || u.hostname !== TR_HOST) { + throw new Error(`refusing non-${TR_HOST} host: ${url}`); + } + return url; +} + +/** The public deed endpoint for one ЕИК. Public JSON, no authentication. */ +export function deedUrl(eik) { + return `https://${TR_HOST}/CR/api/Deeds/${safeEik(eik)}`; +} + +const wait = (ms) => new Promise((r) => setTimeout(r, ms)); + +/** + * Default transport: `node:https`, system roots, secure defaults. + * + * NOT `fetch`. Measured against the live endpoint on 2026-08-05, the identical request returns + * **HTTP 500 with an empty body via undici's fetch and HTTP 200 with the full 34,398-byte deed via + * node:https** — same URL, same accept header, same user-agent. The server rejects something undici + * adds to the wire (encoding negotiation / connection handling); it is not an auth or rate-limit + * problem, and retrying only multiplies the failure. `scripts/cacbg/tls.mjs` is on node:https too, so + * both crawl legs now share one transport primitive. + * + * `rejectUnauthorized` is left at its secure default and named here on purpose: the sibling CACBG + * module deliberately does NOT verify against system roots (it pins a leaf instead, ADR-0011), so a + * reader comparing the two needs to see that this leg takes the ordinary, stronger path. + */ +/** + * Ceiling on a single response body. A measured deed is ~34 KB, so 8 MB is ~240× the real thing — + * this bounds abuse, not the register, and can never refuse a large-but-legitimate company. + */ +export const MAX_BODY_BYTES = 8 * 1024 * 1024; + +/** + * Read one response into a Buffer, refusing past `maxBytes`. + * + * The timeout on the request bounds how long a response may STALL; nothing bounded how much it may + * SEND. Without a counter the crawler buffers whatever arrives, so a wedged or hostile endpoint decides + * how much memory this process holds. It also composes badly with the parser: the erasure regex in + * deed.mjs backtracks quadratically on unclosed markup (measured 34K→3.7ms, 68K→13.5ms, 136K→53.2ms, + * 272K→215.4ms — ×4 per doubling), and that path is reachable only through a body large enough to make + * it matter. One byte counter bounds both, and destroying the request is the load-bearing half: merely + * rejecting the promise would leave the socket draining the rest of the response. + * + * Separated from `httpsGet` so it is testable without TLS or a live socket — `res` needs only to be an + * emitter of 'data'/'end'/'error', which is the same injection posture as politeTrGet's `httpGet`. + */ +export function collectBody(res, { req, maxBytes = MAX_BODY_BYTES } = {}) { + return new Promise((resolve, reject) => { + const chunks = []; + let size = 0; + let done = false; + const fail = (err) => { + if (done) return; + done = true; + req?.destroy(err); // tear the socket down; do not sit and drain what we already refused + reject(err); + }; + res.on('data', (c) => { + if (done) return; + size += c.length; + if (size > maxBytes) { + fail(new Error(`response too large: over ${maxBytes} bytes — refusing to buffer it`)); + return; + } + chunks.push(c); + }); + res.on('end', () => { + if (done) return; + done = true; + resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) }); + }); + res.on('error', fail); + }); +} + +export function httpsGet(url, { timeoutMs = 20_000, maxBytes = MAX_BODY_BYTES } = {}) { + return new Promise((resolve, reject) => { + const req = https.get( + url, + { headers: { accept: 'application/json', 'user-agent': TR_USER_AGENT } }, + (res) => collectBody(res, { req, maxBytes }).then(resolve, reject), + ); + req.setTimeout(timeoutMs, () => req.destroy(new Error(`timeout after ${timeoutMs}ms: ${url}`))); + req.on('error', reject); + }); +} + +/** + * GET with polite retries. 5xx and network faults retry with growing backoff; 2xx/4xx return as-is; + * **429 throws `RateLimitError` immediately and is never retried**. + * @param {string} url + * @param {{httpGet?:Function, sleep?:Function, tries?:number, backoffMs?:number}} [opts] + * `httpGet` and `sleep` are the injection seam — the whole retry policy is testable offline. + */ +export async function politeTrGet(url, opts = {}) { + const { httpGet = httpsGet, sleep = wait, tries = 5, backoffMs = 1000 } = opts; + assertTrHost(url); // before any request — a foreign host must cost zero packets + let backoff = backoffMs; + for (let attempt = 1; ; attempt++) { + let res; + try { + res = await httpGet(url); + } catch (err) { + if (attempt >= tries) throw err; + await sleep(backoff); + backoff *= 2; + continue; + } + // Checked before the retry branch: a 429 arriving mid-retry must abort the whole call, not be + // folded into the 5xx budget and not let a later 200 mask the block. + if (res.status === 429) throw new RateLimitError(url); + if (res.status >= 500) { + if (attempt >= tries) return res; // give the caller the last response to record + await sleep(backoff); + backoff *= 2; + continue; + } + return res; + } +} + +/** One deed by ЕИК. Thin wrapper so callers never build the URL themselves. */ +export function trGet(eik, opts) { + return politeTrGet(deedUrl(eik), opts); +} diff --git a/scripts/tr/client.test.mjs b/scripts/tr/client.test.mjs new file mode 100644 index 00000000..20142602 --- /dev/null +++ b/scripts/tr/client.test.mjs @@ -0,0 +1,197 @@ +// node:test — the Trade Register HTTP client. Offline: every test drives an injected getter. +// +// The load-bearing behaviour here is what the client REFUSES to do. The register rate-limits, and +// when it does the block is sustained (an earlier spike saw HTTP 429 at ~50 cumulative requests, then +// 429 to every subsequent call including ones that had just worked — no Retry-After, no quota header). +// So a 429 is an instruction to stop, not a transient to retry through. ADR-0033 decision 7. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { + TR_HOST, + TR_USER_AGENT, + deedUrl, + RateLimitError, + politeTrGet, + assertTrHost, + httpsGet, + collectBody, + MAX_BODY_BYTES, +} from './client.mjs'; + +const okRes = (body = '{}') => ({ status: 200, headers: {}, body: Buffer.from(body) }); + +test('deedUrl builds the documented endpoint and refuses a non-ЕИК', () => { + assert.equal(deedUrl('115536179'), `https://${TR_HOST}/CR/api/Deeds/115536179`); + assert.equal(deedUrl('000696327'), `https://${TR_HOST}/CR/api/Deeds/000696327`); + for (const bad of ['', null, '../../etc', '115536179?x=1', '11553617x', 'ЕИК 115536179']) + assert.throws(() => deedUrl(bad), /unsafe|ЕИК/i, String(bad)); +}); + +test('assertTrHost refuses every host but the register', () => { + assert.doesNotThrow(() => assertTrHost(`https://${TR_HOST}/CR/api/Deeds/115536179`)); + for (const bad of [ + 'https://evil.example/CR/api/Deeds/115536179', + `http://${TR_HOST}/CR/api/Deeds/115536179`, // plaintext — never + `https://${TR_HOST}.evil.example/x`, // suffix trick + `https://register.cacbg.bg/x`, // the OTHER register: different host, different TLS posture + ]) { + assert.throws(() => assertTrHost(bad), /host/i, bad); + } +}); + +test('a 429 throws RateLimitError and is NEVER retried', async () => { + let calls = 0; + const httpGet = async () => { + calls++; + return { status: 429, headers: {}, body: Buffer.from('') }; + }; + await assert.rejects( + () => politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {}, tries: 5 }), + RateLimitError, + ); + // The whole point: „5 retries with growing backoff" and „429 stops the run" are only consistent + // if the retry set EXCLUDES 429. One call, not five. + assert.equal(calls, 1, 'a 429 must not be retried'); +}); + +test('5xx IS retried with growing backoff, up to the try budget', async () => { + let calls = 0; + const waits = []; + const httpGet = async () => { + calls++; + return { status: 503, headers: {}, body: Buffer.from('') }; + }; + const res = await politeTrGet(deedUrl('115536179'), { + httpGet, + sleep: async (ms) => void waits.push(ms), + tries: 4, + }); + assert.equal(calls, 4); + assert.equal(res.status, 503, 'the last response is returned, not thrown'); + assert.deepEqual( + waits.map((w, i) => (i === 0 ? true : w > waits[i - 1])), + [true, true, true], + `backoff must grow: ${waits.join(',')}`, + ); +}); + +test('a network throw is retried, then rethrown when the budget is spent', async () => { + let calls = 0; + const httpGet = async () => { + calls++; + throw new Error('ECONNRESET'); + }; + await assert.rejects( + () => politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {}, tries: 3 }), + /ECONNRESET/, + ); + assert.equal(calls, 3); +}); + +test('a 429 arriving mid-retry aborts immediately instead of finishing the budget', async () => { + const seq = [503, 503, 429, 200]; + let calls = 0; + const httpGet = async () => { + const status = seq[calls++]; + return { status, headers: {}, body: Buffer.from('') }; + }; + await assert.rejects( + () => politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {}, tries: 5 }), + RateLimitError, + ); + assert.equal(calls, 3, 'must stop AT the 429, not carry on to the 200 behind it'); +}); + +test('200 and 404 return without any retry', async () => { + for (const status of [200, 404]) { + let calls = 0; + const httpGet = async () => { + calls++; + return status === 200 ? okRes() : { status, headers: {}, body: Buffer.from('') }; + }; + const res = await politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {} }); + assert.equal(res.status, status); + assert.equal(calls, 1); + } +}); + +test('politeTrGet refuses a foreign host before making any request', async () => { + let calls = 0; + const httpGet = async () => { + calls++; + return okRes(); + }; + await assert.rejects( + () => politeTrGet('https://evil.example/CR/api/Deeds/1', { httpGet }), + /host/i, + ); + assert.equal(calls, 0, 'the refusal must happen before the network call'); +}); + +test('the crawler identifies itself and points at somewhere to complain', () => { + // The rate limiter is the operator's only lever on us. Crawling anonymously takes away their + // ability to distinguish us from an abusive client, or to ask us to stop. + assert.match(TR_USER_AGENT, /sigma/i); + assert.match(TR_USER_AGENT, /https:\/\/github\.com\//); +}); + +test('the default transport is node:https, not fetch', () => { + // Measured 2026-08-05: the identical live request returns 500/empty via undici fetch and 200 with + // the full deed via node:https. Pinning the choice so a future "modernise to fetch" tidy-up has to + // confront it rather than silently break every lookup. + assert.equal(typeof httpsGet, 'function'); + assert.match(httpsGet.toString(), /https\.get/); +}); + +test('RateLimitError carries the url and is distinguishable from a generic failure', async () => { + const httpGet = async () => ({ status: 429, headers: {}, body: Buffer.from('') }); + const err = await politeTrGet(deedUrl('115536179'), { httpGet, sleep: async () => {} }).catch( + (e) => e, + ); + assert.ok(err instanceof RateLimitError); + assert.ok(err instanceof Error); + assert.match(err.message, /115536179/); +}); + +// ── response size ───────────────────────────────────────────────────────────── +// The timeout bounds how long a response may STALL; nothing bounded how large it may GROW. A deed is +// ~34 KB measured; an unbounded reader buffers whatever arrives, so a wedged or hostile endpoint could +// have the crawler hold an arbitrary amount of memory. It also composes badly with the parser: the +// erasure regex in deed.mjs backtracks quadratically on unclosed markup (measured 34K→3.7ms, +// 68K→13.5ms, 136K→53.2ms, 272K→215.4ms — ×4 per doubling), and that path is only reachable through a +// body large enough to make it matter. One byte counter bounds both. +test('a response past the cap is refused and the request destroyed, not buffered', async () => { + const res = fakeRes(200); + const destroyed = []; + const p = collectBody(res, { req: { destroy: (e) => destroyed.push(e) }, maxBytes: 1024 }); + res.emit('data', Buffer.alloc(700)); + res.emit('data', Buffer.alloc(700)); // 1400 > 1024 + await assert.rejects(p, /too large|1024/i); + assert.equal(destroyed.length, 1, 'the socket must be torn down, not left draining'); +}); + +test('a response under the cap still resolves with the WHOLE body', async () => { + const res = fakeRes(200); + const p = collectBody(res, { req: { destroy: () => {} }, maxBytes: 1024 }); + res.emit('data', Buffer.from('{"uic":')); + res.emit('data', Buffer.from('"115536179"}')); + res.emit('end'); + const out = await p; + assert.equal(out.status, 200); + assert.equal(out.body.toString(), '{"uic":"115536179"}'); +}); + +test('the cap leaves real deeds far under it — it bounds abuse, not the register', () => { + // A measured deed is ~34 KB. The cap must sit well above that or it becomes a correctness bug that + // silently refuses large-but-legitimate companies. + assert.ok(MAX_BODY_BYTES >= 1_000_000, `cap ${MAX_BODY_BYTES} is too tight for a real deed`); +}); + +// A stand-in for an http.IncomingMessage: an emitter carrying a status code. +function fakeRes(status) { + const e = new EventEmitter(); + e.statusCode = status; + e.headers = {}; + return e; +} diff --git a/scripts/tr/deed.mjs b/scripts/tr/deed.mjs new file mode 100644 index 00000000..13837d7a --- /dev/null +++ b/scripts/tr/deed.mjs @@ -0,0 +1,314 @@ +// Pure parser for a Търговски регистър deed (issue #279, ADR-0033). No I/O, no network, no state. +// +// The envelope is clean JSON, but every field's VALUE is an HTML fragment with semi-structured +// Bulgarian inside. That inner parser is where the libel risk lives, so the order of operations below +// is an invariant with tests on it, not a suggestion: +// +// decode HTML entities +// → split into entities on record-container / hr--report +// → drop entities marked erased +// → strip tags WITHIN one entity +// → separate the name from the address/stake WITHIN that entity +// → match tokens ONLY within a single entity +// +// Any other order silently merges blocks. Field CR_F_19_L routinely holds several съдружници in one +// string; matching a declarant against the whole field lets one person's given name combine with +// another's surname, and the output is a named public claim about the wrong human being. + +const OWNERSHIP_FIELDS = ['CR_F_18_L', 'CR_F_19_L', 'CR_F_23_L']; +const MANAGER_FIELD = 'CR_F_7_L'; +export const ROLE_FIELDS = [MANAGER_FIELD, ...OWNERSHIP_FIELDS]; +export { OWNERSHIP_FIELDS, MANAGER_FIELD }; + +// ── html ────────────────────────────────────────────────────────────────────── +const ENTITIES = { + '"': '"', + ''': "'", + '&': '&', + '<': '<', + '>': '>', + ' ': ' ', +}; +// `String.fromCodePoint` THROWS RangeError above U+10FFFF (and on a surrogate half), and the input is +// whatever the register put on the wire. An unguarded throw here does not stay local: it escapes +// entityBlocks and registrySeat, past the crawl loop's refuse-and-continue block (which covers only +// JSON.parse + assertUicEcho) and out of run() — one malformed escape in one deed ends a paced crawl +// that has already spent its request budget, and does the same to load.mjs at decision time. +// Out of range is not a character and cannot be part of a name, so it decodes to nothing: the rest of +// the entity still parses, which is the difference between losing a glyph and losing the run. +const MAX_CODE_POINT = 0x10ffff; +const codePoint = (n) => + Number.isInteger(n) && n >= 0 && n <= MAX_CODE_POINT ? String.fromCodePoint(n) : ''; + +/** Decode the entity set the register actually emits, plus numeric escapes. FIRST step, always. */ +function decodeEntities(s) { + return String(s) + .replace(/&(?:quot|apos|amp|lt|gt|nbsp);/g, (m) => ENTITIES[m]) + .replace(/&#(\d+);/g, (_, n) => codePoint(Number(n))) + .replace(/&#x([0-9a-f]+);/gi, (_, n) => codePoint(parseInt(n, 16))); +} + +/** Strip tags and collapse whitespace, INSIDE one already-isolated entity. */ +function stripTags(html) { + return String(html) + .replace(//gi, ' ') + .replace(/<[^>]*>/g, '') + .replace(/\s+/g, ' ') + .trim(); +} + +// An entity is erased when the register says so. MEASURED on a live deed: the marker is +// `erasure-text-inline` (with `ui-icon-erased`) and the container carries no `field-text` paragraph at +// all. `field-text--erased` is accepted as well — it is the other spelling reported for this register, +// and honouring both costs nothing while assuming one costs a wrong publish. +const ERASED_MARKER = /erasure-text-inline|ui-icon-erased|field-text--erased/; + +// The erasure notice, stripped so „Заличено обстоятелство." never reads as content. +// +// The `{0,2000}` bound is not cosmetic. With a plain `.*?` this backtracks QUADRATICALLY on markup where +// the opening div is never closed: every opening restarts a scan to end-of-input, measured at 34K→3.3ms, +// 68K→13.6ms, 136K→53.8ms, 272K→240ms, 1M→4.0s — ×4 per doubling. Bounding the lazy run makes each start +// position scan a fixed window instead, which is linear: the same inputs measure 5.3 / 10.3 / 24.8 / 41.6 +// / 187ms. A real notice is one short sentence, so 2000 characters is ~80× the live shape. +// +// Failing to match is safe by construction: `erased` is decided independently by ERASED_MARKER above, so +// an over-long notice still marks the block erased and liveFields still drops it. The only visible effect +// is that its text survives into the block — which in strict mode raises the drift alarm, loudly, rather +// than passing anything through silently. +const ERASURE_NOTICE = + /]*class=['"][^'"]*erasure-text-inline[^'"]*['"][^>]*>.{0,2000}?<\/div>/gis; + +/** + * Split one field's htmlData into the separate registered entities it holds. + * + * @param {string} html + * @param {{strict?:boolean}} [opts] `strict` turns the erased-with-content contradiction into a throw. + * @returns {{text:string, erased:boolean}[]} + */ +export function entityBlocks(html, { strict = false } = {}) { + if (html == null || String(html).trim() === '') return []; + const decoded = decodeEntities(html); // decode BEFORE splitting — see the order above + const chunks = decoded + .split(/]*>/i) + .flatMap((part) => part.split(/(?=]*class=['"][^'"]*record-container)/i)) + .map((c) => c.trim()) + .filter((c) => c !== ''); + + const out = []; + for (const chunk of chunks) { + const erased = ERASED_MARKER.test(chunk); + // Read the visible text WITHOUT the erasure notice, so „Заличено обстоятелство." never counts as + // content and an erased block reads as empty. + const withoutNotice = chunk.replace(ERASURE_NOTICE, ' '); + const text = stripTags(withoutNotice); + if (erased && text !== '' && strict) { + throw new Error( + `REFUSE: an erased entity carries content (${JSON.stringify(text.slice(0, 60))}) — the ` + + `"erased ⇒ empty" assumption has drifted; stop rather than guess which state is in force`, + ); + } + if (text === '' && !erased) continue; // structural noise, not an entity + out.push({ text, erased }); + } + return out; +} + +/** Every field in the deed, flattened out of sections → subDeeds → groups. */ +function allFields(deed) { + const out = []; + for (const s of deed?.sections ?? []) + for (const sd of s?.subDeeds ?? []) + for (const g of sd?.groups ?? []) for (const f of g?.fields ?? []) out.push(f); + return out; +} + +const isoDay = (v) => (v ? String(v).slice(0, 10) : null); + +/** + * The LIVE entities of the requested field codes — the single entry point to live state. + * Erased entities are dropped here and nowhere else, so the rule is auditable in one place. + * @returns {{nameCode:string, entryDate:string|null, entryNumber:string|null, entities:string[]}[]} + */ +export function liveFields(deed, nameCodes, opts = {}) { + const want = new Set(nameCodes); + const out = []; + for (const f of allFields(deed)) { + if (!want.has(f.nameCode)) continue; + const entities = entityBlocks(f.htmlData, opts) + .filter((b) => !b.erased) + .map((b) => b.text); + if (entities.length === 0) continue; + out.push({ + nameCode: f.nameCode, + // TEXT, never a number: a fieldEntryNumber like 20130716101007 exceeds 2^53 once combined. + entryNumber: f.fieldEntryNumber == null ? null : String(f.fieldEntryNumber), + entryDate: isoDay(f.fieldEntryDate), + entities, + }); + } + return out; +} + +// ── names ───────────────────────────────────────────────────────────────────── +/** + * Name tokens: NFC, upper case, split on non-letters, keep tokens of length ≥2. + * + * Dropping 1-character tokens is what makes „Г. И. Петров" a ONE-token name rather than a three-token + * one — an abbreviated name can then never reach the ≥3 tokens rung 2 requires, instead of passing on + * initials that match half the register. Latin letters are kept (never folded onto Cyrillic + * look-alikes) so a homoglyph is a non-match rather than a false match, matching companyNameKey's + * posture in packages/shared/src/company-name-key.ts. + * + * A near-twin of the module-private `holderTokens` in scripts/cacbg/parse.mjs — deliberately + * re-implemented rather than imported, because that module pulls in fast-xml-parser and would tie + * these pure tests to a workspace install. Keep the two in step. + */ +export function personTokens(name) { + return String(name ?? '') + .normalize('NFC') + .toUpperCase() + .split(/[^\p{L}]+/u) + .filter((t) => [...t].length >= 2); +} + +/** + * Does EVERY token of the declarant's name appear as a whole token of this ONE entity? + * + * Full subset, not a majority: of 301 measured matches, 46 were two-token only, which is precisely + * the homonym risk. Whole-token, not substring: „ПЕТРОВ" inside „ПЕТРОВА" is a different person. + * + * Callers MUST pass a single entity's text (see entityBlocks). Passing a whole field is the + * cross-entity bug, and no signature can prevent it — the test does. + */ +export function fullSubsetMatch(declarantName, entityText) { + const want = personTokens(declarantName); + if (want.length === 0) return false; + const have = new Set(personTokens(entityText)); + return want.every((t) => have.has(t)); +} + +// ── seat ────────────────────────────────────────────────────────────────────── +// Strip a settlement-type prefix only as a WHOLE token: „гр."/„с."/„общ."/„обл."/„ж.к." followed by a +// dot and optional space. R9 — a loose prefix strip turns СОФИЯ into ОФИЯ and ГРАДЕЦ into АДЕЦ. +const SETTLEMENT_PREFIX = /^(?:ГР|С|ОБЩ|ОБЛ|Ж\.К)\.\s*/u; + +/** Normalise a settlement name for comparison. Empty in ⇒ empty out, and empty NEVER confirms. */ +export function normalizeSettlement(raw) { + let s = String(raw ?? '') + .normalize('NFC') + .toUpperCase() + .replace(/\([^)]*\)/g, ' ') // „(столица)" + .trim(); + s = s.split(/[,/]/)[0].trim(); // cut at the first comma or slash — „с. Марково, п.к. 4108" + s = s.replace(SETTLEMENT_PREFIX, ''); + return s + .replace(/[^\p{L}\s-]/gu, '') + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * The company's registered settlement, from CR_F_5_L's „Населено място:" segment ONLY. + * + * ADR-0010 item 3 (addresses are never extracted) is honoured here, and the field makes that a live + * concern rather than a formality: CR_F_5_L also carries street, postcode, phone, fax, e-mail and + * website. Nothing but the settlement and the entry date leaves this function. + * @returns {{settlement:string, entryDate:string|null}} + */ +export function registrySeat(deed) { + for (const f of liveFields(deed, ['CR_F_5_L'])) { + for (const text of f.entities) { + const m = text.match(/Населено място:\s*([^,]+)/u); + if (m) return { settlement: normalizeSettlement(m[1]), entryDate: f.entryDate }; + } + } + return { settlement: '', entryDate: null }; +} + +// ── legal form ──────────────────────────────────────────────────────────────── +// Codes observed empirically (#279 §3 + the spike's catalogue read). DELIBERATELY incomplete: the +// nomenclature endpoint does not settle the enum (legalForm=4 and =10 return a byte-identical +// catalogue), so anything absent here is `unknown` and WITHHOLDS. In particular whether ЕАД carries +// its own code is unresolved — ООД and ЕООД turned out NOT to share one (4 vs 10), so assuming ЕАД +// shares 5 with АД would be exactly the fail-open this bar exists to prevent. +const FORM_CODES = new Map([ + [1, 'closely_held'], // ЕТ + [4, 'closely_held'], // ООД + [5, 'joint_stock'], // АД + [6, 'joint_stock'], // КДА + [10, 'closely_held'], // ЕООД +]); + +// The фирма's legal form is its SUFFIX under ЗТРРЮЛНЦ, and the deed envelope's `fullName` carries it +// („ПИМК" ООД) while CR_F_2_L is bare („ПИМК"). So the bar has a second, independent signal at zero +// extra cost. +// +// DELIBERATELY a twin of classify.mjs's JOINT_STOCK rather than an import, for the same reason +// personTokens twins parse.mjs's holderTokens: the dependency direction here is cacbg → tr (load.mjs +// imports this module), so importing back out of scripts/cacbg/ would close a cycle across the two +// directories. The two are pinned byte-identical by a test in deed.test.mjs — this comment used to +// claim КДА was missing from classify.mjs, which stopped being true in 5f64f5c, and an unenforced +// „keep these in step" note is exactly how that happens. +export const JOINT_SUFFIX = /(?:^|[\s"„“”«»])(АД|ЕАД|АДСИЦ|КДА)[\s"„“”«»]*$/u; +const CLOSELY_SUFFIX = /(?:^|[\s"„“”«»])(ООД|ЕООД|ЕТ|ДЗЗД|КД|СД|КООПЕРАЦИЯ)[\s"„“”«»]*$/u; + +/** + * Legal-form verdict — a union of the numeric code and the mandated фирма suffix. + * Either signal saying joint-stock bars the link. Neither able to say ⇒ `unknown`, which withholds. + * @returns {{code:number|null, codeVerdict:string, suffixVerdict:string, verdict:string}} + */ +export function registryLegalForm(deed) { + const code = typeof deed?.legalForm === 'number' ? deed.legalForm : null; + const codeVerdict = (code != null && FORM_CODES.get(code)) || 'unknown'; + + const name = String(deed?.fullName ?? '') + .normalize('NFC') + .toUpperCase() + .trim(); + const suffixVerdict = JOINT_SUFFIX.test(name) + ? 'joint_stock' + : CLOSELY_SUFFIX.test(name) + ? 'closely_held' + : 'unknown'; + + const verdict = + codeVerdict === 'joint_stock' || suffixVerdict === 'joint_stock' + ? 'joint_stock' + : codeVerdict === 'closely_held' || suffixVerdict === 'closely_held' + ? 'closely_held' + : 'unknown'; + return { code, codeVerdict, suffixVerdict, verdict }; +} + +// ── refutation input ────────────────────────────────────────────────────────── +/** + * Latest entry date across the LIVE ownership fields, or null when none survives. + * + * The trap this avoids was present in the first company sampled: CR_F_23_L sits in the CURRENT deed + * dated 2013-07-16 carrying only „Заличено обстоятелство.". Counted naively it becomes the latest + * ownership entry and can refute a link it says nothing about. liveFields drops it. + */ +export function latestOwnershipEntryDate(deed) { + const dates = liveFields(deed, OWNERSHIP_FIELDS) + .map((f) => f.entryDate) + .filter(Boolean); + return dates.length ? dates.sort().at(-1) : null; +} + +/** + * The deed we got back must be the deed we asked for. + * + * R8: Bulgarian public bodies carry ЕИК of exactly the `000…` shape, so any numeric round-trip on the + * path silently rewrites the identifier. Without this rail, that failure publishes a claim about a + * different company under the right official's name. + */ +export function assertUicEcho(deed, requestedEik) { + const got = deed?.uic == null ? null : String(deed.uic); + if (got !== String(requestedEik)) { + throw new Error( + `REFUSE: deed uic echo mismatch — requested ${JSON.stringify(String(requestedEik))}, ` + + `deed reports ${JSON.stringify(got)}`, + ); + } + return deed; +} diff --git a/scripts/tr/deed.test.mjs b/scripts/tr/deed.test.mjs new file mode 100644 index 00000000..9c539c87 --- /dev/null +++ b/scripts/tr/deed.test.mjs @@ -0,0 +1,391 @@ +// node:test — the deed parser. Pure, offline, and the most dangerous code in this change. +// +// Every fixture below is REAL markup, copied verbatim from a live deed (ЕИК 115536179, fetched +// 2026-08-05) with person names replaced only where a test needs a specific shape. Writing this +// parser against imagined markup is how the entity-boundary bug ships. +// +// The failure this file exists to prevent: field CR_F_19_L holds THREE separate people in one string, +// separated by
. Matching a declarant's tokens against the whole field lets +// the given name of one person combine with the surname of another, and the result is a named public +// claim that a specific official owns a specific company — about the wrong person. ADR-0033 decision 2. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + entityBlocks, + liveFields, + personTokens, + fullSubsetMatch, + normalizeSettlement, + registrySeat, + registryLegalForm, + latestOwnershipEntryDate, + assertUicEcho, + JOINT_SUFFIX, +} from './deed.mjs'; + +// ── real markup ─────────────────────────────────────────────────────────────── +const F19_THREE = `

ПИМК ХОЛДИНГ ГРУП АД, ЕИК/ПИК 202294392, Държава: БЪЛГАРИЯ, Размер на дяловото участие: 59980.00 лв.

ПЕНКО НЕСТОРОВ НЕСТОРОВ, Държава: БЪЛГАРИЯ, Размер на дяловото участие: 10.00 лв.

ИЛИЯН КОСТАДИНОВ ФИЛИПОВ, Държава: БЪЛГАРИЯ, Размер на дяловото участие: 10.00 лв.

`; + +const F23_ERASED = `
Заличено обстоятелство.
`; + +const F5_SEAT = `

Държава: БЪЛГАРИЯ
Област: Пловдив, Община: Родопи
Населено място: с. Марково, п.к. 4108
бул./ул. местност ЗАХАРИДЕВО № 043А Телефон: 032/901102 и 032/945149
Адрес на електронна поща: office@pimk-bg.eu

`; + +const F7_MANAGER = `

АНТОН ИЦКОВ ЙОРДАНОВ, Държава: БЪЛГАРИЯ

`; + +const field = (nameCode, htmlData, over = {}) => ({ + nameCode, + htmlData, + fieldEntryNumber: '20130716101007', + fieldEntryDate: '2013-07-16T10:10:07', + fieldOperation: 3, + fieldIdent: '00190', + ...over, +}); +const deedOf = (fields, over = {}) => ({ + uic: '115536179', + fullName: '"ПИМК" ООД', + legalForm: 4, + sections: [{ subDeeds: [{ groups: [{ fields }] }] }], + ...over, +}); + +// ── T1 — the entity boundary ────────────────────────────────────────────────── +test('entityBlocks splits one field into its separate registered entities', () => { + const blocks = entityBlocks(F19_THREE); + assert.equal(blocks.length, 3, 'three съдружници, three blocks'); + assert.match(blocks[0].text, /ПИМК ХОЛДИНГ ГРУП АД/); + assert.match(blocks[1].text, /ПЕНКО НЕСТОРОВ НЕСТОРОВ/); + assert.match(blocks[2].text, /ИЛИЯН КОСТАДИНОВ ФИЛИПОВ/); + // No block may carry another block's name — that is the whole point. + assert.ok(!blocks[1].text.includes('ФИЛИПОВ')); + assert.ok(!blocks[2].text.includes('ПЕНКО')); +}); + +test('T1 — tokens from two DIFFERENT people never combine into a match', () => { + // „ПЕНКО … НЕСТОРОВ" and „ИЛИЯН КОСТАДИНОВ ФИЛИПОВ" are both in field 19. A declarant assembled + // from one person's given name and another's patronymic+surname must NOT match. + const frankenstein = 'ПЕНКО КОСТАДИНОВ ФИЛИПОВ'; + const blocks = entityBlocks(F19_THREE); + assert.equal( + blocks.some((b) => fullSubsetMatch(frankenstein, b.text)), + false, + 'cross-entity match — this is the libel bug', + ); + // Whole-field matching is exactly what must not be done; prove the naive approach WOULD have fired, + // so this test cannot silently pass because the matcher is broken in some other way. + assert.equal(fullSubsetMatch(frankenstein, F19_THREE), true, 'naive whole-field match fires'); +}); + +// The live register emits single-quoted attributes today, and every fixture above is verbatim from a +// real deed. But the entity split is the ONLY thing standing between three people and one merged +// token pool, and it must not be quiet about a markup change: if the register ever switches to +// class="…", a quote-specific pattern stops splitting, all three owners collapse into one block, and +// the frankenstein match above starts firing — a named public claim about a person who is not there. +// R7's doctrine for this module is „refuse loudly, never guess"; silently mis-splitting is neither. +const dq = (s) => s.replace(/class='([^']*)'/g, 'class="$1"'); + +test('T1 — the entity split survives double-quoted attributes (markup-drift hardening)', () => { + // Strip the
separators first. entityBlocks splits on BOTH
and record-container precisely + // because either may be absent; with the
present this test would pass on the
rule alone + // and prove nothing about the quote handling it is here to pin. + const blocks = entityBlocks(dq(F19_THREE).replace(/]*>/gi, '')); + assert.equal(blocks.length, 3, 'a quote style change must not merge three owners into one block'); + assert.equal( + blocks.some((b) => fullSubsetMatch('ПЕНКО КОСТАДИНОВ ФИЛИПОВ', b.text)), + false, + 'cross-entity match under double quotes — the libel bug via markup drift', + ); + assert.ok(blocks.some((b) => fullSubsetMatch('ПЕНКО НЕСТОРОВ НЕСТОРОВ', b.text))); +}); + +test('T1 — erasure is still detected and stripped under double-quoted attributes', () => { + const [block] = entityBlocks(dq(F23_ERASED)); + assert.equal(block.erased, true); + assert.equal(block.text, '', 'the erasure notice must not survive as content'); + // The strict contradiction check must not fire merely because the quote style changed. + assert.doesNotThrow(() => entityBlocks(dq(F23_ERASED), { strict: true })); +}); + +test('T1 positive control — the CORRECT declarant does match', () => { + // Without this, a matcher that always returns false passes every negative test above (ADR-0027). + const blocks = entityBlocks(F19_THREE); + assert.equal( + blocks.some((b) => fullSubsetMatch('ИЛИЯН КОСТАДИНОВ ФИЛИПОВ', b.text)), + true, + ); + assert.equal( + blocks.some((b) => fullSubsetMatch('Пенко Несторов Несторов', b.text)), + true, + 'case/spacing drift still matches within the right entity', + ); +}); + +test('T1 — HTML entities are decoded BEFORE splitting, not after', () => { + // Decode-after-split leaves "-bearing names mangled; decode-before-split is the tested order. + const encoded = F19_THREE.replace('ПИМК ХОЛДИНГ ГРУП АД', '"ПИМК ХОЛДИНГ ГРУП" АД'); + const blocks = entityBlocks(encoded); + assert.equal(blocks.length, 3); + assert.match(blocks[0].text, /"ПИМК ХОЛДИНГ ГРУП" АД/); + assert.ok(!blocks[0].text.includes('"'), 'entities must be decoded'); +}); + +test('T1 — a corporate съдружник contributes its own tokens, not a person match', () => { + // „ПИМК ХОЛДИНГ ГРУП АД, ЕИК/ПИК 202294392" is a legal entity. A person whose name happens to + // overlap its words must not match it into existence. + const blocks = entityBlocks(F19_THREE); + assert.equal( + fullSubsetMatch('ПИМК ХОЛДИНГ ГРУП', blocks[0].text), + true, + 'literal overlap exists', + ); + // …which is why rung 2 requires ≥3 tokens of a PERSON name; the guard lives in evidence.mjs (T2). +}); + +// ── erasure ─────────────────────────────────────────────────────────────────── +test('an erased entity is detected structurally and dropped from live state', () => { + // MEASURED: the live deed marks erasure with `erasure-text-inline` and the container carries NO + //

at all. An earlier note claimed the marker was `field-text--erased`; that + // class does not occur in the sampled deed. Both are treated as erasure so either shape is safe. + const blocks = entityBlocks(F23_ERASED); + assert.equal(blocks.length, 1); + assert.equal(blocks[0].erased, true); + const live = liveFields(deedOf([field('CR_F_23_L', F23_ERASED, { fieldOperation: 2 })]), [ + 'CR_F_23_L', + ]); + assert.deepEqual(live, [], 'an erased field contributes no live entity'); +}); + +test('the OTHER erasure spelling is honoured too', () => { + const alt = `

СТАР СОБСТВЕНИК

Заличено обстоятелство.
`; + assert.equal(entityBlocks(alt)[0].erased, true); +}); + +test('an erased-looking block carrying real content REFUSES the deed (drift alarm)', () => { + // R7: "erased ⇒ empty" is an empirical observation (93 of 93). If the register ever emits an erased + // marker beside live content, the assumption is broken and we must stop, not silently drop a real + // owner or silently keep a removed one. + const contradictory = `

ЖИВО ИМЕ ТУК

Заличено обстоятелство.
`; + assert.throws(() => entityBlocks(contradictory, { strict: true }), /erased.*content|drift/i); +}); + +test('an empty htmlData yields no entities and does not throw', () => { + assert.deepEqual(entityBlocks(''), []); + assert.deepEqual(entityBlocks(null), []); +}); + +// A numeric entity is attacker-shaped input in the only sense that matters here: it comes off the +// wire, and `String.fromCodePoint` throws RangeError above U+10FFFF. That throw escapes decodeEntities, +// escapes entityBlocks and registrySeat, and — because the crawl loop's try/catch covers only +// JSON.parse + assertUicEcho — escapes run() and kills the process. One malformed entity in one deed +// would end a paced crawl that has already spent its request budget. Out of range is not a person, so +// the only defensible reading is „no character": drop it and keep parsing the rest of the entity. +test('an out-of-range numeric entity is dropped, never thrown out of the parser', () => { + const overflow = F19_THREE.replace('ПЕНКО', '�ПЕНКО'); + const blocks = entityBlocks(overflow); + assert.equal(blocks.length, 3, 'the deed still parses into its three entities'); + assert.match(blocks[1].text, /ПЕНКО НЕСТОРОВ НЕСТОРОВ/, 'the surrounding name survives intact'); + assert.ok(!blocks[1].text.includes('&#'), 'the escape itself does not survive as literal text'); +}); + +test('the hex numeric form is guarded too — both decode lines, not just the decimal one', () => { + const overflow = F19_THREE.replace('ИЛИЯН', '�ИЛИЯН'); + const blocks = entityBlocks(overflow); + assert.equal(blocks.length, 3); + assert.match(blocks[2].text, /ИЛИЯН КОСТАДИНОВ ФИЛИПОВ/); +}); + +test('an in-range numeric entity still decodes — the guard bounds, it does not disable', () => { + // П is „П". A guard that dropped every numeric escape would silently mangle real names. + const blocks = entityBlocks(F19_THREE.replace('ПЕНКО', 'ПЕНКО')); + assert.match(blocks[1].text, /ПЕНКО НЕСТОРОВ НЕСТОРОВ/); +}); + +test('registrySeat survives the same malformed entity rather than aborting the load', () => { + // registrySeat sits OUTSIDE the crawl loop's refuse-and-continue block (fetch-deeds.mjs), and + // load.mjs calls it again at decision time — so an unguarded throw here takes down both legs. + const d = deedOf([field('CR_F_5_L', F5_SEAT.replace('с. Марково', '�с. Марково'))]); + const seat = registrySeat(d); + assert.equal(seat.settlement, 'МАРКОВО'); +}); + +test('liveFields keeps only the requested codes and reports entry date/number', () => { + const d = deedOf([ + field('CR_F_7_L', F7_MANAGER, { nameCode: 'CR_F_7_L', fieldEntryDate: '2017-09-15T00:00:00' }), + field('CR_F_19_L', F19_THREE), + field('CR_F_5_L', F5_SEAT), + ]); + const live = liveFields(d, ['CR_F_7_L', 'CR_F_19_L']); + assert.deepEqual(live.map((f) => f.nameCode).sort(), ['CR_F_19_L', 'CR_F_7_L']); + const f19 = live.find((f) => f.nameCode === 'CR_F_19_L'); + assert.equal(f19.entities.length, 3); + assert.equal(f19.entryDate, '2013-07-16'); + assert.equal(f19.entryNumber, '20130716101007'); + assert.equal(typeof f19.entryNumber, 'string', 'entry numbers exceed 2^53 — never a number'); +}); + +// ── T2 — the token rule ─────────────────────────────────────────────────────── +test('personTokens keeps tokens of length ≥2 and folds case/spacing', () => { + assert.deepEqual(personTokens('Иван Петров Георгиев'), ['ИВАН', 'ПЕТРОВ', 'ГЕОРГИЕВ']); + assert.deepEqual(personTokens(' иван петров '), ['ИВАН', 'ПЕТРОВ']); + // An initial is not a token — „Г. И. Петров" is ONE token, so it can never reach three. + assert.deepEqual(personTokens('Г. И. Петров'), ['ПЕТРОВ']); + // A hyphenated surname is one token, not two. + assert.deepEqual(personTokens('Мария Иванова-Петрова'), ['МАРИЯ', 'ИВАНОВА', 'ПЕТРОВА']); +}); + +test('fullSubsetMatch requires EVERY declarant token, not a majority', () => { + const entity = 'ИЛИЯН КОСТАДИНОВ ФИЛИПОВ, Държава: БЪЛГАРИЯ'; + assert.equal(fullSubsetMatch('ИЛИЯН КОСТАДИНОВ ФИЛИПОВ', entity), true); + // 2-of-3 must fail: of 301 measured matches, 46 were two-token only — the homonym risk itself. + assert.equal(fullSubsetMatch('ИЛИЯН КОСТАДИНОВ ПЕТРОВ', entity), false); + assert.equal(fullSubsetMatch('ИЛИЯН ПЕТРОВ ФИЛИПОВ', entity), false); +}); + +test('a token must match a WHOLE token, never a substring', () => { + // „ПЕТРОВ" must not be found inside „ПЕТРОВА"; that is a different person. + assert.equal(fullSubsetMatch('ИВАН ПЕТРОВ ГЕОРГИЕВ', 'ИВАН ПЕТРОВА ГЕОРГИЕВА'), false); +}); + +// ── T5 — settlement normalization ───────────────────────────────────────────── +test('T5 — the settlement prefix is stripped only as a whole token', () => { + // R9: a naive prefix strip turns СОФИЯ into ОФИЯ and ГРАДЕЦ into АДЕЦ. + assert.equal(normalizeSettlement('гр. Русе'), 'РУСЕ'); + assert.equal(normalizeSettlement('с. Марково'), 'МАРКОВО'); + assert.equal(normalizeSettlement('София'), 'СОФИЯ'); + assert.equal(normalizeSettlement('СОФИЯ'), 'СОФИЯ'); + assert.equal(normalizeSettlement('Градец'), 'ГРАДЕЦ'); + assert.equal(normalizeSettlement('гр.Пловдив'), 'ПЛОВДИВ'); + assert.equal(normalizeSettlement('София (столица)'), 'СОФИЯ'); +}); + +test('T5 — an empty settlement never equals another empty settlement', () => { + // „both blank ⇒ confirmed" would rubber-stamp every link with no seat data at all. + assert.equal(normalizeSettlement(''), ''); + assert.equal(normalizeSettlement(null), ''); + assert.equal(normalizeSettlement(' '), ''); +}); + +test('registrySeat reads the „Населено място" segment and NOTHING else', () => { + const seat = registrySeat(deedOf([field('CR_F_5_L', F5_SEAT, { nameCode: 'CR_F_5_L' })])); + assert.equal(seat.settlement, 'МАРКОВО'); + assert.equal(seat.entryDate, '2013-07-16'); + // ADR-0010 item 3: the parser never returns an address, phone, e-mail or website — and the deed + // demonstrably carries all four. + const blob = JSON.stringify(seat); + for (const leak of ['ЗАХАРИДЕВО', '032/901102', 'office@pimk-bg.eu', 'п.к.', '4108', 'Родопи']) + assert.ok(!blob.includes(leak), `seat must not carry ${leak}`); +}); + +// ── T3 — the joint-stock bar ────────────────────────────────────────────────── +test('T3 — the legal-form verdict is a UNION of the code and the ЗТРРЮЛНЦ suffix', () => { + const jointByCode = registryLegalForm(deedOf([], { legalForm: 5, fullName: 'НЕЩО СИ' })); + assert.equal(jointByCode.verdict, 'joint_stock'); + + // An UNKNOWN code must not fall through: the suffix decides, and if it cannot, we withhold. + const unknownButEad = registryLegalForm(deedOf([], { legalForm: 99, fullName: '"ГАМА" ЕАД' })); + assert.equal(unknownButEad.verdict, 'joint_stock', 'barred by the mandated suffix'); + + const unknownButEood = registryLegalForm(deedOf([], { legalForm: 99, fullName: '"БЕТА" ЕООД' })); + assert.equal(unknownButEood.verdict, 'closely_held', 'the bar is not blanket'); + + const unknownAndUnreadable = registryLegalForm(deedOf([], { legalForm: 99, fullName: 'НЕЩО' })); + assert.equal(unknownAndUnreadable.verdict, 'unknown', 'unknown withholds — it never publishes'); +}); + +test('T3 — КДА is barred (it is not in the existing closelyHeldForm token list)', () => { + assert.equal( + registryLegalForm(deedOf([], { legalForm: 99, fullName: '"X" КДА' })).verdict, + 'joint_stock', + ); + assert.equal( + registryLegalForm(deedOf([], { legalForm: 6, fullName: 'X' })).verdict, + 'joint_stock', + ); +}); + +test('T3 — a real ООД deed reads as closely held, by code AND by suffix', () => { + const v = registryLegalForm(deedOf([])); + assert.equal(v.code, 4); + assert.equal(v.verdict, 'closely_held'); + assert.equal(v.suffixVerdict, 'closely_held', 'fullName carries the form: "ПИМК" ООД'); +}); + +// ── T7 — the UIC echo ───────────────────────────────────────────────────────── +test('T7 — a deed whose UIC does not echo the request is REFUSED', () => { + // R8: ЕИК leading zeros are significant (public bodies are exactly 000…). If anything on the path + // rewrites the identifier, this is the rail that catches it before a claim is made about the + // wrong company. + assert.doesNotThrow(() => assertUicEcho(deedOf([]), '115536179')); + assert.throws(() => assertUicEcho(deedOf([]), '000696327'), /uic|echo/i); + assert.throws(() => assertUicEcho(deedOf([], { uic: '696327' }), '000696327'), /uic|echo/i); + assert.throws(() => assertUicEcho(deedOf([], { uic: null }), '115536179'), /uic|echo/i); +}); + +// ── the refutation input ────────────────────────────────────────────────────── +test('latestOwnershipEntryDate ignores ERASED ownership fields', () => { + // The trap, present in the first company sampled: CR_F_23_L is live in the current deed, dated + // 2013-07-16, and contains only „Заличено обстоятелство.". Read naively it becomes „latest + // ownership entry: 2013-07-16" and can refute a link it knows nothing about. + const d = deedOf([ + field('CR_F_19_L', F19_THREE, { fieldEntryDate: '2011-05-02T00:00:00' }), + field('CR_F_23_L', F23_ERASED, { fieldEntryDate: '2013-07-16T10:10:07', fieldOperation: 2 }), + ]); + assert.equal(latestOwnershipEntryDate(d), '2011-05-02', 'the erased 2013 entry must not count'); +}); + +test('latestOwnershipEntryDate is null when no live ownership field survives', () => { + const d = deedOf([field('CR_F_23_L', F23_ERASED, { fieldOperation: 2 })]); + assert.equal(latestOwnershipEntryDate(d), null); +}); + +// The erasure-notice strip used an unbounded lazy `.*?`, which backtracks quadratically when the opening +// div is never closed — each opening restarts a scan to end-of-input. Measured on that shape: 34K→3.3ms, +// 68K→13.6ms, 136K→53.8ms, 272K→240ms, 1M→4.0s (×4 per doubling). The parser runs against whatever the +// register returns, so that is remote-controlled CPU on a paced crawl with a per-request budget. +test('adversarial unclosed markup parses in linear time, not quadratically', () => { + // ~1 MB of unclosed erasure openings — the exact shape that triggers the backtracking. + const doc = '
z'.repeat(32_000); + const t = process.hrtime.bigint(); + entityBlocks(doc); + const ms = Number(process.hrtime.bigint() - t) / 1e6; + // Bounded measures ~190ms here and unbounded ~4000ms, so 1500ms separates them with ~8× headroom + // over the bounded path — wide enough not to flake on a loaded runner, tight enough to catch a + // reintroduced `.*?`. + assert.ok(ms < 1500, `entityBlocks took ${ms.toFixed(0)}ms on 1MB of unclosed markup`); +}); + +test('an over-long erasure notice still marks the block erased — the bound cannot leak a live owner', () => { + // If the notice exceeds the bound the regex simply does not strip it. `erased` is decided separately + // by ERASED_MARKER, so the block is still erased and liveFields still drops it: the failure mode of + // the bound is a noisier block, never a resurrected owner. + const long = `
${'Заличено. '.repeat(400)}
`; + const [block] = entityBlocks(long); + assert.equal(block.erased, true); + assert.deepEqual( + liveFields(deedOf([field('CR_F_19_L', long)]), ['CR_F_19_L']), + [], + 'an erased block contributes no live entity regardless of its notice length', + ); +}); + +// JOINT_SUFFIX here and JOINT_STOCK in scripts/cacbg/classify.mjs are the SAME rule — which legal-form +// suffixes mark a share-issuing company — held in two places because the TR parser cannot import out of +// scripts/cacbg/ without closing a cacbg↔tr cycle. The drift this risks has already happened once: 5f64f5c +// added КДА to classify.mjs while deed.mjs's comment still asserted it was absent there. A prose „keep +// these in step" note does not keep anything in step; this does. +test('the joint-stock suffix rule is identical in the TR parser and the classifier', async () => { + const { JOINT_STOCK } = await import('../cacbg/classify.mjs'); + assert.equal(JOINT_SUFFIX.source, JOINT_STOCK.source, 'the two patterns have diverged'); + assert.equal(JOINT_SUFFIX.flags, JOINT_STOCK.flags, 'the two patterns have diverged in flags'); + // Behavioural pin as well as textual: identical sources with different behaviour is impossible, but a + // future refactor could legitimately change BOTH sources while breaking one. These are the forms the + // bar exists for — every one must be caught by both, or a joint-stock parcel publishes as ownership. + for (const name of ['ТРЕЙС ГРУП ХОЛД АД', 'НЕЩО ЕАД', 'ФОНД АДСИЦ', 'НЕЩО КДА']) { + assert.equal(JOINT_SUFFIX.test(name), true, name); + assert.equal(JOINT_STOCK.test(name), true, name); + } + for (const name of ['АЛФА СТРОЙ ООД', 'БЕТА ЕООД', 'АД ГРУП ООД']) { + assert.equal(JOINT_SUFFIX.test(name), false, name); + assert.equal(JOINT_STOCK.test(name), false, name); + } +}); diff --git a/scripts/tr/eik.mjs b/scripts/tr/eik.mjs new file mode 100644 index 00000000..5a4d5b12 --- /dev/null +++ b/scripts/tr/eik.mjs @@ -0,0 +1,61 @@ +// ЕИК validity — the Node twin of the rule that already lives in SQL. +// +// `eik_valid` in scripts/normalize-raw.sql decides which bidders get an ЕИК-keyed identity at all, so +// it defines the ЕИК space the whole matcher works in. Until now that rule was callable ONLY from +// SQL, which is why the registry leg needs this: the crawler must decide, in Node, whether a code is +// worth a lookup. The two implementations are pinned against each other by a test that lifts the CASE +// expression straight out of the .sql file and runs both over the same values (eik.test.mjs) — a copy +// of the rule would drift from the thing it is copying. +// +// The rule (ЗТРРЮЛНЦ / БУЛСТАТ): +// 9-digit — weight digits 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by 3..10; +// a second 10 becomes 0. Digit 9 must equal the control. +// 13-digit — the leading 9 must themselves be a valid 9-digit ЕИК, then weight digits 9..12 by +// 2,7,3,5 (fallback 4,9,5,7; a second 10 becomes 0). Digit 13 must equal that control. +// +// Everything here is string-in, string-out. An ЕИК is an identifier, not a number: public bodies carry +// codes of exactly the `000…` shape, and a numeric round-trip drops the leading zeros and silently +// turns one company's identifier into another's. + +/** Weighted control digit over `digits`, with the standard second-pass fallback. @returns {number} */ +function control(digits, primary, fallback) { + const sum = (ws) => ws.reduce((acc, w, i) => acc + w * digits[i], 0) % 11; + const first = sum(primary); + if (first < 10) return first; + const second = sum(fallback); + return second < 10 ? second : 0; +} + +/** + * Is this a structurally valid ЕИК (9 or 13 digits, correct control digit)? + * Service codes (`000000000`, `0000000000000`) are rejected outright, matching the SQL: they pass the + * arithmetic but are placeholders, and letting them through collapsed unrelated foreign suppliers onto + * one node (#195). + * @param {unknown} eik @returns {boolean} + */ +export function eikChecksumValid(eik) { + const s = String(eik ?? ''); + if (s === '000000000' || s === '0000000000000') return false; + if (!/^\d+$/.test(s)) return false; + if (s.length !== 9 && s.length !== 13) return false; + + const d = [...s].map(Number); + const c9 = control(d.slice(0, 8), [1, 2, 3, 4, 5, 6, 7, 8], [3, 4, 5, 6, 7, 8, 9, 10]); + if (c9 !== d[8]) return false; + if (s.length === 9) return true; + + const c13 = control(d.slice(8, 12), [2, 7, 3, 5], [4, 9, 5, 7]); + return c13 === d[12]; +} + +/** + * Normalise a raw ЕИК string to digits only, or null when it is not one. + * Mirrors the SQL's `eik_clean`: strips a leading „ЕИК " label and surrounding whitespace, and does + * NOT otherwise repair the value. Validity is a separate question — `eikChecksumValid`. + * @param {unknown} raw @returns {string|null} + */ +export function normalizeEik(raw) { + const s = String(raw ?? '').trim(); + const stripped = (s.startsWith('ЕИК ') ? s.slice(4) : s).trim(); + return /^\d{9}$|^\d{13}$/.test(stripped) ? stripped : null; +} diff --git a/scripts/tr/eik.test.mjs b/scripts/tr/eik.test.mjs new file mode 100644 index 00000000..7a80a191 --- /dev/null +++ b/scripts/tr/eik.test.mjs @@ -0,0 +1,135 @@ +// node:test — the ЕИК checksum, and its PARITY with the SQL that already owns this rule. +// +// Why parity matters: `eik_valid` in scripts/normalize-raw.sql decides which bidders get an ЕИК-keyed +// identity at all, so it defines the ЕИК space the whole matcher works in. A JS twin that disagrees +// would silently accept a code the pipeline rejects (or vice versa) and the registry lookup would be +// made against an entity the rest of the system does not believe exists. The parity test below runs +// both implementations over the same values through node:sqlite, so they cannot drift apart. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { DatabaseSync } from 'node:sqlite'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { eikChecksumValid, normalizeEik } from './eik.mjs'; + +// Real ЕИК, verifiable against the public registers — the only external anchor this rule has. +const REAL = [ + '000696327', // Община София + '831661388', // Министерство на регионалното развитие + '115536179', // „ПИМК" ООД + '207695026', // „Профит Екстра" ЕООД +]; + +test('accepts real 9-digit ЕИК', () => { + for (const e of REAL) assert.equal(eikChecksumValid(e), true, e); +}); + +test('rejects a wrong-digit twin of every real ЕИК', () => { + // The failure this rule exists to stop (#195): a typo twin passing as a distinct real company and + // collapsing unrelated suppliers onto one node. Mutating the LAST digit must always break it. + for (const e of REAL) { + const bad = e.slice(0, 8) + ((Number(e[8]) + 1) % 10); + assert.equal(eikChecksumValid(bad), false, `${bad} (twin of ${e})`); + } +}); + +test('rejects service codes, wrong lengths and non-digits', () => { + for (const bad of [ + '000000000', + '0000000000000', + '', + null, + undefined, + '12345678', // 8 + '1234567890', // 10 — never a valid ЕИК length, which is what makes the ЕГН guard sound + '11553617x', + '115 536 179', + ]) { + assert.equal(eikChecksumValid(bad), false, String(bad)); + } +}); + +test('13-digit branch requires BOTH the 9-digit prefix and the 13th control digit', () => { + // A клон/поделение code: the leading 9 must themselves be a valid ЕИК, then weights 2,7,3,5. + const base = '115536179'; + const valid13 = thirteen(base, '001'); + assert.equal(valid13.length, 13, valid13); + assert.equal(eikChecksumValid(valid13), true, valid13); + // break the 13th digit + const broken = valid13.slice(0, 12) + ((Number(valid13[12]) + 1) % 10); + assert.equal(eikChecksumValid(broken), false, broken); + // a valid 13th control over an INVALID 9-prefix must still fail — both halves are load-bearing + assert.equal(eikChecksumValid(thirteen('115536170', '001')), false); +}); + +// Build a 13-digit ЕИК with a correct final control digit: 9-digit prefix + 3 free digits + control. +// The 2,7,3,5 weights run over positions 9..12 — that is the prefix's OWN control digit plus the three +// free ones — and position 13 is the result. +function thirteen(prefix9, three) { + const s = prefix9 + three; + const d = [...s].map(Number); + const w = (ws) => ws.reduce((a, x, i) => a + x * d[8 + i], 0) % 11; + let c = w([2, 7, 3, 5]); + if (c === 10) { + c = w([4, 9, 5, 7]); + if (c === 10) c = 0; + } + return s + c; +} + +test('normalizeEik strips the „ЕИК " prefix and surrounding whitespace, like the SQL does', () => { + assert.equal(normalizeEik('ЕИК 115536179'), '115536179'); + assert.equal(normalizeEik(' 115536179 '), '115536179'); + assert.equal(normalizeEik('115536179'), '115536179'); + assert.equal(normalizeEik('не е ЕИК'), null); + assert.equal(normalizeEik(null), null); +}); + +test('normalizeEik preserves leading zeros (public bodies are exactly 000…)', () => { + // Lose these and the crawler fetches a DIFFERENT company's deed. String in, string out — never a + // numeric round-trip. + assert.equal(normalizeEik('000696327'), '000696327'); + assert.equal(typeof normalizeEik('000696327'), 'string'); +}); + +// ── the anti-drift pin ──────────────────────────────────────────────────────── +test('JS twin agrees with normalize-raw.sql eik_valid on every value', () => { + const sql = readFileSync(fileURLToPath(new URL('../normalize-raw.sql', import.meta.url)), 'utf8'); + const expr = extractEikValidExpression(sql); + + const db = new DatabaseSync(':memory:'); + db.exec('CREATE TABLE probe (eik_clean TEXT)'); + const ins = db.prepare('INSERT INTO probe VALUES (?)'); + + const cases = [...REAL, ...REAL.map((e) => e.slice(0, 8) + ((Number(e[8]) + 1) % 10))]; + for (const e of REAL) cases.push(thirteen(e, '001'), thirteen(e, '002')); + // Every 9-digit code over a fixed prefix — exercises BOTH weight passes incl. the second-10 → 0 fallback. + for (let i = 0; i < 100; i++) cases.push('2011223' + String(i).padStart(2, '0')); + for (const bad of ['000000000', '0000000000000', '12345678', '1234567890', '11553617x']) + cases.push(bad); + + for (const c of cases) ins.run(c); + const rows = db.prepare(`SELECT eik_clean AS e, ${expr} AS v FROM probe`).all(); + db.close(); + + assert.ok(rows.length >= 120, `expected a broad probe set, got ${rows.length}`); + for (const { e, v } of rows) { + assert.equal( + eikChecksumValid(e), + v === 1, + `disagreement on ${JSON.stringify(e)} (SQL said ${v})`, + ); + } +}); + +/** + * Lift the `CASE … END AS eik_valid` expression straight out of normalize-raw.sql, so the test + * compares against the FILE and not a copy of it. A copy would drift with the thing it is pinning. + */ +function extractEikValidExpression(sql) { + const start = sql.indexOf(' CASE\n WHEN eik_clean IS NULL'); + assert.ok(start > 0, 'eik_valid CASE not found in normalize-raw.sql — the pin lost its anchor'); + const end = sql.indexOf('AS eik_valid', start); + assert.ok(end > start, 'eik_valid terminator not found'); + return sql.slice(start, end); +} diff --git a/scripts/tr/evidence.mjs b/scripts/tr/evidence.mjs new file mode 100644 index 00000000..56bc01ef --- /dev/null +++ b/scripts/tr/evidence.mjs @@ -0,0 +1,306 @@ +// The evidence ladder (issue #279 §5, ADR-0033 decision 1). Pure: deed in, verdict out. Zero network. +// +// Six outcomes, FIRST MATCH WINS: +// 1 bar_joint_stock АД / ЕАД / КДА — never published, whatever follows +// 2 document the declarant's full name is in a live CR_F_7/18/19/23_L entity +// 3 confirmed declared seat == registered seat, or the declarant wrote the ЕИК +// 4 refuted own stake only: absent from live state, and the live ownership record +// predates the declared period — the register covers it and does not name them +// 5 unknown everything else — held +// 6 outside_tr not in the register at all (ДЗЗД, БУЛСТАТ associations) — held +// +// WHAT THIS ESTABLISHES, precisely: the identity of the COMPANY — that the company behind the declared +// name is the same legal entity as the winner we matched. It does NOT establish that the official owns +// it; that claim comes from their own filed declaration and is not a heuristic at all. The failure mode +// of a wrong match is therefore not an invented ownership claim but a real official attached to the +// WRONG company's ЕИК, contracts and money. Still a false public statement about a named person, which +// is why rung 2 requires a full three-token subset match inside a single registry entity, and why the +// filters that can only withhold are kept (ADR-0033 decision 2). + +import { + liveFields, + fullSubsetMatch, + personTokens, + normalizeSettlement, + registrySeat, + registryLegalForm, + latestOwnershipEntryDate, + OWNERSHIP_FIELDS, + MANAGER_FIELD, + ROLE_FIELDS, +} from './deed.mjs'; + +/** + * Version of the RULES, not of the code. §8's monotonicity gate keys on this: a previously published + * link disappearing under an UNCHANGED rules version is a hard finding; under a changed one it is an + * expected diff. Bump it whenever a rung's meaning changes. + */ +export const RULES_VERSION = 'tr-rules-1'; + +/** Rung 2 needs a real three-part Bulgarian name (ЗГР чл. 9). Two tokens is the homonym risk itself. */ +const MIN_NAME_TOKENS = 3; + +/** + * The CLOSED vocabulary a sealed `matched_fact` may take: `seat:`, `role:owner:`, + * `role:manager:`, or `eik`. It must NEVER carry the matched NAME — the deed's names are read + * only to produce a boolean and never leave git-ignored scratch (#279 §9, ADR-0033 decision 5). + * + * The seat token bound is the whole rail. `seat:` is a legitimate prefix, so an unbounded settlement + * pattern admits `seat:ИВАН ПЕТРОВ ГЕОРГИЕВ` — a full three-part Bulgarian name (ЗГР чл. 9) wearing an + * allowed prefix, which is exactly the value a mis-split of the seat field would produce and exactly + * what the rail exists to reject. A settlement is one or two tokens („СОФИЯ", „ВЕЛИКО ТЪРНОВО", + * „ГЕНЕРАЛ ТОШЕВО"); a three-part name is exactly three. Bounding at two separates them cleanly, and a + * rarer 3-token seat stops the run for a human rather than publishing — the correct direction for a rail + * whose failure mode is putting somebody's name on a served column. + * + * Defined ONCE and consumed by both the writer (load.mjs) and the audit, so the two cannot drift into + * a state where the gate permits what the writer emits. + */ +export const MATCHED_FACT_RE = + /^(?:seat:\p{Lu}[\p{Lu}-]*(?: \p{Lu}[\p{Lu}-]*)?|role:(?:owner|manager):CR_F_\d+[a-z]?_L|eik)$/u; + +/** True when `fact` is a member of the closed vocabulary. `null` is legal — a rung may match no fact. */ +export function isSealedFact(fact) { + return fact == null || MATCHED_FACT_RE.test(String(fact)); +} + +// Court-registered companies were re-registered into the Търговски регистър in a single administrative +// push, which flattened their entry dates into this window. „Strictly before the declared period" +// certifies nothing when the date is an artefact of the migration rather than of the ownership, so the +// refutation rung is suppressed inside it (R13). A suppressed refutation falls through to `unknown` — +// held, not published, which is the safe direction. +const REREGISTRATION_START = '2011-01-01'; +const REREGISTRATION_END = '2012-12-31'; + +/** + * Find the declarant inside the live entities of the given field codes. + * Matching happens per ENTITY — never against a whole field — because one field routinely holds + * several people and combining tokens across them is the libel bug. + * @returns {{nameCode:string, entryNumber:string|null, entryDate:string|null}|null} + */ +function findPerson(deed, name, nameCodes) { + for (const f of liveFields(deed, nameCodes)) { + for (const entity of f.entities) { + if (fullSubsetMatch(name, entity)) { + return { nameCode: f.nameCode, entryNumber: f.entryNumber, entryDate: f.entryDate }; + } + } + } + return null; +} + +/** + * The registered seat, when it matches one THIS person declared for THIS company and was in force for the + * declared period. Shared by rung 3 (which publishes on it) and rung 2's company-identity corroborator. + * + * R10: seats move. A company that relocated INTO the declared settlement afterwards would confirm falsely, + * so the registered seat must predate the period. + * + * A null `firstDeclaredYear` FAILS the guard rather than skipping it. `load.mjs` passes null whenever no + * history row carried a parseable year, and an unknown year is not a satisfied temporal test — it is the + * absence of one. Reading it as „covers the period" made the weakest rung the only one with no temporal + * check, on exactly the links where we know least, and rung 4 already refuses to run without a year on the + * same ground. The undated-SEAT leg is different and stays: a seat with no entry date is the ordinary shape + * for a company that never moved, and it is checkable — a known year is still on the other side. + * + * @returns {{settlement:string, entryDate:string|null}|null} + */ +function matchDeclaredSeat(deed, declaredSeats, firstDeclaredYear) { + const seat = registrySeat(deed); + // Empty NEVER matches — otherwise every link with no seat data on either side rubber-stamps itself. + if (seat.settlement === '') return null; + if (firstDeclaredYear == null) return null; + if (seat.entryDate != null && seat.entryDate > `${firstDeclaredYear}-12-31`) return null; + const declared = declaredSeats.map(normalizeSettlement).filter((s) => s !== ''); + return declared.includes(seat.settlement) ? seat : null; +} + +/** + * Decide the evidence for one link. + * + * @param {object} input + * @param {object|null} input.deed parsed deed JSON; null only when `outsideTr` + * @param {boolean} [input.outsideTr] the ЕИК is not in the register at all + * @param {string} input.declarantName the office-holder's name as filed + * @param {string[]} [input.declaredSeats] seats declared BY THIS PERSON FOR THIS COMPANY only — + * 4.9% of company-name keys carry more than one distinct + * declared seat, so a company-only key would let one + * person's seat confirm another person's link + * @param {boolean} [input.declaredEik] the declarant wrote the ЕИК in the declaration + * @param {number|null} [input.firstDeclaredYear] + * @param {'self'|'family'} [input.scope] + * @param {boolean} [input.nameGloballyUnique] AND-gate on the WEAKEST rung only + * @param {boolean} [input.companyNameDistinctive] the declared фирма is unlikely to have a national + * twin. Gates an UNCORROBORATED rung 2 (ADR-0035). + * Defaults to FALSE: a caller that forgets it withholds. + * @returns {{kind:string, publishable:boolean, registryRole:string|null, matchedFact:string|null, + * entryNumber:string|null, entryDate:string|null, rulesVersion:string, + * shortName:boolean, latinInName:boolean}} + */ +export function evidenceVerdict(input) { + const { + deed, + outsideTr = false, + declarantName, + declaredSeats = [], + declaredEik = false, + firstDeclaredYear = null, + scope = 'self', + nameGloballyUnique = true, + // Fail-CLOSED, unlike `nameGloballyUnique` above. That one's permissive default is bounded — it gates + // only the weakest rung. This one gates the PRIMARY publishing rung, so a caller that forgets to pass + // it must withhold rather than publish a claim naming a real person against a company we did not + // establish. There is exactly one production caller (load.mjs) and it passes it explicitly. + companyNameDistinctive = false, + } = input; + + const tokens = personTokens(declarantName); + const telemetry = { + rulesVersion: RULES_VERSION, + // Counted, not silently dropped: a refusal we cannot see is a recall hole nobody can size. + shortName: tokens.length < MIN_NAME_TOKENS, + latinInName: /[A-Za-z]/.test(String(declarantName ?? '')), + }; + const verdict = (kind, publishable, extra = {}) => ({ + kind, + publishable, + registryRole: null, + matchedFact: null, + entryNumber: null, + entryDate: null, + ...telemetry, + ...extra, + }); + + if (outsideTr) return verdict('outside_tr', false); + if (deed == null) { + // Fail closed and loudly. A missing deed quietly downgraded to „unknown" is indistinguishable + // from a real hold, and hides a cache gap that should stop the run. + throw new Error('evidenceVerdict: deed is required unless outsideTr is set'); + } + + // ── rung 1 ────────────────────────────────────────────────────────────────── + // A union of the numeric code and the ЗТРРЮЛНЦ suffix; either saying joint-stock bars the link, and + // neither able to say means we withhold rather than guess. + const form = registryLegalForm(deed); + if (form.verdict === 'joint_stock') return verdict('bar_joint_stock', false); + if (form.verdict === 'unknown') return verdict('unknown', false); + + // The registered seat, matched against what THIS person declared for THIS company, with R10's temporal + // guard applied. Computed once and consumed by two rungs: rung 3 publishes „Потвърдено" on it, and rung 2 + // uses it as a COMPANY-IDENTITY corroborator. One implementation, because two copies of "what counts as a + // seat match" would eventually disagree about which links may be published. + const matchedSeat = matchDeclaredSeat(deed, declaredSeats, firstDeclaredYear); + + // ── rung 2 ────────────────────────────────────────────────────────────────── + // Only a full three-token name may assert. A Latin homoglyph makes the name a non-match rather than + // a false match — company-name-key.ts's posture, applied to people. + // + // The company gate (ADR-0035). A name match proves someone with these three tokens is registered in the + // company we LOOKED UP — never that this is the company the official declared. `resolveEntity` picks the + // sole WINNER holding the declared name and `nameGloballyUnique` ranges over bidders only, so an official + // whose real company never bid resolves to a same-named winner, and a homonym in that winner's deed + // completes a link false in both halves. Before rung 2 may assert, something other than the фирма must + // say the company is the declared one: + // • the declarant wrote the ЕИК — the national identifier resolves it outright (ADR-0028); or + // • the declared seat matches the registered one — a twin in another town is excluded; or + // • the фирма is distinctive enough that a national twin is improbable in the first place. + // The third is a bound, not a proof, and it is COUNTED (`documentUncorroborated`) so F8 can decide from + // the measured residual whether to tighten to the first two. Strict corroboration was the alternative; + // declared seats exist only on the ООД/ЕООД table of asset declarations, so its recall cost cannot be + // known before that measurement. + const companyCorroborated = declaredEik || matchedSeat != null; + const eligibleForDocument = !telemetry.shortName && !telemetry.latinInName; + if (eligibleForDocument) { + const owner = findPerson(deed, declarantName, OWNERSHIP_FIELDS); + const manager = owner ? null : findPerson(deed, declarantName, [MANAGER_FIELD]); + const hit = owner ?? manager; + if (hit && !companyCorroborated && !companyNameDistinctive) { + // A DISTINCT withholding kind, not a fall-through to `unknown`. „We matched a person but could not + // establish the company" and „we matched nothing" are different facts about a link, and the review + // queue (which is sealed for held links precisely to be reviewable) has to be able to tell them + // apart. It never publishes, and it carries no role or fact — asserting either would leak the very + // claim the rung just refused to make. + return verdict('document_uncorroborated', false); + } + if (owner) { + return verdict('document', true, { + registryRole: 'owner', + matchedFact: `role:owner:${owner.nameCode}`, + entryNumber: owner.entryNumber, + entryDate: owner.entryDate, + }); + } + if (manager) { + return verdict('document', true, { + registryRole: 'manager', + matchedFact: `role:manager:${manager.nameCode}`, + entryNumber: manager.entryNumber, + entryDate: manager.entryDate, + }); + } + } + + // ── rung 3 ────────────────────────────────────────────────────────────────── + // The weakest publishing rung, so it carries the extra AND-gate: a nationally shared company name + // cannot ride it (ADR-0017's holding, carried forward). The stronger „Документ" rung above is not + // gated — the register named this person in THIS company, which makes the name key moot. + // The declared-ЕИК leg is NOT name-gated. ADR-0028: the ЕИК is the identity, not the name, so it + // resolves the company deterministically even behind a nationally shared фирма — which is exactly the + // case ADR-0017 was written about. Gating it on name uniqueness would discard the strongest + // identifier we have precisely where it is most needed. + if (declaredEik) return verdict('confirmed', true, { matchedFact: 'eik' }); + + // The SEAT leg is name-gated, and only this one. ADR-0017's holding carried forward: a name shared by + // two ЕИК cannot support a name-derived identity claim. The seat still rescues a GENERIC name — that + // is the whole point of the rung — it just cannot rescue a NATIONALLY SHARED one. + if (nameGloballyUnique && matchedSeat != null) { + return verdict('confirmed', true, { + matchedFact: `seat:${matchedSeat.settlement}`, + entryDate: matchedSeat.entryDate, + }); + } + + // ── rung 4 ────────────────────────────────────────────────────────────────── + // OWN stakes only. For a family stake the registered owner is the relative, whose name we neither + // store nor check, so absence of the OFFICIAL from the deed is evidence of nothing. An early branch, + // not a caller convention. + if (scope === 'self' && firstDeclaredYear != null) { + const stillPresent = findPerson(deed, declarantName, ROLE_FIELDS); + const latest = latestOwnershipEntryDate(deed); + const inRereg = + latest != null && latest >= REREGISTRATION_START && latest <= REREGISTRATION_END; + if (!stillPresent && latest != null && !inRereg && latest < `${firstDeclaredYear}-01-01`) { + return verdict('refuted', false, { entryDate: latest }); + } + } + + // ── rung 5 ────────────────────────────────────────────────────────────────── + return verdict('unknown', false); +} + +/** + * Reconcile a DECLARED termination against the live deed (#279 §7). + * + * „Terminated" is an inference from silence — the commonest cause is a finished mandate, not a sale — + * so ADR-0021 E11's withdrawal is checked against the register before it takes effect. + * + * Phase 1 uses `terminated` ONLY. `label` is computed but deliberately not rendered: „и към днешна + * дата" asserts a present tense about a named person on evidence whose freshness is bounded by the + * cache refresh cycle, and it is deferred behind an LIA addendum (ADR-0033 decision 4). + * + * @returns {{terminated:boolean, label:'owner_today'|'manager_today'|null}} + */ +export function reconcileTermination({ deed, declarantName, scope = 'self' }) { + // Family first, structurally: there is nothing to look for, and looking would be an attempt to + // identify the relative. + if (scope !== 'self' || deed == null) return { terminated: true, label: null }; + + if (findPerson(deed, declarantName, OWNERSHIP_FIELDS)) { + return { terminated: false, label: 'owner_today' }; + } + if (findPerson(deed, declarantName, [MANAGER_FIELD])) { + return { terminated: true, label: 'manager_today' }; + } + return { terminated: true, label: null }; +} diff --git a/scripts/tr/evidence.test.mjs b/scripts/tr/evidence.test.mjs new file mode 100644 index 00000000..d3f71085 --- /dev/null +++ b/scripts/tr/evidence.test.mjs @@ -0,0 +1,564 @@ +// node:test — the evidence ladder (ADR-0033 decision 1). Pure: deed in, verdict out. +// +// Six outcomes, first match wins. What each rung is allowed to CONCLUDE is the whole subject: +// the registry proves the identity of the COMPANY, never that the official owns it — the ownership +// claim comes from the official's own filed declaration. So a wrong match here does not invent an +// ownership claim, it attaches a real official to the wrong company's ЕИК, contracts and money. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + RULES_VERSION, + evidenceVerdict, + reconcileTermination, + MATCHED_FACT_RE, + isSealedFact, +} from './evidence.mjs'; + +const container = (t) => + `

${t}

`; +const joined = (...ts) => ts.map(container).join(`
`); +const ERASED = `
Заличено обстоятелство.
`; + +const fld = (nameCode, htmlData, entryDate = '2011-05-02T00:00:00') => ({ + nameCode, + htmlData, + fieldEntryNumber: '20110502101007', + fieldEntryDate: entryDate, + fieldOperation: 3, +}); +const deed = (fields, over = {}) => ({ + uic: '201122335', + fullName: '"АЛФА СТРОЙ" ООД', + legalForm: 4, + sections: [{ subDeeds: [{ groups: [{ fields }] }] }], + ...over, +}); + +const OWNER_DEED = deed([ + fld('CR_F_19_L', joined('ИВАН ПЕТРОВ ТЕСТОВ, Държава: БЪЛГАРИЯ', 'МАРИЯ СТОЯНОВА ИВАНОВА')), + fld('CR_F_5_L', container('Държава: БЪЛГАРИЯ
Населено място: гр. Пловдив, п.к. 4000')), +]); + +const base = { + deed: OWNER_DEED, + declarantName: 'Иван Петров Тестов', + declaredSeats: [], + declaredEik: false, + firstDeclaredYear: 2021, + scope: 'self', + nameGloballyUnique: true, + // The company was resolved by a name unlikely to have a national twin. The rung-2 tests below are about + // NAME matching inside a deed, so they hold this dimension fixed; the gate itself is tested separately. + companyNameDistinctive: true, +}; + +test('RULES_VERSION is a stable, non-empty identifier — §8 hangs off it', () => { + assert.equal(typeof RULES_VERSION, 'string'); + assert.ok(RULES_VERSION.length > 0); +}); + +// ── rung 1: the joint-stock bar wins over everything ────────────────────────── +test('rung 1 — a joint-stock company is barred even when the person IS in the deed', () => { + const ad = deed([fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))], { + legalForm: 5, + fullName: '"ГАМА ИНВЕСТ" АД', + }); + const v = evidenceVerdict({ ...base, deed: ad }); + assert.equal(v.kind, 'bar_joint_stock'); + assert.equal(v.publishable, false); +}); + +test('rung 1 — an UNKNOWN legal form withholds; it never falls through to a lower rung', () => { + const odd = deed([fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))], { + legalForm: 99, + fullName: 'НЕЩО БЕЗ ФОРМА', + }); + const v = evidenceVerdict({ ...base, deed: odd }); + assert.equal(v.kind, 'unknown'); + assert.equal(v.publishable, false); +}); + +// ── rung 2: „Документ" ──────────────────────────────────────────────────────── +test('rung 2 — a full-name match in a live ownership field publishes, with the role kept', () => { + const v = evidenceVerdict(base); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); + assert.equal(v.registryRole, 'owner'); + assert.equal(v.matchedFact, 'role:owner:CR_F_19_L'); + assert.equal(v.entryNumber, '20110502101007'); + assert.equal(v.entryDate, '2011-05-02'); +}); + +test('rung 2 — a manager-only match publishes but records the weaker role', () => { + const mgr = deed([ + fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ, Държава: БЪЛГАРИЯ')), + fld('CR_F_19_L', container('ДРУГО ЛИЦЕ ТУК')), + ]); + const v = evidenceVerdict({ ...base, deed: mgr }); + assert.equal(v.kind, 'document'); + assert.equal(v.registryRole, 'manager'); + assert.equal(v.matchedFact, 'role:manager:CR_F_7_L'); +}); + +test('rung 2 — a TWO-token declarant can never earn „Документ"', () => { + // 46 of 301 measured matches were two-token only, which is exactly the homonym risk. Falls to a + // lower rung rather than publishing on a name that half a register could satisfy. + const two = deed([fld('CR_F_19_L', container('ИВАН ТЕСТОВ, Държава: БЪЛГАРИЯ'))]); + const v = evidenceVerdict({ ...base, deed: two, declarantName: 'Иван Тестов' }); + assert.notEqual(v.kind, 'document'); + assert.equal(v.shortName, true, 'the refusal is counted, not silently dropped'); +}); + +test('rung 2 — the match must fall inside ONE entity (the libel guard, end to end)', () => { + const two = deed([ + fld('CR_F_19_L', joined('ПЕНКО НЕСТОРОВ НЕСТОРОВ', 'ИЛИЯН КОСТАДИНОВ ФИЛИПОВ')), + ]); + const v = evidenceVerdict({ ...base, deed: two, declarantName: 'ПЕНКО КОСТАДИНОВ ФИЛИПОВ' }); + assert.notEqual(v.kind, 'document'); +}); + +test('rung 2 — an ERASED ownership entry cannot produce a document match', () => { + const gone = deed([fld('CR_F_23_L', ERASED, '2013-07-16T10:10:07')]); + const v = evidenceVerdict({ ...base, deed: gone }); + assert.notEqual(v.kind, 'document'); +}); + +test('rung 2 — a Latin homoglyph in the name is a NON-match, and is counted', () => { + // company-name-key.ts deliberately does not fold Cyrillic↔Latin; person names take the same posture. + const v = evidenceVerdict({ ...base, declarantName: 'ИBAH ПЕТРОВ ТЕСТОВ' }); // Latin B, A, H + assert.notEqual(v.kind, 'document'); + assert.equal(v.latinInName, true); +}); + +// ── rung 3: „Потвърдено" ────────────────────────────────────────────────────── +test('rung 3 — a declared seat matching the registered seat confirms the company', () => { + const other = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив, п.к. 4000'), '2015-01-01T00:00:00'), + ]); + const v = evidenceVerdict({ ...base, deed: other, declaredSeats: ['Пловдив'] }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.publishable, true); + assert.equal(v.matchedFact, 'seat:ПЛОВДИВ'); +}); + +test('rung 3 — a declared ЕИК confirms the company on its own', () => { + const other = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК'))]); + const v = evidenceVerdict({ ...base, deed: other, declaredEik: true }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'eik'); +}); + +test('rung 3 — an EMPTY declared seat never confirms', () => { + const noSeat = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК'))]); + const v = evidenceVerdict({ ...base, deed: noSeat, declaredSeats: ['', ' '] }); + assert.notEqual(v.kind, 'confirmed'); +}); + +test('rung 3 — a seat registered AFTER the declared period does not confirm', () => { + // R10, and W0 measured that seats move: a company that relocated INTO the declared settlement after + // the fact would otherwise produce a false „Потвърдено". + const moved = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив'), '2024-06-01T00:00:00'), + ]); + const v = evidenceVerdict({ + ...base, + deed: moved, + declaredSeats: ['Пловдив'], + firstDeclaredYear: 2021, + }); + assert.notEqual(v.kind, 'confirmed'); +}); + +test('rung 3 — an UNKNOWN first declared year cannot confirm on a seat', () => { + // R10 again, from the other side. `load.mjs` passes `firstDeclaredYear: null` whenever no history row + // carried a parseable year, and a null year means the temporal check has NOTHING to compare against — + // not that the seat covers the period. The same relocated company as the test above, with the year + // unknown instead of 2021, must reach the same held outcome: an unknown guard is a failed guard. + const moved = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив'), '2024-06-01T00:00:00'), + ]); + const v = evidenceVerdict({ + ...base, + deed: moved, + declaredSeats: ['Пловдив'], + firstDeclaredYear: null, + }); + assert.notEqual(v.kind, 'confirmed'); +}); + +test('rung 3 — an unknown year holds a seat match even when the seat has NO entry date', () => { + // The nastier half: with no entry date on the register side AND no year on the declaration side there + // are two unknowns and zero evidence about the period, yet both legs of the old disjunction read TRUE. + // Rung 4's refutation leg already refuses to run without a year (`firstDeclaredYear != null`); the seat + // leg must refuse on the same ground, or the weakest rung is the one with no temporal check at all. + const undated = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: undated, + declaredSeats: ['Пловдив'], + firstDeclaredYear: null, + }); + assert.notEqual(v.kind, 'confirmed', 'two unknowns must not multiply into a public claim'); +}); + +test('rung 3 — a KNOWN year with an undated seat still confirms (the guard is not a blanket)', () => { + // Positive control. Bounding the null case must not quietly kill the rung: a seat with no entry date + // is the ordinary shape for a company that never moved, and it still confirms under a known year. + const undated = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: undated, + declaredSeats: ['Пловдив'], + firstDeclaredYear: 2021, + }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'seat:ПЛОВДИВ'); +}); + +test('rung 3 — the weakest rung ALSO requires global name uniqueness (ADR-0017 carried forward)', () => { + const other = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: other, + declaredSeats: ['Пловдив'], + nameGloballyUnique: false, + }); + assert.notEqual(v.kind, 'confirmed', 'a nationally shared name cannot ride the weakest rung'); +}); + +test('rung 3 — a declared ЕИК is NOT gated by name uniqueness (ADR-0028)', () => { + // The case ADR-0017 was written about — a фирма backing two ЕИК — is exactly where a declarant-supplied + // ЕИК is most valuable. Gating it on the name would discard the strongest identifier precisely when the + // name is useless. + const other = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК'))]); + const v = evidenceVerdict({ + ...base, + deed: other, + declaredEik: true, + nameGloballyUnique: false, + }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'eik'); +}); + +test('rung 3 — the seat rung DOES rescue a merely generic name; it is uniqueness that gates it', () => { + // The seat leg exists to rescue generic names (a bare one- or two-word фирма). Requiring the name to + // be distinctive would empty the rung of its entire purpose; only NATIONAL non-uniqueness blocks it. + const generic = deed([ + fld('CR_F_19_L', container('НЯКОЙ ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]); + const v = evidenceVerdict({ + ...base, + deed: generic, + declaredSeats: ['Пловдив'], + nameGloballyUnique: true, + }); + assert.equal(v.kind, 'confirmed'); + assert.equal(v.matchedFact, 'seat:ПЛОВДИВ'); +}); + +test('rung 3 — name uniqueness does NOT gate the stronger „Документ" rung', () => { + const v = evidenceVerdict({ ...base, nameGloballyUnique: false }); + assert.equal( + v.kind, + 'document', + 'the registry named the person in THIS company; the name key is moot', + ); +}); + +test('rung 2 — every OWNERSHIP field code can carry the match, not just CR_F_19_L', () => { + // OWNERSHIP_FIELDS is ['CR_F_18_L','CR_F_19_L','CR_F_23_L'] — едноличен собственик, съдружници, and + // ФЛ-търговец. Every owner test used CR_F_19_L, so a typo or a dropped entry in the other two would + // have silently withheld an entire ownership shape: a sole owner (the commonest ЕООД form) publishing + // as „Неизвестна" is a recall hole with no symptom. + for (const code of ['CR_F_18_L', 'CR_F_19_L', 'CR_F_23_L']) { + const d = deed([fld(code, container('ИВАН ПЕТРОВ ТЕСТОВ'))]); + const v = evidenceVerdict({ ...base, deed: d }); + assert.equal(v.kind, 'document', `${code} must carry an ownership match`); + assert.equal(v.registryRole, 'owner', `${code} is an OWNERSHIP field, not management`); + assert.equal(v.matchedFact, `role:owner:${code}`); + } + // POSITIVE CONTROL — a field that is NOT an ownership or manager field must not match at all, or the + // loop above would pass for a reason other than the one it claims. + const other = deed([fld('CR_F_99_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))]); + assert.notEqual(evidenceVerdict({ ...base, deed: other }).kind, 'document'); +}); + +// ── rung 2's company gate: the winner-vs-non-winner homonym (ADR-0035) ──────── +// +// The ladder proves a person with these three tokens is registered in the company we LOOKED UP. It cannot +// prove that company is the one the official declared. `resolveEntity` maps a declared name to the sole +// WINNER holding it, and `nameGloballyUnique` ranges over procurement bidders only — never the whole +// register. So when an official owns a same-named company that never bid, we resolve to the winner, and a +// three-token homonym in the winner's deed „proves" a link that is false in both halves. +// +// Two coincidences, and neither is rare in Bulgaria: a shared фирма and a shared three-part name. The gate +// asks for a reason to believe the COMPANY is the declared one before rung 2 may assert. +test('rung 2 — a GENERIC company name with no corroboration cannot publish on a name match alone', () => { + const v = evidenceVerdict({ ...base, companyNameDistinctive: false }); + assert.equal( + v.kind, + 'document_uncorroborated', + 'the deed names SOMEONE with this name in THIS company — not that this is the declared company', + ); + assert.equal(v.publishable, false); + // It must not fall through to `unknown`: the residual is the input to F8's decision on whether this gate + // tightens, and a rung-2 match withheld for want of company identity is a different fact from no match. + assert.equal(v.registryRole, null); + assert.equal(v.matchedFact, null); +}); + +test('rung 2 — a declared ЕИК corroborates the company, so the name match publishes', () => { + // POSITIVE CONTROL. The ЕИК IS the identity (ЗТРРЮЛНЦ, ADR-0028) — it resolves the company behind any + // shared фирма, which is exactly the collision the gate is about. + const v = evidenceVerdict({ ...base, companyNameDistinctive: false, declaredEik: true }); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); + assert.equal(v.registryRole, 'owner'); +}); + +test('rung 2 — a declared seat matching the registered seat corroborates the company', () => { + // POSITIVE CONTROL. The declarant put this company in гр. Пловдив and the register agrees; a national + // twin in another town is excluded by the same fact rung 3 publishes on. + const v = evidenceVerdict({ + ...base, + companyNameDistinctive: false, + declaredSeats: ['гр. Пловдив'], + }); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); +}); + +test('rung 2 — a DISTINCTIVE company name publishes uncorroborated (the gate is not a blanket)', () => { + // POSITIVE CONTROL, and the one that distinguishes this fix from disabling rung 2. A predicate that + // always withheld would satisfy the bar above; this is what it must NOT do. + const v = evidenceVerdict({ ...base, companyNameDistinctive: true }); + assert.equal(v.kind, 'document'); + assert.equal(v.publishable, true); +}); + +test('rung 2 — a seat that does NOT match cannot corroborate a generic name', () => { + // The corroborator has to actually corroborate. A declared seat in another town is evidence AGAINST the + // company being the declared one, so it certainly cannot rescue the rung. + const v = evidenceVerdict({ + ...base, + companyNameDistinctive: false, + declaredSeats: ['гр. Бургас'], // the deed says Пловдив + }); + assert.equal(v.kind, 'document_uncorroborated'); + assert.equal(v.publishable, false); +}); + +test('rung 2 — the seat corroborator carries the SAME temporal guard as rung 3 (R10)', () => { + // A seat registered after the declared period cannot corroborate anything: the company may have moved + // INTO that town afterwards. Rung 3 already refuses it; rungs 2 and 3 share one implementation so they + // cannot drift into disagreeing about what a seat match means. + const moved = deed([ + fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ')), + fld( + 'CR_F_5_L', + container('Държава: БЪЛГАРИЯ
Населено място: гр. Пловдив, п.к. 4000'), + '2023-07-01T00:00:00', + ), + ]); + const v = evidenceVerdict({ + ...base, + deed: moved, + companyNameDistinctive: false, + declaredSeats: ['гр. Пловдив'], + firstDeclaredYear: 2021, + }); + assert.equal(v.kind, 'document_uncorroborated'); +}); + +test('rung 2 — the company gate never rescues a link rung 1 has barred', () => { + // Ordering: a joint-stock bar outranks everything, corroborated or not. The gate adds a way to WITHHOLD, + // never a way to publish something a stronger rung refused. + const ad = deed([fld('CR_F_19_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))], { + legalForm: 5, + fullName: '"ГАМА ИНВЕСТ" АД', + }); + const v = evidenceVerdict({ ...base, deed: ad, companyNameDistinctive: true, declaredEik: true }); + assert.equal(v.kind, 'bar_joint_stock'); +}); + +// ── rung 4: „Оборена" ───────────────────────────────────────────────────────── +test('rung 4 — absent from a deed whose ownership predates the declaration refutes the link', () => { + const older = deed([ + fld('CR_F_19_L', container('СЪВСЕМ ДРУГ СОБСТВЕНИК'), '2015-03-01T00:00:00'), + ]); + const v = evidenceVerdict({ ...base, deed: older, firstDeclaredYear: 2021 }); + assert.equal(v.kind, 'refuted'); + assert.equal(v.publishable, false); +}); + +test('rung 4 — the comparison is date-to-DATE, not date-to-year', () => { + // R17: „strictly before the first declared year" means before YYYY-01-01. An entry inside the first + // declared year does NOT cover the period and must not refute. + const inYear = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2021-06-15T00:00:00')]); + assert.notEqual( + evidenceVerdict({ ...base, deed: inYear, firstDeclaredYear: 2021 }).kind, + 'refuted', + ); + const justBefore = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2020-12-31T00:00:00')]); + assert.equal( + evidenceVerdict({ ...base, deed: justBefore, firstDeclaredYear: 2021 }).kind, + 'refuted', + ); +}); + +test('rung 4 — NEVER applies to a family stake', () => { + // The owner there is the relative, whose name we neither store nor check (ADR-0010 item 4, + // ADR-0032 decision 2), so „the official is not in the deed" says nothing at all. + const older = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2015-03-01T00:00:00')]); + const v = evidenceVerdict({ ...base, deed: older, scope: 'family', firstDeclaredYear: 2021 }); + assert.notEqual(v.kind, 'refuted'); + assert.equal(v.kind, 'unknown'); +}); + +test('rung 4 — suppressed inside the 2011–2012 re-registration window', () => { + // R13: court-registered companies had every entry date flattened into the re-registration window, + // so „strictly before" certifies nothing there. + const flattened = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2011-11-04T00:00:00')]); + const v = evidenceVerdict({ ...base, deed: flattened, firstDeclaredYear: 2021 }); + assert.notEqual(v.kind, 'refuted'); + assert.equal(v.kind, 'unknown'); +}); + +// ── rungs 5 and 6 ───────────────────────────────────────────────────────────── +test('rung 5 — everything else is „Неизвестна" and stays hidden', () => { + const recent = deed([fld('CR_F_19_L', container('ДРУГ СОБСТВЕНИК'), '2023-01-01T00:00:00')]); + const v = evidenceVerdict({ ...base, deed: recent, firstDeclaredYear: 2021 }); + assert.equal(v.kind, 'unknown'); + assert.equal(v.publishable, false); +}); + +test('rung 6 — outside the register is its own outcome, and is not publishable', () => { + const v = evidenceVerdict({ ...base, deed: null, outsideTr: true }); + assert.equal(v.kind, 'outside_tr'); + assert.equal(v.publishable, false); +}); + +test('a missing deed that is NOT marked outside-ТР is an error, not a silent hold', () => { + // Fail closed: a cache gap must be visible, never quietly downgraded to „unknown". + assert.throws(() => evidenceVerdict({ ...base, deed: null, outsideTr: false }), /deed/i); +}); + +// ── the seal ────────────────────────────────────────────────────────────────── +test('MATCHED_FACT_RE bounds a settlement to two tokens — a NAME cannot wear the seat: prefix', () => { + // The rail tested DIRECTLY, not just through whatever verdicts the ladder happens to produce. Both + // seal tests previously restated this regex locally and got it WRONG in the permissive direction — + // `seat:` followed by unlimited uppercase tokens — so a three-part Bulgarian name (ЗГР чл. 9) wearing + // an allowed prefix passed them. That value is exactly what a mis-split of the seat field produces, + // and it is the one shape this rail exists to keep off a served column. + for (const ok of [ + 'seat:СОФИЯ', + 'seat:ВЕЛИКО ТЪРНОВО', // a real two-token settlement must still pass + 'seat:ГЕНЕРАЛ ТОШЕВО', + 'seat:ЦАР-КАЛОЯН', // hyphenated is one token + 'role:owner:CR_F_19_L', + 'role:manager:CR_F_7_L', + 'role:owner:CR_F_23_L', + 'eik', + ]) + assert.equal(MATCHED_FACT_RE.test(ok), true, `wrongly rejected: ${ok}`); + + for (const bad of [ + 'seat:ИВАН ПЕТРОВ ГЕОРГИЕВ', // THE case: three tokens is a name, not a settlement + 'seat:ИВАН ПЕТРОВ ГЕОРГИЕВ ДРУГ', + 'ИВАН ПЕТРОВ ГЕОРГИЕВ', // a bare name with no prefix at all + 'role:owner:ИВАН ПЕТРОВ', // a name where a field code belongs + 'role:cashier:CR_F_19_L', // a role outside the vocabulary + 'seat:', // an empty settlement asserts nothing + 'eik:201122335', // the ЕИК itself is never stored, only the fact that one matched + ]) + assert.equal(MATCHED_FACT_RE.test(bad), false, `wrongly accepted: ${bad}`); + + // null is legal — a rung may match no fact — and that is isSealedFact's job, not the regex's. + assert.equal(isSealedFact(null), true); + assert.equal(isSealedFact('seat:ИВАН ПЕТРОВ ГЕОРГИЕВ'), false); +}); + +test('matched_fact stays inside the closed vocabulary — it can never carry a name', () => { + // The PRODUCTION predicate, imported — never a local copy of it. A re-stated regex here was looser + // than `MATCHED_FACT_RE` (it allowed `seat:` + unlimited tokens), so this loop certified values the + // real rail rejects and could not fail on the regression it exists to catch (cefothe, #309). + for (const v of [ + evidenceVerdict(base), + evidenceVerdict({ ...base, deed: deed([fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ'))]) }), + evidenceVerdict({ ...base, declaredEik: true }), + evidenceVerdict({ + ...base, + deed: deed([ + fld('CR_F_19_L', container('ДРУГ ЧОВЕК')), + fld('CR_F_5_L', container('Населено място: гр. Пловдив')), + ]), + declaredSeats: ['Пловдив'], + }), + ]) { + if (v.matchedFact == null) continue; + assert.ok(isSealedFact(v.matchedFact), `matched_fact escaped the vocabulary: ${v.matchedFact}`); + assert.ok(!/ИВАН|ПЕТРОВ|ТЕСТОВ/.test(v.matchedFact), 'a NAME reached matched_fact'); + } +}); + +test('every verdict carries the rules version that produced it', () => { + assert.equal(evidenceVerdict(base).rulesVersion, RULES_VERSION); +}); + +// ── §7 reconciliation ───────────────────────────────────────────────────────── +test('reconcileTermination — still a registered owner ⇒ NOT terminated', () => { + const r = reconcileTermination({ + deed: OWNER_DEED, + declarantName: 'Иван Петров Тестов', + scope: 'self', + }); + assert.equal(r.terminated, false); + assert.equal(r.label, 'owner_today'); +}); + +test('reconcileTermination — manager only ⇒ terminated as a stake, but the tie continues', () => { + const mgr = deed([ + fld('CR_F_7_L', container('ИВАН ПЕТРОВ ТЕСТОВ')), + fld('CR_F_19_L', container('ДРУГ')), + ]); + const r = reconcileTermination({ deed: mgr, declarantName: 'Иван Петров Тестов', scope: 'self' }); + assert.equal(r.terminated, true); + assert.equal(r.label, 'manager_today'); +}); + +test('reconcileTermination — absent from the live deed ⇒ the declared termination stands', () => { + const none = deed([fld('CR_F_19_L', container('НЯКОЙ ДРУГ'))]); + const r = reconcileTermination({ + deed: none, + declarantName: 'Иван Петров Тестов', + scope: 'self', + }); + assert.equal(r.terminated, true); + assert.equal(r.label, null); +}); + +test('reconcileTermination — a FAMILY stake is never reconciled, by an early branch', () => { + // Structural, not a caller convention: the relative's name is not stored, so there is nothing to + // look for, and looking would be a de-anonymisation attempt. + const r = reconcileTermination({ + deed: OWNER_DEED, + declarantName: 'Иван Петров Тестов', + scope: 'family', + }); + assert.equal(r.terminated, true); + assert.equal(r.label, null); +}); diff --git a/scripts/tr/fetch-deeds.mjs b/scripts/tr/fetch-deeds.mjs new file mode 100644 index 00000000..cdea4f73 --- /dev/null +++ b/scripts/tr/fetch-deeds.mjs @@ -0,0 +1,296 @@ +// The deed crawler (issue #279 §3, ADR-0033). One request per candidate ЕИК, sequential and paced. +// +// This is the only component in the project that touches a public register at volume, so what it +// REFUSES to do is the substance: +// +// • It never goes faster than 1 request / 3 s, and the flag that sets the pace cannot be used to go +// faster — only slower. Spec §3.3 permits a bounded per-ЕИК lookup and forbids bulk scraping; the +// limiter is the operator's only way to state a preference, and tuning around it empirically is +// what that rule exists to prevent. +// • It never retries a 429. The block is sustained (see client.mjs), so a 429 ends the run with its +// own exit code and records NOTHING about the ЕИК that hit it — that ЕИК is unknown, not absent. +// • It never follows a link out of a deed. The candidate set is closed: whatever the caller passes +// in, nothing more. This is what keeps a bounded lookup from drifting into a crawl. +// • It only writes „outside the register" on a DOCUMENTED negative — measured to be an HTTP 200 +// with an empty body, not the 404 the issue predicts. A 5xx or a timeout is transient, and caching +// it as permanent would turn an outage into data that §8 never revisits. +// +// Resumable by construction: the cache is consulted first, so an interrupted run picks up exactly +// where it stopped and a complete cache costs zero requests. + +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { assertTrScratchIgnored, TR_DB, TR_RAW, safeEik, deedPath } from './paths.mjs'; +import { eikChecksumValid } from './eik.mjs'; +import { deedUrl, politeTrGet, RateLimitError, httpsGet } from './client.mjs'; +import { + openCache, + upsertDeed, + markOutsideTr, + pendingEiks, + purgeExpired, + RETENTION_DAYS, +} from './cache.mjs'; +import { + assertUicEcho, + registryLegalForm, + registrySeat, + latestOwnershipEntryDate, +} from './deed.mjs'; + +/** The documented polite pace: 1 request / 3 s (#279 §3). A floor, never a target to tune down. */ +export const MIN_INTERVAL_MS = 3000; + +/** + * Consecutive unresolved ЕИК before the run gives up rather than keep hammering. + * + * Deliberately small, because the unit is ЕИК and not requests: each unresolved candidate costs up to + * 5 attempts (#279 §3's documented retry budget), so the breaker's real cost is `BREAKER_TRIP × 5` + * requests against an endpoint that is already failing. At 10 that would be ~50 — the exact volume at + * which the register was observed to start returning a sustained 429. At 5 the worst case is ~25, + * which the same observation saw pass without a block. + */ +export const BREAKER_TRIP = 5; +/** Attempts per candidate, per #279 §3. Exported so the breaker's request budget is derivable. */ +export const TRIES_PER_EIK = 5; + +export function parseTrOptions(argv) { + const get = (name, def) => { + const i = argv.indexOf(`--${name}`); + return i >= 0 && argv[i + 1] ? argv[i + 1] : def; + }; + const posInt = (raw, name) => { + const n = Number(raw); + if (!Number.isInteger(n) || n < 1) + throw new Error(`--${name} must be a positive integer, got ${JSON.stringify(raw)}`); + return n; + }; + + const eiksFile = get('eiks-file', ''); + if (!eiksFile) throw new Error('--eiks-file is required (the closed candidate set)'); + + const limitRaw = get('limit', ''); + const intervalRaw = get('min-interval-ms', ''); + const minIntervalMs = intervalRaw ? posInt(intervalRaw, 'min-interval-ms') : MIN_INTERVAL_MS; + // Slower is always allowed; faster is not a knob. Making this un-passable is the point. + if (minIntervalMs < MIN_INTERVAL_MS) { + throw new Error( + `--min-interval-ms may not go below the documented pace of ${MIN_INTERVAL_MS}ms — ` + + `the register's rate limit is a stated preference, not an obstacle to tune around`, + ); + } + const maxAgeRaw = get('max-age-days', ''); + // --max-age-days is FRESHNESS (re-request past it); --retention-days is RETENTION (delete past it). + // They are separate flags because they are separate obligations: refreshing rewrites the personal + // data, only purging removes it. Retention defaults to the ADR's 35 days rather than to „off", so + // the rail holds for an operator who passes neither. + const retentionRaw = get('retention-days', ''); + return { + eiksFile, + limit: limitRaw ? posInt(limitRaw, 'limit') : Infinity, + minIntervalMs, + maxAgeDays: maxAgeRaw ? posInt(maxAgeRaw, 'max-age-days') : null, + retentionDays: retentionRaw ? posInt(retentionRaw, 'retention-days') : RETENTION_DAYS, + }; +} + +/** Read the closed candidate set: one ЕИК per line, blanks and `#` comments ignored. */ +export function readEiksFile(file) { + return fs + .readFileSync(file, 'utf8') + .split('\n') + .map((l) => l.trim()) + .filter((l) => l !== '' && !l.startsWith('#')); +} + +function atomicWrite(file, buf) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const tmp = `${file}.tmp-${process.pid}`; + fs.writeFileSync(tmp, buf); + fs.renameSync(tmp, file); +} + +/** + * Crawl the candidate ЕИК. Returns the intended process exit code, so the decision is testable + * without a global side effect: + * 0 — every candidate resolved (or the run was deliberately bounded by --limit) + * 1 — at least one candidate is unresolved (transient failure, refused deed, breaker tripped) + * 2 — the register rate-limited us; the run stopped and nothing was marked + * + * Every I/O edge is injectable so the whole policy is exercised offline. + */ +export async function run({ + httpGet = httpsGet, + sleep = (ms) => new Promise((r) => setTimeout(r, ms)), + now = () => new Date(), + guard = assertTrScratchIgnored, + dbFile = TR_DB, + rawDir = TR_RAW, + argv = process.argv, +} = {}) { + guard(); + const { eiksFile, limit, minIntervalMs, maxAgeDays, retentionDays } = parseTrOptions(argv); + + const requested = readEiksFile(eiksFile); + // A shape- or checksum-invalid code is dropped BEFORE any request: it cannot name a real company, + // so asking about it would spend the register's budget to learn nothing. + const candidates = []; + let invalid = 0; + for (const raw of requested) { + try { + const eik = safeEik(raw); + if (!eikChecksumValid(eik)) throw new Error('checksum'); + candidates.push(eik); + } catch { + invalid++; + } + } + + const db = openCache(dbFile); + try { + const pending = pendingEiks(db, candidates, { maxAgeDays, now: now() }); + const todo = Number.isFinite(limit) ? pending.slice(0, limit) : pending; + console.log( + `candidates ${candidates.length} · invalid ${invalid} · cached ${candidates.length - pending.length} · to fetch ${todo.length}`, + ); + + let unresolved = 0; + let consecutive = 0; + let first = true; + + for (const eik of todo) { + if (!first) await sleep(minIntervalMs); // pace BETWEEN requests, not before the first + first = false; + + let res; + try { + res = await politeTrGet(deedUrl(eik), { httpGet, sleep, tries: TRIES_PER_EIK }); + } catch (err) { + if (err instanceof RateLimitError) { + // Stop the whole run. Recording anything here would attribute the register's throttle to + // this ЕИК, which is a fact about us, not about the company. + console.error(`${err.message}\nSTOPPING — re-run later; progress so far is cached.`); + return 2; + } + console.error( + ` ${eik}: ${err instanceof Error ? err.message : err} (transient, not cached)`, + ); + unresolved++; + consecutive++; + if (consecutive >= BREAKER_TRIP) { + console.error(`breaker: ${consecutive} consecutive failures — aborting the run`); + return 1; + } + continue; + } + + // ── the documented negatives ──────────────────────────────────────────── + // MEASURED 2026-08-05: an ЕИК that is not a търговец answers **HTTP 200 with a ZERO-BYTE body**, + // not a 404 and not the HTML #279 §3 predicts. Verified on Община София (000696327): empty on + // two consecutive requests, while a real company returned its full 34,398-byte deed in the same + // window — so it is the register's answer, not an outage. + // + // The distinction that keeps R6 honest is the STATUS, not the empty body: an empty body under + // 200 is the register saying „no deed"; an empty body under 5xx is a failure and stays + // transient. Getting this backwards either caches a false negative forever or leaves ~4 ЕИК + // permanently unresolved so the run can never exit 0. + if (res.status === 200 && res.body.length === 0) { + markOutsideTr(db, eik, 'HTTP 200, empty body — no deed in the Търговски регистър', now()); + consecutive = 0; + continue; + } + if (res.status === 404) { + markOutsideTr(db, eik, 'HTTP 404 — not in the Търговски регистър (BULSTAT/ДЗЗД?)', now()); + consecutive = 0; + continue; + } + if (res.status !== 200) { + console.error(` ${eik}: HTTP ${res.status} after retries (transient, not cached)`); + unresolved++; + consecutive++; + if (consecutive >= BREAKER_TRIP) { + console.error(`breaker: ${consecutive} consecutive failures — aborting the run`); + return 1; + } + continue; + } + + // ONE refuse-and-continue block around EVERYTHING derived from the response — the JSON, the UIC + // echo, and the HTML parsing alike. The parsing used to sit outside it, which made the block's + // own promise false: a throw from deed.mjs escaped the loop, escaped run(), and killed the + // process, so one malformed deed ended a crawl that had already spent its paced request budget. + // The decode guard in deed.mjs removes the one throw we know of; this is the rail that holds when + // the next one appears, and the cost of being wrong here is measured in hours of pacing. + try { + const deed = JSON.parse(res.body.toString('utf8')); + // The deed we got back must be the deed we asked for, or every claim derived from it names + // the wrong company (R8). + assertUicEcho(deed, eik); + + // The raw response is the ONLY place names live; it stays under git-ignored scratch. Written + // only after the echo check, so a deed for the wrong company never lands on disk. + atomicWrite(deedPath(eik, rawDir), res.body); + const seat = registrySeat(deed); + const form = registryLegalForm(deed); + upsertDeed(db, { + eik, + status: 'fetched', + httpStatus: 200, + fetchedAt: now().toISOString(), + rawPath: path.relative(rawDir, deedPath(eik, rawDir)), + bodySha256: crypto.createHash('sha256').update(res.body).digest('hex'), + legalFormCode: form.code, + legalFormVerdict: form.verdict, + seatNormalized: seat.settlement || null, + seatEntryDate: seat.entryDate, + latestOwnEntryDate: latestOwnershipEntryDate(deed), + }); + } catch (err) { + console.error(` ${eik}: REFUSED — ${err instanceof Error ? err.message : err}`); + unresolved++; + consecutive++; + continue; + } + consecutive = 0; + } + + if (unresolved > 0) { + console.error(`${unresolved} candidate(s) unresolved — the cache is incomplete`); + return 1; + } + return 0; + } finally { + // The purge step ADR-0033 decision 5 puts „in the same job" — in `finally`, and that placement is + // the point. Retention is an obligation about other people's data, not a reward for a clean run, + // so it must also happen on the paths that leave early: a 429 (exit 2), a tripped breaker, an + // unresolved candidate. Under normal operation it removes nothing, because the monthly refresh + // rewrites each row well inside the window; anything it does delete is residue — a company that + // left the candidate set, or a refresh that never landed. + try { + const purged = purgeExpired(db, rawDir, { retentionDays, now: now() }); + if (purged.rows || purged.files || purged.orphans) { + console.log( + `purged ${purged.rows} row(s), ${purged.files} raw deed(s), ${purged.orphans} orphan(s) past ${retentionDays}d retention`, + ); + } + } catch (e) { + // A failed purge must be loud but must not mask the run's own outcome — especially not a 429, + // whose exit code is what tells the operator to back off. + console.error(`purge failed: ${e.message}`); + } + db.close(); + } +} + +// CLI entry. Kept off the import path so the module stays testable. +if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) { + run() + .then((code) => { + process.exitCode = code; + }) + .catch((err) => { + console.error(err); + process.exitCode = 1; + }); +} diff --git a/scripts/tr/fetch-deeds.test.mjs b/scripts/tr/fetch-deeds.test.mjs new file mode 100644 index 00000000..3a4b1dc0 --- /dev/null +++ b/scripts/tr/fetch-deeds.test.mjs @@ -0,0 +1,427 @@ +// node:test — the deed crawler, driven entirely through injected I/O. No network, no real scratch. +// +// This is the only component that touches a public register at volume, so what it must NOT do is the +// substance: never exceed the pace, never retry a 429, never turn a transient wall into permanent +// data, and never re-request what it already holds. Spec §3.3 permits a bounded per-ЕИК lookup and +// forbids bulk scraping; the difference between the two is enforced here. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { + parseTrOptions, + run, + MIN_INTERVAL_MS, + BREAKER_TRIP, + TRIES_PER_EIK, +} from './fetch-deeds.mjs'; + +const DEED = (uic) => ({ + uic, + fullName: '"АЛФА СТРОЙ" ООД', + legalForm: 4, + sections: [ + { + subDeeds: [ + { + groups: [ + { + fields: [ + { + nameCode: 'CR_F_19_L', + htmlData: `

ИВАН ПЕТРОВ ТЕСТОВ

`, + fieldEntryNumber: '20110502101007', + fieldEntryDate: '2011-05-02T00:00:00', + }, + { + nameCode: 'CR_F_5_L', + htmlData: `

Населено място: гр. Пловдив

`, + fieldEntryNumber: '20110502101008', + fieldEntryDate: '2011-05-02T00:00:00', + }, + ], + }, + ], + }, + ], + }, + ], +}); + +const ok = (uic) => ({ + status: 200, + headers: {}, + body: Buffer.from(JSON.stringify(DEED(uic)), 'utf8'), +}); +const status = (s) => ({ status: s, headers: {}, body: Buffer.from('') }); + +// Two valid ЕИК (real checksums) plus one that is shape-valid but checksum-invalid. +const A = '201122335'; +const B = '203445566'; +const C = '204556676'; +const BAD_CHECKSUM = '201122336'; + +function ctx() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-crawl-')); + return { + dir, + dbFile: path.join(dir, 'tr-cache.sqlite'), + rawDir: path.join(dir, 'deeds'), + cleanup: () => fs.rmSync(dir, { recursive: true, force: true }), + }; +} +const eiksFile = (dir, eiks) => { + const f = path.join(dir, 'eiks.txt'); + fs.writeFileSync(f, eiks.join('\n') + '\n'); + return f; +}; + +/** Drive run() with a recording transport. `routes` maps ЕИК → response (or a function of attempt). */ +function harness(c, eiks, routes, extraArgv = []) { + const calls = []; + const waits = []; + const httpGet = async (url) => { + const eik = url.split('/').pop(); + calls.push(eik); + const r = routes[eik]; + return typeof r === 'function' ? r(calls.filter((e) => e === eik).length) : (r ?? status(404)); + }; + return { + calls, + waits, + promise: run({ + httpGet, + sleep: async (ms) => void waits.push(ms), + now: () => new Date('2026-08-05T12:00:00Z'), + guard: () => {}, + dbFile: c.dbFile, + rawDir: c.rawDir, + argv: ['node', 'fetch-deeds.mjs', '--eiks-file', eiksFile(c.dir, eiks), ...extraArgv], + }), + }; +} + +const openCacheRO = (dbFile) => new DatabaseSync(dbFile); +const rows = (dbFile) => { + const db = openCacheRO(dbFile); + const r = db.prepare('SELECT eik, status, outside_reason FROM deeds ORDER BY eik').all(); + db.close(); + return r; +}; + +// ── options ─────────────────────────────────────────────────────────────────── +test('parseTrOptions: defaults are the documented polite pace', () => { + const o = parseTrOptions(['node', 'x', '--eiks-file', '/tmp/e.txt']); + assert.equal(o.eiksFile, '/tmp/e.txt'); + assert.equal(o.limit, Infinity); + assert.equal(o.minIntervalMs, MIN_INTERVAL_MS); + assert.ok(MIN_INTERVAL_MS >= 3000, 'the documented pace is 1 request / 3 s'); +}); + +test('parseTrOptions: rejects a pace FASTER than the documented one', () => { + // Tuning around a limiter empirically is precisely what spec §3.3 forbids; make it un-passable. + assert.throws( + () => parseTrOptions(['node', 'x', '--eiks-file', '/e', '--min-interval-ms', '100']), + /min-interval/i, + ); +}); + +test('parseTrOptions: --eiks-file is required, and numeric flags are validated', () => { + assert.throws(() => parseTrOptions(['node', 'x']), /eiks-file/i); + for (const bad of ['0', '-1', 'abc', '1.5']) + assert.throws( + () => parseTrOptions(['node', 'x', '--eiks-file', '/e', '--limit', bad]), + /limit/i, + bad, + ); +}); + +// ── pacing ──────────────────────────────────────────────────────────────────── +test('requests are sequential and spaced by at least the documented interval', async () => { + const c = ctx(); + try { + const h = harness(c, [A, B, C], { [A]: ok(A), [B]: ok(B), [C]: ok(C) }); + assert.equal(await h.promise, 0); + assert.deepEqual(h.calls, [A, B, C], 'sequential, in order — never concurrent'); + const paces = h.waits.filter((w) => w >= MIN_INTERVAL_MS); + assert.ok(paces.length >= 2, `expected a pace wait between requests, got ${h.waits.join(',')}`); + } finally { + c.cleanup(); + } +}); + +// ── 429 ─────────────────────────────────────────────────────────────────────── +test('a 429 ends the run with exit 2 and marks NOTHING', async () => { + const c = ctx(); + try { + const h = harness(c, [A, B, C], { [A]: ok(A), [B]: status(429), [C]: ok(C) }); + assert.equal(await h.promise, 2, 'a rate-limit block is its own exit code'); + assert.deepEqual(h.calls, [A, B], 'stops AT the 429 — C is never requested'); + // B must not be recorded at all: it is unknown, not absent, and certainly not outside the register. + assert.deepEqual( + rows(c.dbFile).map((r) => r.eik), + [A], + ); + } finally { + c.cleanup(); + } +}); + +test('after a 429 the run resumes exactly where it stopped', async () => { + const c = ctx(); + try { + const first = harness(c, [A, B, C], { [A]: ok(A), [B]: status(429), [C]: ok(C) }); + assert.equal(await first.promise, 2); + const second = harness(c, [A, B, C], { [A]: ok(A), [B]: ok(B), [C]: ok(C) }); + assert.equal(await second.promise, 0); + assert.deepEqual(second.calls, [B, C], 'A is already cached — not re-requested'); + } finally { + c.cleanup(); + } +}); + +// ── resumability ────────────────────────────────────────────────────────────── +test('a re-run over a complete cache makes ZERO requests', async () => { + const c = ctx(); + try { + assert.equal(await harness(c, [A, B], { [A]: ok(A), [B]: ok(B) }).promise, 0); + const again = harness(c, [A, B], { [A]: ok(A), [B]: ok(B) }); + assert.equal(await again.promise, 0); + assert.deepEqual(again.calls, [], 'nothing to fetch'); + } finally { + c.cleanup(); + } +}); + +// ── permanence ──────────────────────────────────────────────────────────────── +test('an empty 200 is the register saying „no deed" and is cached as outside-ТР', async () => { + // MEASURED against the live API: an ЕИК that is not a търговец (Община София, 000696327) answers + // HTTP 200 with a ZERO-BYTE body — not the 404 or HTML #279 §3 predicts. Reproduced twice, with a + // real company returning its full deed in the same window, so it is an answer and not an outage. + const c = ctx(); + try { + const empty = { status: 200, headers: {}, body: Buffer.alloc(0) }; + assert.equal(await harness(c, [A], { [A]: empty }).promise, 0); + const [row] = rows(c.dbFile); + assert.equal(row.status, 'outside_tr'); + assert.match(row.outside_reason, /empty body/i); + } finally { + c.cleanup(); + } +}); + +test('an empty body under a 5xx stays TRANSIENT — the status decides, not the emptiness', async () => { + // The pair that keeps R6 honest. Both responses have a zero-byte body; only the 200 is an answer. + const c = ctx(); + try { + assert.equal(await harness(c, [A], { [A]: status(503) }).promise, 1); + assert.deepEqual(rows(c.dbFile), [], 'a 5xx must never become permanent „outside ТР"'); + } finally { + c.cleanup(); + } +}); + +test('a 404 is a DOCUMENTED negative and is cached as outside-ТР', async () => { + const c = ctx(); + try { + assert.equal(await harness(c, [A], { [A]: status(404) }).promise, 0); + const [row] = rows(c.dbFile); + assert.equal(row.status, 'outside_tr'); + assert.match(row.outside_reason, /404/); + } finally { + c.cleanup(); + } +}); + +test('a persistent 5xx is TRANSIENT and is never cached as outside-ТР', async () => { + // R6: „outside the register" is permanent by intent. Writing it after a transient wall turns a + // temporary outage into permanent data that §8 never re-examines. + const c = ctx(); + try { + const code = await harness(c, [A], { [A]: status(503) }).promise; + assert.equal(code, 1, 'an unresolved ЕИК makes the run incomplete'); + assert.deepEqual(rows(c.dbFile), [], 'nothing may be recorded from a 5xx'); + } finally { + c.cleanup(); + } +}); + +test('a deed whose UIC does not echo the request is REFUSED, not cached', async () => { + // R8, at the crawl boundary: if anything rewrote the identifier we would be caching one company's + // deed under another company's ЕИК, and every downstream claim about it would name the wrong firm. + const c = ctx(); + try { + const code = await harness(c, [A], { [A]: ok('999999999') }).promise; + assert.equal(code, 1); + assert.deepEqual(rows(c.dbFile), []); + assert.deepEqual(fs.existsSync(c.rawDir) ? fs.readdirSync(c.rawDir) : [], []); + } finally { + c.cleanup(); + } +}); + +test('a checksum-invalid ЕИК is skipped without ever being requested', async () => { + const c = ctx(); + try { + const h = harness(c, [BAD_CHECKSUM, A], { [A]: ok(A) }); + assert.equal(await h.promise, 0); + assert.deepEqual(h.calls, [A], 'the invalid code costs the register nothing'); + } finally { + c.cleanup(); + } +}); + +// ── output ──────────────────────────────────────────────────────────────────── +test('the raw deed is written under the raw dir and the index records only non-PII', async () => { + const c = ctx(); + try { + assert.equal(await harness(c, [A], { [A]: ok(A) }).promise, 0); + assert.deepEqual(fs.readdirSync(c.rawDir), [`${A}.json`]); + const db = openCacheRO(c.dbFile); + const row = db.prepare('SELECT * FROM deeds WHERE eik = ?').get(A); + db.close(); + assert.equal(row.status, 'fetched'); + assert.equal(row.legal_form_verdict, 'closely_held'); + assert.equal(row.seat_normalized, 'ПЛОВДИВ'); + assert.equal(row.body_sha256.length, 64); + // The raw file holds the names; the index must not. + assert.ok(!JSON.stringify(row).includes('ТЕСТОВ')); + assert.ok(fs.readFileSync(path.join(c.rawDir, `${A}.json`), 'utf8').includes('ТЕСТОВ')); + } finally { + c.cleanup(); + } +}); + +test('--limit bounds a run without marking the remainder as anything', async () => { + const c = ctx(); + try { + const h = harness(c, [A, B, C], { [A]: ok(A), [B]: ok(B), [C]: ok(C) }, ['--limit', '2']); + assert.equal(await h.promise, 0, 'a deliberately bounded run is not an incomplete one'); + assert.deepEqual(h.calls, [A, B]); + assert.deepEqual( + rows(c.dbFile).map((r) => r.eik), + [A, B], + ); + } finally { + c.cleanup(); + } +}); + +/** + * N genuinely checksum-valid 9-digit ЕИК. Generated, not hand-written: an earlier version of the + * breaker test below used made-up codes, of which only 1 in 10 was valid — so the crawler dropped + * them all before requesting anything and the assertion passed on zero calls. A test that exercises + * nothing is worse than no test (ADR-0027). + */ +function validEiks(n) { + const control = (p8) => { + const d = [...p8].map(Number); + let s = d.reduce((a, x, i) => a + x * (i + 1), 0) % 11; + if (s === 10) { + s = d.reduce((a, x, i) => a + x * (i + 3), 0) % 11; + if (s === 10) s = 0; + } + return s; + }; + const out = []; + for (let i = 0; out.length < n; i++) { + const p8 = String(20000000 + i); + out.push(p8 + control(p8)); + } + return out; +} + +test('the ЕИК generator used by the breaker test really produces valid codes', () => { + const eiks = validEiks(15); + assert.equal(eiks.length, 15); + assert.equal(new Set(eiks).size, 15); + // Proves the breaker test below actually reaches the network path rather than being filtered out. + assert.deepEqual( + eiks.filter((e) => e.length !== 9), + [], + ); +}); + +test('the circuit breaker aborts a sustained wall of failures', async () => { + // A long run against a broken endpoint must stop hammering, even though each individual 5xx is + // „transient". Distinct from the 429 path: that stops instantly and deliberately. + const c = ctx(); + try { + const many = validEiks(BREAKER_TRIP + 10); + const routes = Object.fromEntries(many.map((e) => [e, status(503)])); + const h = harness(c, many, routes); + assert.equal(await h.promise, 1); + + const attempted = [...new Set(h.calls)]; + assert.equal(attempted.length, BREAKER_TRIP, 'the breaker cuts the run at its threshold'); + assert.ok(attempted.length < many.length, 'and therefore short of the full candidate set'); + // The number that actually matters is REQUESTS, not candidates: each unresolved ЕИК costs the + // full retry budget, so the breaker's real cost is BREAKER_TRIP × TRIES_PER_EIK. Keep that under + // the ~50 at which the register was observed to start returning a sustained 429 — otherwise the + // safety mechanism is itself what trips the block. + assert.equal(h.calls.length, BREAKER_TRIP * TRIES_PER_EIK); + assert.ok( + h.calls.length <= 25, + `a failing run must not spend ${h.calls.length} requests before giving up`, + ); + assert.deepEqual(rows(c.dbFile), [], 'a wall of 5xx records nothing'); + } finally { + c.cleanup(); + } +}); + +// ── retention (ADR-0033 decision 5: „a purge step in the same job") ─────────── +test('the job purges past-retention deeds, and does so even when a 429 ends the run', async () => { + // Retention is an obligation about other people's data, not a reward for a clean run: the paths that + // leave early — a 429, a tripped breaker — are exactly the ones where a naive placement after the + // fetch loop would skip it and let third-party names sit on disk indefinitely. + for (const [label, route] of [ + ['clean run', ok(A)], + ['429 stops the run', status(429)], + ]) { + const c = ctx(); + try { + // An old deed with its raw file — well past the 35-day window at the harness's fixed clock. + fs.mkdirSync(c.rawDir, { recursive: true }); + fs.writeFileSync(path.join(c.rawDir, `${C}.json`), '{"owner":"ТРЕТО ЛИЦЕ"}'); + const db = new DatabaseSync(c.dbFile); + db.exec(`CREATE TABLE IF NOT EXISTS deeds ( + eik TEXT PRIMARY KEY, status TEXT NOT NULL, http_status INTEGER, fetched_at TEXT NOT NULL, + raw_path TEXT, body_sha256 TEXT, legal_form_code INTEGER, legal_form_verdict TEXT, + seat_normalized TEXT, seat_entry_date TEXT, latest_own_entry_date TEXT, + attempts INTEGER NOT NULL DEFAULT 1, outside_reason TEXT); + INSERT INTO deeds (eik,status,fetched_at,raw_path) VALUES ('${C}','fetched','2026-01-01T00:00:00Z','${C}.json');`); + db.close(); + + await harness(c, [A], { [A]: route }).promise; + + assert.equal( + fs.existsSync(path.join(c.rawDir, `${C}.json`)), + false, + `${label}: the past-retention raw deed must be deleted`, + ); + assert.equal( + rows(c.dbFile).some((r) => r.eik === C), + false, + `${label}: its index row must go with it`, + ); + } finally { + c.cleanup(); + } + } +}); + +test('the purge leaves in-window deeds alone — it is a privacy rail, not a cache eviction', async () => { + // If it evicted live cache, every run would re-request deeds it already holds, which is precisely + // the volume against the register the pacing exists to avoid. + const c = ctx(); + try { + await harness(c, [A], { [A]: ok(A) }).promise; + assert.equal(rows(c.dbFile).length, 1, 'the deed just fetched must survive its own job'); + assert.equal(fs.existsSync(path.join(c.rawDir, `${A}.json`)), true); + } finally { + c.cleanup(); + } +}); diff --git a/scripts/tr/paths.mjs b/scripts/tr/paths.mjs new file mode 100644 index 00000000..e325d293 --- /dev/null +++ b/scripts/tr/paths.mjs @@ -0,0 +1,54 @@ +// Paths and path sanitizers for the Търговски регистър leg (issue #279, ADR-0033). +// +// Everything this leg writes lives under scratch/tr/, git-ignored, behind the same refuse-to-run rail +// the CACBG crawl uses — a deed carries third-party personal data (owner and manager names, the +// company's street address), so it is ADR-0010 decision 6 territory, extended by ADR-0033 to a second +// source with a stated retention. +// +// scratch/tr/deeds/.json raw response, atomic write +// scratch/tr/tr-cache.sqlite the index — ЕИК, dates, codes, verdicts. NO names. +// +// The constants below are only DEFAULTS for the CLI. Every function that touches the filesystem takes +// its path explicitly, so tests drive temp directories without mutating process state. + +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { assertScratchIgnored } from '../cacbg/guard.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); + +export const TR_SCRATCH = path.join(ROOT, 'scratch', 'tr'); +export const TR_RAW = path.join(TR_SCRATCH, 'deeds'); +export const TR_DB = path.join(TR_SCRATCH, 'tr-cache.sqlite'); + +/** Refuse to run unless scratch/tr is git-ignored. Call before the first fetch. */ +export function assertTrScratchIgnored() { + assertScratchIgnored('tr'); +} + +// A bare ЕИК: 9 or 13 digits, nothing else. Deliberately NOT `path.basename`-normalised — an ЕИК that +// needed normalising did not come from where we think it did, and silently repairing it is how you end +// up fetching a different company's deed. +const EIK_SHAPE = /^(?:\d{9}|\d{13})$/; + +/** + * Sanitize an ЕИК before it becomes a path segment or a URL segment. + * + * Returns the value VERBATIM — as a string, always. Bulgarian public bodies carry codes of exactly the + * `000…` shape, so a numeric round-trip anywhere on this path silently rewrites the identifier and the + * crawler fetches somebody else's deed (R8). + * + * Shape only. Whether the code's CHECKSUM is valid is a different question with a different remedy, + * answered by eik.mjs — conflating the two would report a real-but-invalid code as a path attack. + * @param {unknown} eik @returns {string} + */ +export function safeEik(eik) { + const s = String(eik ?? ''); + if (!EIK_SHAPE.test(s)) throw new Error(`unsafe ЕИК: ${JSON.stringify(eik)}`); + return s; +} + +/** Absolute path of the cached raw deed for an ЕИК, under `rawDir` (default TR_RAW). */ +export function deedPath(eik, rawDir = TR_RAW) { + return path.join(rawDir, `${safeEik(eik)}.json`); +} diff --git a/scripts/tr/paths.test.mjs b/scripts/tr/paths.test.mjs new file mode 100644 index 00000000..3401260d --- /dev/null +++ b/scripts/tr/paths.test.mjs @@ -0,0 +1,64 @@ +// node:test — path sanitizers and the refuse-to-run rail for the Trade Register leg. +// +// The deed cache holds third-party personal data (owner/manager names, company addresses), so the +// same rail the CACBG crawl runs behind applies here: everything is written under scratch/, and +// scratch/ must be git-ignored, asserted BEFORE any fetch. ADR-0010 decision 6 as extended by ADR-0033. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { TR_SCRATCH, TR_RAW, TR_DB, safeEik, assertTrScratchIgnored } from './paths.mjs'; +import { assertScratchIgnored } from '../cacbg/guard.mjs'; + +test('the TR scratch tree sits under scratch/ and is git-ignored in this repo', () => { + assert.ok(TR_SCRATCH.split(path.sep).includes('scratch'), TR_SCRATCH); + assert.ok(TR_RAW.startsWith(TR_SCRATCH)); + assert.ok(TR_DB.startsWith(TR_SCRATCH)); + assert.doesNotThrow(() => assertTrScratchIgnored()); +}); + +test('the guard is the CACBG one generalized, not a second copy', () => { + // A duplicated safety rail drifts from the original. assertScratchIgnored now takes the + // subdirectory, and its existing no-argument callers keep working unchanged. + assert.doesNotThrow(() => assertScratchIgnored()); + assert.doesNotThrow(() => assertScratchIgnored('tr')); + // .gitignore ignores `scratch/` WHOLESALE, so every subdirectory of it passes — an unknown name is + // not a way to reach the failure branch. Escape scratch/ instead (path.join normalises this to + // `docs/.probe`, which is tracked) to prove the guard still refuses when the target is not ignored. + assert.throws(() => assertScratchIgnored(path.join('..', 'docs')), /REFUSE TO RUN/); +}); + +test('safeEik accepts only a bare 9/13-digit code and returns it verbatim', () => { + assert.equal(safeEik('115536179'), '115536179'); + assert.equal(safeEik('1155361790001'), '1155361790001'); + // Leading zeros survive — public bodies are exactly this shape, and losing them fetches a + // DIFFERENT company's deed. + assert.equal(safeEik('000696327'), '000696327'); +}); + +test('safeEik refuses anything that could leave the intended path or URL', () => { + for (const bad of [ + '', + null, + undefined, + '..', + '../115536179', + '/115536179', + '115536179/../x', + '115536179?x=1', + '115536179#f', + '11553617x', + '11553617', + '1155361790', + 'ЕИК 115536179', + ' 115536179', + '115536179 ', + ]) { + assert.throws(() => safeEik(bad), /unsafe/i, JSON.stringify(bad)); + } +}); + +test('safeEik does not validate the CHECKSUM — that is a separate question', () => { + // Path safety and identity validity are different concerns: a shape-valid but checksum-invalid code + // must still be rejectable by the caller with a specific reason, not conflated into „unsafe path". + assert.equal(safeEik('115536170'), '115536170'); +}); diff --git a/scripts/work-staging-schema.sql b/scripts/work-staging-schema.sql index 9e85b00d..0e9a581d 100644 --- a/scripts/work-staging-schema.sql +++ b/scripts/work-staging-schema.sql @@ -159,7 +159,9 @@ CREATE TABLE raw_amendments ( fetched_at TEXT NOT NULL, seq_no TEXT, document_number TEXT, - contract_number TEXT, -- ← link to raw_contracts + contract_number TEXT, -- ← link to raw_contracts (rewritten in place by the #306 value resolver) + contract_number_raw TEXT, -- #306: the annex-side number before the value resolver rewrote it (provenance; NULL = never rewritten) + link_method TEXT, -- #306: 'value_anchor' when the resolver linked this row by value; NULL = matched by number / unlinked contract_date TEXT, published_at TEXT, -- amendment publication date (ordering key) unp TEXT, -- ← link to raw_contracts @@ -174,6 +176,13 @@ CREATE TABLE raw_amendments ( value_before REAL, -- Стойност преди изменението value_after REAL, -- Стойност след изменението → current_value value_delta REAL, -- Изменение на стойността + -- #305 Tier-2 text-based value correction (computed in TS ingest, packages/ingest/src/amendment-total.ts): + -- value_treatment labels how the основание text reads value_delta ('total_restated' / 'unchanged_restated' + -- / 'genuine_increment', NULL when no signal); value_after_restated is the corrected (true) total when a + -- double-count was confirmed, else NULL. Derive/normalize use COALESCE(value_after_restated, value_after) + -- as the effective after, and skip the arithmetic annex_total_suspect flag when value_treatment IS NOT NULL. + value_treatment TEXT, + value_after_restated REAL, currency TEXT, description TEXT, -- Описание на измененията reason TEXT, -- Причини за изменение (ЗОП основание) diff --git a/tsconfig.base.json b/tsconfig.base.json index 939300bc..bcaee8ac 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -17,6 +17,7 @@ "forceConsistentCasingInFileNames": true, "declaration": false, "sourceMap": true, - "noEmit": true + "noEmit": true, + "allowImportingTsExtensions": true } }