From f2f703d2e870270698d4e0345a58bfa98fdb3319 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Sat, 15 Aug 2026 06:58:43 -0400 Subject: [PATCH 1/2] ci: fail-slow one-sweep CI (PR-0) --- .github/workflows/ci.yml | 204 +++++++++++++++---- docs/documentation-index.json | 10 + docs/documentation-manifest.seed.json | 10 + package.json | 2 +- scripts/check-ci-preflight.mjs | 256 +++++++++++++++--------- scripts/check-ci-preflight.test.mjs | 183 +++++++++-------- scripts/ci-collect-failures.mjs | 22 ++ scripts/lib/ci-workflow-executables.mjs | 7 +- scripts/verify.mjs | 7 + tools/ci/cargo-test-runner.sh | 82 ++++++++ tools/ci/cargo-test-runner.test.mjs | 113 +++++++++++ tools/ci/cargo_needs_postgres.sh | 47 ++--- 12 files changed, 692 insertions(+), 251 deletions(-) create mode 100644 scripts/ci-collect-failures.mjs create mode 100755 tools/ci/cargo-test-runner.sh create mode 100644 tools/ci/cargo-test-runner.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd3dd58e8..2c8331316 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,13 +34,15 @@ jobs: steps: - name: Checkout + id: checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false fetch-depth: 0 - name: Derive exact console C/T/M train - if: ${{ github.event_name == 'pull_request' }} + id: derive + if: ${{ !cancelled() && steps.checkout.outcome == 'success' && github.event_name == 'pull_request' }} shell: bash run: | set -euo pipefail @@ -55,16 +57,21 @@ jobs: } >> "$GITHUB_ENV" - name: Set up Node.js + id: setup-node + if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: "24.16.0" cache: npm - name: Install workspace dependencies + id: npm-ci + if: ${{ !cancelled() && steps.setup-node.outcome == 'success' }} run: npm ci - name: Classify path class id: path_class + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} shell: bash env: PATH_CLASS_EVENT_NAME: ${{ github.event_name }} @@ -79,26 +86,35 @@ jobs: # DotSlash + Rust follow classification so docs-only can skip them without # reading an empty steps.path_class output (GHA evaluates if before later steps). - name: Install pinned DotSlash runtime - if: ${{ steps.path_class.outputs.run_heavy == 'true' }} + id: dotslash + if: ${{ !cancelled() && steps.checkout.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }} run: tools/buck/install_dotslash.sh - name: Install Rust toolchain for Cargo.lock consistency - if: ${{ steps.path_class.outputs.run_heavy == 'true' }} + id: rust-toolchain + if: ${{ !cancelled() && steps.checkout.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }} uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: toolchain: "1.97.1" - name: Cheap Buck2 generated-face admission - if: ${{ steps.path_class.outputs.run_heavy == 'true' }} + id: buck-admission + if: ${{ !cancelled() && steps.dotslash.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }} run: tools/buck/preflight.sh - name: Foundation gate contract + id: foundation-gates + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: npm run check:foundation-gates - name: Reasoning lens contract regression + id: reasoning-lens-regression + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: node --test scripts/check-reasoning-lens-contract.test.mjs - name: Reasoning lens changed-record admission + id: reasoning-lens-admission + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} shell: bash env: REASONING_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} @@ -140,56 +156,83 @@ jobs: esac - name: Console truth-ledger exact-M admission - if: ${{ github.event_name == 'pull_request' }} + id: truth-ledger-admission + if: ${{ !cancelled() && steps.derive.outcome == 'success' && steps.npm-ci.outcome == 'success' && github.event_name == 'pull_request' }} run: npm run check:console-truth-ledger - name: Console fanout planner exact-M admission - if: ${{ github.event_name == 'pull_request' }} + id: fanout-admission + if: ${{ !cancelled() && steps.derive.outcome == 'success' && steps.npm-ci.outcome == 'success' && github.event_name == 'pull_request' }} run: node scripts/console/plan-fanout.mjs --candidate "$CONSOLE_CANDIDATE_SHA" --authority-tip "$CONSOLE_AUTHORITY_TIP_SHA" --synthetic-merge "$CONSOLE_SYNTHETIC_MERGE_SHA" - name: CI preflight contract tests + id: ci-preflight-tests + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: node --test scripts/check-ci-preflight.test.mjs - name: Console route inventory regression + id: route-inventory + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: node --test scripts/console/route-inventory.test.mjs - name: Console authority-train regression + id: authority-train + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: node --test scripts/console/verify-console-authority-train.test.mjs - name: Console lane-receipt validator regression + id: lane-receipt + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: npm run test:lane-receipt - name: Console PR authority bootstrap regression + id: pr-authority-bootstrap + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: node --test scripts/console/verify-console-pr-authority-bootstrap.test.mjs scripts/console/release-please-bot-candidate.test.mjs - name: Executed-tests baseline set regression + id: executed-tests-baseline + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: npm run test:executed-tests-baseline - name: Local CI mirror contract + id: ci-mirror + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: node --test scripts/verify.test.mjs - name: Console truth-ledger validator exact-M regression + id: truth-ledger-validator + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: node --test scripts/console/validate-console-truth-ledger.test.mjs - name: Console fanout planner exact-M regression + id: fanout-planner + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: node --test scripts/console/plan-fanout.test.mjs - name: Buck PostgreSQL environment wrapper regression - if: ${{ steps.path_class.outputs.run_heavy == 'true' }} + id: buck-pg-env + if: ${{ !cancelled() && steps.dotslash.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }} run: tools/buck/run_test_with_postgres_env.test.sh - name: Buck disposable PostgreSQL harness regression - if: ${{ steps.path_class.outputs.run_heavy == 'true' }} + id: buck-pg-harness + if: ${{ !cancelled() && steps.dotslash.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }} run: tools/buck/test_needs_postgres.test.sh - name: CI preflight contract + id: ci-preflight-contract + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: npm run check:ci-preflight - name: Canonical npm lockfile + id: package-lock + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: npm run check:package-lock - name: Cargo.lock consistency - if: ${{ steps.path_class.outputs.run_heavy == 'true' }} + id: cargo-lock + if: ${{ !cancelled() && steps.rust-toolchain.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }} run: cargo metadata --manifest-path backend/Cargo.toml --locked --format-version=1 >/dev/null # Moved here from repo-gates, which has npm and no Rust: the ratchet now @@ -197,18 +240,33 @@ jobs: # where cargo does. The step above it is `cargo metadata` on the same # manifest, and preflight is the only job carrying both toolchains. - name: Executed-tests ratchet — a test binary must have a path from a workflow step - if: ${{ steps.path_class.outputs.run_heavy == 'true' }} + id: executed-tests-ratchet + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }} run: npm run check:executed-tests - name: JavaScript test reachability ratchet + id: js-reachability-ratchet + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: npm run check:js-test-reachability - name: JavaScript test reachability unit tests + id: js-reachability-tests + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: npm run test:js-test-reachability - name: Lane fan-out harness preflight + id: lane-fanout + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: node .claude/workflows/lane-fanout.test.mjs - name: Workflow test-runner credential literals + id: test-credentials + if: ${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: npm run check:test-credentials + - name: Collect failures + if: ${{ !cancelled() }} + env: + CI_STEPS: ${{ toJSON(steps) }} + run: node scripts/ci-collect-failures.mjs + domain-unit: name: Domain crates — unit tests runs-on: ubuntu-latest @@ -295,6 +353,18 @@ jobs: # crate the same measurement exposed, platform/db, is 9 #[sqlx::test] and # needs a database, so it is named in executed-tests-baseline.json instead. run: | + # ci-keep-going: run every invocation below; failures are collected and the + # summary at the end exits 1 if any failed, so these cargo runs still gate the + # job (scripts/lib/ci-workflow-executables.mjs attributes them accordingly). + set +e + failed_count=0 + check_status() { + local rc=$? + if [[ $rc -ne 0 ]]; then + echo "FAILED (exit $rc): $1" >&2 + failed_count=$((failed_count + 1)) + fi + } SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml --lib \ -p console-support-domain -p console-payroll-domain -p console-payroll-adapter-postgres \ -p console-attendance-application -p console-compliance-domain -p console-governance-domain \ @@ -332,24 +402,32 @@ jobs: -p console-logistics-rest -p console-ontology-canonical-domain \ -p console-ontology-canonical-adapter-postgres -p console-orgchange-adapter-postgres \ -p console-gate-writer-ownership -p console-identity-adapter-postgres + check_status "domain --lib sweep" # Doc tests, same step so no new step name has to be declared anywhere. # `compile_fail` doctests are the only artifact that can hold a NEGATIVE # type-boundary claim ("this must not compile"), and they execute nowhere # unless --doc is invoked explicitly. SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ --doc -p console-kernel-core + check_status "domain --doc" SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-attendance-application --test attendance_policy + check_status "attendance_policy" SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-compliance-domain --test location_consent_fsm --test location_ping_policy + check_status "location_consent_fsm + location_ping_policy" SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-platform-authz --test cedar_pbac_readiness_cases --test cedar_pbac_legacy_only_observe_and_record + check_status "cedar_pbac" SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-attendance-domain --test range_and_history + check_status "range_and_history" SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-contracts --test compose + check_status "compose" SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-todos-rest --test openapi_fragment + check_status "openapi_fragment" # openapi.yaml is generated from per-face console-contracts Fragments. # Regeneration must be a no-op on a clean tree; a hand-edit that skips # the owning face slice fails this diff. The generator itself refuses @@ -357,7 +435,9 @@ jobs: # registry cannot pass by rewriting nothing. SQLX_OFFLINE=true cargo run --locked --manifest-path backend/Cargo.toml \ -p console-contracts --bin console-openapi-gen + check_status "openapi regen" git diff --exit-code -- backend/openapi/openapi.yaml + check_status "openapi diff" # ONE -p PER INVOCATION whenever --test is used. Two packages and two --test names on # a single line is ambiguous to any reader that pairs them positionally, and # check-executed-tests.mjs pairs them as a CROSS PRODUCT — 2x2 candidates where only @@ -367,22 +447,30 @@ jobs: # the source rather than parsed around. SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-financial-domain --test quote_and_residual + check_status "quote_and_residual" SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-registry-domain --test equipment + check_status "equipment" SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-messenger-domain --test mentions --test object_code_refs --test parity --test thread_kind + check_status "messenger" SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-workorder-domain --test approval_and_assignment --test serde_roundtrips \ --test settlement_fsm --test workorder_fsm + check_status "workorder" SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-platform-auth --test jwt_es256 --test jwt_verifier --test well_known + check_status "jwt" SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-platform-excel --test template_fidelity --test template_fill_engine + check_status "excel" SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-platform-realtime --test hub --test notify_payload + check_status "realtime" SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-app --test config --test dev_seed_notification_links --test openslo_files \ --test well_known --test workbench_api + check_status "app" # The writer-ownership gate's mutation suite. It lives here and not in # the backend job's Buck2 mutation-suite step because 3 of its 41 tests # read THIS repository -- the whole backend crate tree, and @@ -390,6 +478,12 @@ jobs: # materialize. A real checkout is the requirement, and this job has one. SQLX_OFFLINE=true cargo test --locked --manifest-path backend/Cargo.toml \ -p console-gate-writer-ownership --test gate_detects_violation + check_status "gate_detects_violation" + if [[ $failed_count -ne 0 ]]; then + echo "domain-unit: $failed_count invocation(s) failed" >&2 + exit 1 + fi + echo "domain-unit: all invocations passed" # PostgreSQL reachability facets (S1 + S2 domain split). Package partition via --shard-id. # Do NOT put the load-bearing branch-protection display name on these jobs @@ -788,11 +882,13 @@ jobs: steps: - name: Checkout + id: checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false - name: Path-class skip proof + id: skip-proof if: ${{ needs.preflight.outputs.run_heavy != 'true' }} shell: bash run: | @@ -800,15 +896,18 @@ jobs: printf 'path-class skip proof: %s not required for class=%s\n' "${GITHUB_JOB}" "${{ needs.preflight.outputs.path_class }}" - name: Install pinned DotSlash runtime - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: dotslash + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: ../tools/buck/install_dotslash.sh - name: Free runner disk for Rust backend - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: free-disk + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} uses: ./.github/actions/free-runner-disk - name: Install Rust toolchain (pinned via rust-toolchain.toml) - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: rust + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: # rust-toolchain.toml drives the exact version; this step just @@ -817,7 +916,8 @@ jobs: components: rustfmt, clippy - name: Cache Rust dependencies + build artifacts - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: rust-cache + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} # rust-cache keys on Cargo.lock + rustc + job, caches registry/git deps # and prunes stale crates from target/ (so the cache does not grow # unbounded on every Cargo.lock change the way a raw target/ cache does). @@ -836,46 +936,56 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - name: rustfmt check - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: fmt + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: cargo fmt --all -- --check - name: clippy -D warnings - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: clippy + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: SQLX_OFFLINE=true cargo clippy --all-targets -- -D warnings # Wave C (DN-0006): first Required-CI gate family on Buck2. Invoked from # backend/ so the binary's cwd remains the Cargo workspace root it scans; # Buck2 walks up to the repo .buckconfig. Keep warm-cache overlays opt-in # (fail-closed) — do not attach --config-file infra/ci/buckconfig/warm-*. - name: Layer-boundary gate - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: layer-boundary + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: ../tools/buck2 run //backend/ci/gates/layer-boundary:console-gate-layer-boundary - name: Audit-coverage gate - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: audit-coverage + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: cargo run -p console-gate-audit-coverage - name: Migration-safety gate - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: migration-safety + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: cargo run -p console-gate-migration-safety - name: Tenant-isolation gate - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: tenant-isolation + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: cargo run -p console-gate-tenant-isolation - name: PII-no-logs gate - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: pii-no-logs + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: cargo run -p console-gate-pii-no-logs - name: RLS-arming gate - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: rls-arming + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: cargo run -p console-gate-rls-arming - name: Dev-auth-absence gate - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: dev-auth-absence + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: cargo run -p console-gate-dev-auth-absence - name: IaC tier-discipline gate - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: iac-tier + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: cargo run -p console-gate-iac-tier # An authorization branch must come from the RESOURCE. Deriving it from @@ -892,11 +1002,13 @@ jobs: # named in the module doc and in docs/CI-GATES.md; the control that closes # them is a `ResourceBranch` newtype, queued separately. - name: Fabricated-branch gate - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: fabricated-branch + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: cargo run -p console-gate-fabricated-branch - name: Personal-data-classification gate - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: personal-data-classification + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: cargo run -p console-gate-personal-data-classification # The STATIC half of writer ownership: at most one production crate may @@ -907,7 +1019,8 @@ jobs: # through the disposable-PostgreSQL cargo harness. Two residual shapes are # pinned by `known_residual_` tests; bead console-tai.1 carries the total fix. - name: Writer-ownership gate - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: writer-ownership + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: cargo run -p console-gate-writer-ownership # The ten remaining `cargo run -p console-gate-*` steps above (plus the @@ -938,7 +1051,8 @@ jobs: # Its database half runs in the PostgreSQL reachability facets via # tools/ci/postgres-cargo-map.json. - name: Buck2 CI-gate mutation suites — every gate proven to still reject - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: gate-mutation-suites + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} working-directory: . run: | env -u DATABASE_URL tools/buck2 test \ @@ -956,12 +1070,14 @@ jobs: # migration, using the distinct service administrator only for this # cluster-scoped bootstrap. - name: PR 473 migration operational contract tests - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: pr473-contract-tests + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} working-directory: . run: python3 scripts/check-pr473-migration-operational.test.py -v - name: Reconcile portable PostgreSQL role topology - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: topology + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} run: | APP_PASSWORD="$(openssl rand -hex 32)" RT_PASSWORD="$(openssl rand -hex 32)" @@ -1023,7 +1139,8 @@ jobs: } >> "$GITHUB_ENV" - name: PR 473 migration operational gate - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: pr473-gate + if: ${{ !cancelled() && steps.topology.outcome == 'success' && needs.preflight.outputs.run_heavy == 'true' }} working-directory: . # SQLX_OFFLINE=true so the query! macros compile against the committed # .sqlx cache (the documented contract — see crates/platform/db/src/lib.rs) @@ -1046,7 +1163,8 @@ jobs: # image-release.yml's release-probe job uses to boot the shipped image; # a throwaway keypair is fine, this only proves the app comes up. - name: Boot smoke — migrate + serve + /readyz - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: boot-smoke + if: ${{ !cancelled() && steps.topology.outcome == 'success' && needs.preflight.outputs.run_heavy == 'true' }} run: | set -e openssl ecparam -name prime256v1 -genkey -noout | openssl pkcs8 -topk8 -nocrypt -out /tmp/smoke-priv.pem @@ -1116,7 +1234,8 @@ jobs: fi - name: Buck2 dev-auth feature PostgreSQL suites - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: dev-auth-suites + if: ${{ !cancelled() && steps.topology.outcome == 'success' && needs.preflight.outputs.run_heavy == 'true' }} working-directory: . # `dev-auth` is NOT in default features (proven above). These exact # feature-sensitive auth-rest and provisioning binaries run through @@ -1143,12 +1262,14 @@ jobs: # only evidence that widening `is_stable_key` for dots did not widen the # Cedar injection boundary on any other axis. - name: Buck2 platform-authz unit suite - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: authz-unit + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} working-directory: . run: env -u DATABASE_URL tools/buck2 test //backend/crates/platform/authz:console-platform-authz-unit - name: Buck2 console-app unit suite - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: app-unit + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} working-directory: . run: env -u DATABASE_URL tools/buck2 test //backend/app:console-app-unit @@ -1160,18 +1281,27 @@ jobs: # regexes /api/platform/ paths). The byte-compare of the served document # against the file was deleted: include_str! made it tautological. - name: Buck2 console-app OpenAPI drift suite - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: openapi-drift + if: ${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }} working-directory: . run: env -u DATABASE_URL tools/buck2 test //backend/app:console-app-itest-openapi_drift - name: Buck2 console-app inline PostgreSQL suites - if: ${{ needs.preflight.outputs.run_heavy == 'true' }} + id: app-inline-pg + if: ${{ !cancelled() && steps.topology.outcome == 'success' && needs.preflight.outputs.run_heavy == 'true' }} working-directory: . run: | tools/buck/test_needs_postgres.sh --num-threads=1 \ //tools/buck:app-inline-postgres \ //tools/buck:app-dev-auth-persona-guard-postgres + - name: Collect failures + if: ${{ !cancelled() }} + env: + CI_STEPS: ${{ toJSON(steps) }} + working-directory: . + run: node scripts/ci-collect-failures.mjs + dev-up-smoke: name: dev-up.mjs smoke — compose deps + migrate + /readyz runs-on: ubuntu-latest diff --git a/docs/documentation-index.json b/docs/documentation-index.json index 48a3424f6..7b44e64f4 100644 --- a/docs/documentation-index.json +++ b/docs/documentation-index.json @@ -3903,6 +3903,16 @@ "blob_sha": "17b866de92a67baec26cc671f50ba95fa21c1a9b", "archive_tag": null }, + { + "path": "docs/program/ledger/2026-08-15-ci-fail-slow-sweep.md", + "class": "evidence", + "owner": "repository maintainers", + "status": "active", + "replacement": null, + "retention": "retain", + "blob_sha": "d90911541a74869b7baa2ff02d672e46a5fe9116", + "archive_tag": null + }, { "path": "docs/program/legacy-intent-register.md", "class": "evidence", diff --git a/docs/documentation-manifest.seed.json b/docs/documentation-manifest.seed.json index 58bd3e938..e8d7a6e5a 100644 --- a/docs/documentation-manifest.seed.json +++ b/docs/documentation-manifest.seed.json @@ -3819,6 +3819,16 @@ "blob_sha": "17b866de92a67baec26cc671f50ba95fa21c1a9b", "archive_tag": null }, + { + "path": "docs/program/ledger/2026-08-15-ci-fail-slow-sweep.md", + "class": "evidence", + "owner": "repository maintainers", + "status": "active", + "replacement": null, + "retention": "retain", + "blob_sha": "d90911541a74869b7baa2ff02d672e46a5fe9116", + "archive_tag": null + }, { "path": "docs/program/legacy-intent-register.md", "class": "evidence", diff --git a/package.json b/package.json index cfb838f8d..16f0205e6 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "check:undeclared-imports": "node --test scripts/check-undeclared-imports.test.mjs && node scripts/check-undeclared-imports.mjs", "check:request-body-contract": "node --test scripts/check-request-body-contract.test.mjs scripts/check-openapi-refs.test.mjs scripts/check-prototype-chain-lookups.test.mjs && node scripts/check-request-body-contract.mjs && node scripts/check-openapi-refs.mjs && node scripts/check-prototype-chain-lookups.mjs", "check:prototype-chain-lookups": "node --test scripts/check-prototype-chain-lookups.test.mjs && node scripts/check-prototype-chain-lookups.mjs", - "check:ci-preflight": "node scripts/check-ci-preflight.mjs && node --test tools/ci/postgres-shard.test.mjs tools/ci/work-order-request-no-seed.test.mjs tools/ci/check-product-buck-residual.test.mjs tools/ci/check-nextest-config.test.mjs tools/ci/ingest-soft-reds.test.mjs scripts/local-admission.test.mjs tools/ci/assess-tip-contention.test.mjs tools/ci/check-mjs-dark-suites.test.mjs tools/ci/classify-ci-failure.test.mjs && node tools/ci/check-product-buck-residual.mjs && node tools/ci/check-nextest-config.mjs && node tools/ci/check-mjs-dark-suites.mjs --strict && node --test scripts/check-foundation-gates.test.mjs && node --test scripts/check-non-oci-mail-imessage-relay.test.mjs && node --test scripts/console/run-verification-queue.test.mjs", + "check:ci-preflight": "node scripts/check-ci-preflight.mjs && node --test tools/ci/postgres-shard.test.mjs tools/ci/cargo-test-runner.test.mjs tools/ci/work-order-request-no-seed.test.mjs tools/ci/check-product-buck-residual.test.mjs tools/ci/check-nextest-config.test.mjs tools/ci/ingest-soft-reds.test.mjs scripts/local-admission.test.mjs tools/ci/assess-tip-contention.test.mjs tools/ci/check-mjs-dark-suites.test.mjs tools/ci/classify-ci-failure.test.mjs && node tools/ci/check-product-buck-residual.mjs && node tools/ci/check-nextest-config.mjs && node tools/ci/check-mjs-dark-suites.mjs --strict && node --test scripts/check-foundation-gates.test.mjs && node --test scripts/check-non-oci-mail-imessage-relay.test.mjs && node --test scripts/console/run-verification-queue.test.mjs", "check:reasoning-lens-contract": "node scripts/check-reasoning-lens-contract.mjs", "check:doc-links": "node scripts/check-doc-links.mjs", "check:doc-manifest": "node scripts/console/generate-documentation-manifest.mjs --check", diff --git a/scripts/check-ci-preflight.mjs b/scripts/check-ci-preflight.mjs index 767e4bb63..fed137fa0 100644 --- a/scripts/check-ci-preflight.mjs +++ b/scripts/check-ci-preflight.mjs @@ -81,6 +81,34 @@ const runHeavyUnlessCancelledCondition = const runHeavyAlwaysCondition = "${{ always() && needs.preflight.outputs.run_heavy == 'true' }}"; +// Fail-slow one-sweep CI (D4): independent steps run even after a sibling step +// fails; dependent steps skip (not red) when their dependency failed. The step +// ids below are the dependency roots named in ci.yml. +const preflightCheckoutDependentCondition = + "${{ !cancelled() && steps.checkout.outcome == 'success' }}"; +const preflightSetupNodeDependentCondition = + "${{ !cancelled() && steps.setup-node.outcome == 'success' }}"; +const preflightNpmCiDependentCondition = + "${{ !cancelled() && steps.npm-ci.outcome == 'success' }}"; +const preflightNpmCiPrCondition = + "${{ !cancelled() && steps.derive.outcome == 'success' && steps.npm-ci.outcome == 'success' && github.event_name == 'pull_request' }}"; +const preflightDeriveCondition = + "${{ !cancelled() && steps.checkout.outcome == 'success' && github.event_name == 'pull_request' }}"; +const preflightCheckoutHeavyCondition = + "${{ !cancelled() && steps.checkout.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }}"; +const preflightBuckHeavyCondition = + "${{ !cancelled() && steps.dotslash.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }}"; +const preflightRustHeavyCondition = + "${{ !cancelled() && steps.rust-toolchain.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }}"; +const preflightNpmCiHeavyCondition = + "${{ !cancelled() && steps.npm-ci.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }}"; +const collectFailuresCondition = "${{ !cancelled() }}"; +/** Backend fail-slow: independent (no prior-step dependency) and topology-dependent guards. */ +const backendIndependentCondition = + "${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }}"; +const backendTopologyDependentCondition = + "${{ !cancelled() && steps.topology.outcome == 'success' && needs.preflight.outputs.run_heavy == 'true' }}"; + export const PATH_CLASS_RULES_VERSION = "3"; const docsOnlyRootFiles = new Set([ "README.md", @@ -786,66 +814,68 @@ const proofDigest = (name, digest, options) => runDigestContract("proof", name, // familiar name. const requiredJobRunContracts = Object.freeze({ "preflight": [ - setupDigest("Derive exact console C/T/M train", "b8e69e979347fa526773bb2a740bdb198be414077281026c23be96501ffd4da1", { if: consolePrCondition, shell: "bash" }), - setupRun("Install workspace dependencies", "npm ci"), - setupDigest("Classify path class", "d963a8aa99e66c44a4ed8e3ef25725d206a11973545758d6c67a055b1f48cbbd", { shell: "bash" }), - setupRun("Install pinned DotSlash runtime", "tools/buck/install_dotslash.sh", { if: preflightRunHeavyCondition }), - proofRun("Cheap Buck2 generated-face admission", "tools/buck/preflight.sh", { if: preflightRunHeavyCondition }), - proofRun("Foundation gate contract", "npm run check:foundation-gates"), - proofRun("Reasoning lens contract regression", "node --test scripts/check-reasoning-lens-contract.test.mjs"), - proofDigest("Reasoning lens changed-record admission", "b4d78de511586e6f3cb7edafcf780fbc0361279dc8f0fe544b6128cfad9d3ab9", { shell: "bash" }), - proofRun("Console truth-ledger exact-M admission", "npm run check:console-truth-ledger", { if: consolePrCondition }), - proofRun("Console fanout planner exact-M admission", "node scripts/console/plan-fanout.mjs --candidate \"$CONSOLE_CANDIDATE_SHA\" --authority-tip \"$CONSOLE_AUTHORITY_TIP_SHA\" --synthetic-merge \"$CONSOLE_SYNTHETIC_MERGE_SHA\"", { if: consolePrCondition }), - proofRun("CI preflight contract tests", "node --test scripts/check-ci-preflight.test.mjs"), - proofRun("Console route inventory regression", "node --test scripts/console/route-inventory.test.mjs"), - proofRun("Console authority-train regression", "node --test scripts/console/verify-console-authority-train.test.mjs"), - proofRun("Console lane-receipt validator regression", "npm run test:lane-receipt"), - proofRun("Console PR authority bootstrap regression", "node --test scripts/console/verify-console-pr-authority-bootstrap.test.mjs scripts/console/release-please-bot-candidate.test.mjs"), - proofRun("Executed-tests baseline set regression", "npm run test:executed-tests-baseline"), - proofRun("Local CI mirror contract", "node --test scripts/verify.test.mjs"), - proofRun("Console truth-ledger validator exact-M regression", "node --test scripts/console/validate-console-truth-ledger.test.mjs"), - proofRun("Console fanout planner exact-M regression", "node --test scripts/console/plan-fanout.test.mjs"), - proofRun("Buck PostgreSQL environment wrapper regression", "tools/buck/run_test_with_postgres_env.test.sh", { if: preflightRunHeavyCondition }), - proofRun("Buck disposable PostgreSQL harness regression", "tools/buck/test_needs_postgres.test.sh", { if: preflightRunHeavyCondition }), - proofRun("CI preflight contract", "npm run check:ci-preflight"), - proofRun("Canonical npm lockfile", "npm run check:package-lock"), - proofRun("Cargo.lock consistency", "cargo metadata --manifest-path backend/Cargo.toml --locked --format-version=1 >/dev/null", { if: preflightRunHeavyCondition }), - proofRun("Executed-tests ratchet — a test binary must have a path from a workflow step", "npm run check:executed-tests", { if: preflightRunHeavyCondition }), - proofRun("JavaScript test reachability ratchet", "npm run check:js-test-reachability"), - proofRun("JavaScript test reachability unit tests", "npm run test:js-test-reachability"), - proofRun("Lane fan-out harness preflight", "node .claude/workflows/lane-fanout.test.mjs"), - proofRun("Workflow test-runner credential literals", "npm run check:test-credentials"), + setupDigest("Derive exact console C/T/M train", "b8e69e979347fa526773bb2a740bdb198be414077281026c23be96501ffd4da1", { if: preflightDeriveCondition, shell: "bash" }), + setupRun("Install workspace dependencies", "npm ci", { if: preflightSetupNodeDependentCondition }), + setupDigest("Classify path class", "d963a8aa99e66c44a4ed8e3ef25725d206a11973545758d6c67a055b1f48cbbd", { if: preflightNpmCiDependentCondition, shell: "bash" }), + setupRun("Install pinned DotSlash runtime", "tools/buck/install_dotslash.sh", { if: preflightCheckoutHeavyCondition }), + proofRun("Cheap Buck2 generated-face admission", "tools/buck/preflight.sh", { if: preflightBuckHeavyCondition }), + proofRun("Foundation gate contract", "npm run check:foundation-gates", { if: preflightNpmCiDependentCondition }), + proofRun("Reasoning lens contract regression", "node --test scripts/check-reasoning-lens-contract.test.mjs", { if: preflightNpmCiDependentCondition }), + proofDigest("Reasoning lens changed-record admission", "b4d78de511586e6f3cb7edafcf780fbc0361279dc8f0fe544b6128cfad9d3ab9", { if: preflightNpmCiDependentCondition, shell: "bash" }), + proofRun("Console truth-ledger exact-M admission", "npm run check:console-truth-ledger", { if: preflightNpmCiPrCondition }), + proofRun("Console fanout planner exact-M admission", "node scripts/console/plan-fanout.mjs --candidate \"$CONSOLE_CANDIDATE_SHA\" --authority-tip \"$CONSOLE_AUTHORITY_TIP_SHA\" --synthetic-merge \"$CONSOLE_SYNTHETIC_MERGE_SHA\"", { if: preflightNpmCiPrCondition }), + proofRun("CI preflight contract tests", "node --test scripts/check-ci-preflight.test.mjs", { if: preflightNpmCiDependentCondition }), + proofRun("Console route inventory regression", "node --test scripts/console/route-inventory.test.mjs", { if: preflightNpmCiDependentCondition }), + proofRun("Console authority-train regression", "node --test scripts/console/verify-console-authority-train.test.mjs", { if: preflightNpmCiDependentCondition }), + proofRun("Console lane-receipt validator regression", "npm run test:lane-receipt", { if: preflightNpmCiDependentCondition }), + proofRun("Console PR authority bootstrap regression", "node --test scripts/console/verify-console-pr-authority-bootstrap.test.mjs scripts/console/release-please-bot-candidate.test.mjs", { if: preflightNpmCiDependentCondition }), + proofRun("Executed-tests baseline set regression", "npm run test:executed-tests-baseline", { if: preflightNpmCiDependentCondition }), + proofRun("Local CI mirror contract", "node --test scripts/verify.test.mjs", { if: preflightNpmCiDependentCondition }), + proofRun("Console truth-ledger validator exact-M regression", "node --test scripts/console/validate-console-truth-ledger.test.mjs", { if: preflightNpmCiDependentCondition }), + proofRun("Console fanout planner exact-M regression", "node --test scripts/console/plan-fanout.test.mjs", { if: preflightNpmCiDependentCondition }), + proofRun("Buck PostgreSQL environment wrapper regression", "tools/buck/run_test_with_postgres_env.test.sh", { if: preflightBuckHeavyCondition }), + proofRun("Buck disposable PostgreSQL harness regression", "tools/buck/test_needs_postgres.test.sh", { if: preflightBuckHeavyCondition }), + proofRun("CI preflight contract", "npm run check:ci-preflight", { if: preflightNpmCiDependentCondition }), + proofRun("Canonical npm lockfile", "npm run check:package-lock", { if: preflightNpmCiDependentCondition }), + proofRun("Cargo.lock consistency", "cargo metadata --manifest-path backend/Cargo.toml --locked --format-version=1 >/dev/null", { if: preflightRustHeavyCondition }), + proofRun("Executed-tests ratchet — a test binary must have a path from a workflow step", "npm run check:executed-tests", { if: preflightNpmCiHeavyCondition }), + proofRun("JavaScript test reachability ratchet", "npm run check:js-test-reachability", { if: preflightNpmCiDependentCondition }), + proofRun("JavaScript test reachability unit tests", "npm run test:js-test-reachability", { if: preflightNpmCiDependentCondition }), + proofRun("Lane fan-out harness preflight", "node .claude/workflows/lane-fanout.test.mjs", { if: preflightNpmCiDependentCondition }), + proofRun("Workflow test-runner credential literals", "npm run check:test-credentials", { if: preflightNpmCiDependentCondition }), + proofRun("Collect failures", "node scripts/ci-collect-failures.mjs", { if: collectFailuresCondition }), ], "domain-unit": [ proofDigest("Path-class skip proof", "1fdf99dda32af815824808d703216d2c0cf04a0adc146dd29f24746e549c44e0", { if: skipProofCondition, shell: "bash" }), - proofDigest("Domain crate unit tests", "3c40987e306ba9e84199083af183ae1135735b77c5f7a2277dcfe23b27d46caf", { if: runHeavyCondition }), + proofDigest("Domain crate unit tests", "754df637470658378c9449a3daf2eaa8ca05446b3a024660591c528d14489215", { if: runHeavyCondition }), ], "backend": [ proofDigest("Path-class skip proof", "1fdf99dda32af815824808d703216d2c0cf04a0adc146dd29f24746e549c44e0", { if: skipProofCondition, shell: "bash" }), - setupRun("Install pinned DotSlash runtime", "../tools/buck/install_dotslash.sh", { if: runHeavyCondition }), - proofRun("rustfmt check", "cargo fmt --all -- --check", { if: runHeavyCondition }), - proofRun("clippy -D warnings", "SQLX_OFFLINE=true cargo clippy --all-targets -- -D warnings", { if: runHeavyCondition }), - proofRun("Layer-boundary gate", "../tools/buck2 run //backend/ci/gates/layer-boundary:console-gate-layer-boundary", { if: runHeavyCondition }), - proofRun("Audit-coverage gate", "cargo run -p console-gate-audit-coverage", { if: runHeavyCondition }), - proofRun("Migration-safety gate", "cargo run -p console-gate-migration-safety", { if: runHeavyCondition }), - proofRun("Tenant-isolation gate", "cargo run -p console-gate-tenant-isolation", { if: runHeavyCondition }), - proofRun("PII-no-logs gate", "cargo run -p console-gate-pii-no-logs", { if: runHeavyCondition }), - proofRun("RLS-arming gate", "cargo run -p console-gate-rls-arming", { if: runHeavyCondition }), - proofRun("Dev-auth-absence gate", "cargo run -p console-gate-dev-auth-absence", { if: runHeavyCondition }), - proofRun("IaC tier-discipline gate", "cargo run -p console-gate-iac-tier", { if: runHeavyCondition }), - proofRun("Fabricated-branch gate", "cargo run -p console-gate-fabricated-branch", { if: runHeavyCondition }), - proofRun("Personal-data-classification gate", "cargo run -p console-gate-personal-data-classification", { if: runHeavyCondition }), - proofRun("Writer-ownership gate", "cargo run -p console-gate-writer-ownership", { if: runHeavyCondition }), - proofDigest("Buck2 CI-gate mutation suites — every gate proven to still reject", "f6614509bd73220754a83d449b8bf422e616309ba48965f730f0d3dcff9d2cf4", { if: runHeavyCondition, workingDirectory: "." }), - proofRun("PR 473 migration operational contract tests", "python3 scripts/check-pr473-migration-operational.test.py -v", { if: runHeavyCondition, workingDirectory: "." }), - setupDigest("Reconcile portable PostgreSQL role topology", "5da0f2d8c399657dbc0a9d358c81d71399af1ea6c659074a365653db21fcaded", { if: runHeavyCondition }), - proofRun("PR 473 migration operational gate", "npm run check:pr473-migration-operational", { if: runHeavyCondition, workingDirectory: "." }), - proofDigest("Boot smoke — migrate + serve + /readyz", "d51d75f8cd49be1557c5b5c1f5f641345bc82f842d2384e9608e9872b0714d79", { if: runHeavyCondition }), - proofDigest("Buck2 dev-auth feature PostgreSQL suites", "f059b50b432f8cafc4e58b14272fe76f5dd3d21842b8683f08c0a5f1f7a84001", { if: runHeavyCondition, workingDirectory: "." }), - proofRun("Buck2 platform-authz unit suite", "env -u DATABASE_URL tools/buck2 test //backend/crates/platform/authz:console-platform-authz-unit", { if: runHeavyCondition, workingDirectory: "." }), - proofRun("Buck2 console-app unit suite", "env -u DATABASE_URL tools/buck2 test //backend/app:console-app-unit", { if: runHeavyCondition, workingDirectory: "." }), - proofRun("Buck2 console-app OpenAPI drift suite", "env -u DATABASE_URL tools/buck2 test //backend/app:console-app-itest-openapi_drift", { if: runHeavyCondition, workingDirectory: "." }), - proofDigest("Buck2 console-app inline PostgreSQL suites", "2a59f90874addb48871158b672a9016159caba7382f49252d43beba2372daf63", { if: runHeavyCondition, workingDirectory: "." }), + setupRun("Install pinned DotSlash runtime", "../tools/buck/install_dotslash.sh", { if: backendIndependentCondition }), + proofRun("rustfmt check", "cargo fmt --all -- --check", { if: backendIndependentCondition }), + proofRun("clippy -D warnings", "SQLX_OFFLINE=true cargo clippy --all-targets -- -D warnings", { if: backendIndependentCondition }), + proofRun("Layer-boundary gate", "../tools/buck2 run //backend/ci/gates/layer-boundary:console-gate-layer-boundary", { if: backendIndependentCondition }), + proofRun("Audit-coverage gate", "cargo run -p console-gate-audit-coverage", { if: backendIndependentCondition }), + proofRun("Migration-safety gate", "cargo run -p console-gate-migration-safety", { if: backendIndependentCondition }), + proofRun("Tenant-isolation gate", "cargo run -p console-gate-tenant-isolation", { if: backendIndependentCondition }), + proofRun("PII-no-logs gate", "cargo run -p console-gate-pii-no-logs", { if: backendIndependentCondition }), + proofRun("RLS-arming gate", "cargo run -p console-gate-rls-arming", { if: backendIndependentCondition }), + proofRun("Dev-auth-absence gate", "cargo run -p console-gate-dev-auth-absence", { if: backendIndependentCondition }), + proofRun("IaC tier-discipline gate", "cargo run -p console-gate-iac-tier", { if: backendIndependentCondition }), + proofRun("Fabricated-branch gate", "cargo run -p console-gate-fabricated-branch", { if: backendIndependentCondition }), + proofRun("Personal-data-classification gate", "cargo run -p console-gate-personal-data-classification", { if: backendIndependentCondition }), + proofRun("Writer-ownership gate", "cargo run -p console-gate-writer-ownership", { if: backendIndependentCondition }), + proofDigest("Buck2 CI-gate mutation suites — every gate proven to still reject", "f6614509bd73220754a83d449b8bf422e616309ba48965f730f0d3dcff9d2cf4", { if: backendIndependentCondition, workingDirectory: "." }), + proofRun("PR 473 migration operational contract tests", "python3 scripts/check-pr473-migration-operational.test.py -v", { if: backendIndependentCondition, workingDirectory: "." }), + setupDigest("Reconcile portable PostgreSQL role topology", "5da0f2d8c399657dbc0a9d358c81d71399af1ea6c659074a365653db21fcaded", { if: backendIndependentCondition }), + proofRun("PR 473 migration operational gate", "npm run check:pr473-migration-operational", { if: backendTopologyDependentCondition, workingDirectory: "." }), + proofDigest("Boot smoke — migrate + serve + /readyz", "d51d75f8cd49be1557c5b5c1f5f641345bc82f842d2384e9608e9872b0714d79", { if: backendTopologyDependentCondition }), + proofDigest("Buck2 dev-auth feature PostgreSQL suites", "f059b50b432f8cafc4e58b14272fe76f5dd3d21842b8683f08c0a5f1f7a84001", { if: backendTopologyDependentCondition, workingDirectory: "." }), + proofRun("Buck2 platform-authz unit suite", "env -u DATABASE_URL tools/buck2 test //backend/crates/platform/authz:console-platform-authz-unit", { if: backendIndependentCondition, workingDirectory: "." }), + proofRun("Buck2 console-app unit suite", "env -u DATABASE_URL tools/buck2 test //backend/app:console-app-unit", { if: backendIndependentCondition, workingDirectory: "." }), + proofRun("Buck2 console-app OpenAPI drift suite", "env -u DATABASE_URL tools/buck2 test //backend/app:console-app-itest-openapi_drift", { if: backendIndependentCondition, workingDirectory: "." }), + proofDigest("Buck2 console-app inline PostgreSQL suites", "2a59f90874addb48871158b672a9016159caba7382f49252d43beba2372daf63", { if: backendTopologyDependentCondition, workingDirectory: "." }), + proofRun("Collect failures", "node scripts/ci-collect-failures.mjs", { if: collectFailuresCondition, workingDirectory: "." }), ], "dev-up-smoke": [ proofDigest("Path-class skip proof", "1fdf99dda32af815824808d703216d2c0cf04a0adc146dd29f24746e549c44e0", { if: skipProofCondition, shell: "bash" }), @@ -939,14 +969,14 @@ const requiredJobRunContracts = Object.freeze({ }); function actionStep(index, name, uses, withInputs, options = {}) { - // Allow `actionStep(i, name, uses, { if })` when the action has no `with:`. + // Allow `actionStep(i, name, uses, { if, id })` when the action has no `with:`. if ( options && Object.keys(options).length === 0 && withInputs && typeof withInputs === "object" && !Array.isArray(withInputs) - && Object.keys(withInputs).every((key) => key === "if") + && Object.keys(withInputs).every((key) => key === "if" || key === "id") ) { options = withInputs; withInputs = undefined; @@ -955,6 +985,7 @@ function actionStep(index, name, uses, withInputs, options = {}) { ? { name, uses } : { name, uses, with: withInputs }; if (options.if != null) step.if = options.if; + if (options.id != null) step.id = options.id; return { index, step }; } @@ -964,9 +995,9 @@ function actionStep(index, name, uses, withInputs, options = {}) { // be installed on a hosted runner and presenting that accident as proof. const requiredJobActionContracts = Object.freeze({ "preflight": [ - actionStep(0, "Checkout", "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", {"persist-credentials":false,"fetch-depth":0}), - actionStep(2, "Set up Node.js", "actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e", {"node-version":"24.16.0","cache":"npm"}), - actionStep(6, "Install Rust toolchain for Cargo.lock consistency", "dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8", {"toolchain":"1.97.1"}, { if: preflightRunHeavyCondition }), + actionStep(0, "Checkout", "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", {"persist-credentials":false,"fetch-depth":0}, { id: "checkout" }), + actionStep(2, "Set up Node.js", "actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e", {"node-version":"24.16.0","cache":"npm"}, { if: preflightCheckoutDependentCondition, id: "setup-node" }), + actionStep(6, "Install Rust toolchain for Cargo.lock consistency", "dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8", {"toolchain":"1.97.1"}, { if: preflightCheckoutHeavyCondition, id: "rust-toolchain" }), ], "domain-unit": [ actionStep(1, "Checkout", "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", {"persist-credentials":false}, { if: runHeavyCondition }), @@ -974,10 +1005,10 @@ const requiredJobActionContracts = Object.freeze({ actionStep(3, "Cache Rust dependencies + build artifacts", "Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4", {"workspaces":"backend","shared-key":"backend-cargo","cache-all-crates":"true","save-if":false}, { if: runHeavyCondition }), ], "backend": [ - actionStep(0, "Checkout", "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", {"persist-credentials":false}), - actionStep(3, "Free runner disk for Rust backend", "./.github/actions/free-runner-disk", { if: runHeavyCondition }), - actionStep(4, "Install Rust toolchain (pinned via rust-toolchain.toml)", "dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8", {"toolchain":"1.97.1","components":"rustfmt, clippy"}, { if: runHeavyCondition }), - actionStep(5, "Cache Rust dependencies + build artifacts", "Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4", {"workspaces":"backend","shared-key":"backend-cargo","cache-all-crates":"true","save-if":"${{ github.ref == 'refs/heads/main' }}"}, { if: runHeavyCondition }), + actionStep(0, "Checkout", "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", {"persist-credentials":false}, { id: "checkout" }), + actionStep(3, "Free runner disk for Rust backend", "./.github/actions/free-runner-disk", { if: backendIndependentCondition, id: "free-disk" }), + actionStep(4, "Install Rust toolchain (pinned via rust-toolchain.toml)", "dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8", {"toolchain":"1.97.1","components":"rustfmt, clippy"}, { if: backendIndependentCondition, id: "rust" }), + actionStep(5, "Cache Rust dependencies + build artifacts", "Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4", {"workspaces":"backend","shared-key":"backend-cargo","cache-all-crates":"true","save-if":"${{ github.ref == 'refs/heads/main' }}"}, { if: backendIndependentCondition, id: "rust-cache" }), ], "dev-up-smoke": [ actionStep(1, "Checkout", "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", {"persist-credentials":false}, { if: runHeavyCondition }), @@ -1137,6 +1168,9 @@ const protectedJobExecutionMetadata = { env: Object.fromEntries( reasoningLensAdmissionEnvironment.map((entry) => entry.split(": ", 2)), ), + }, { + name: "Collect failures", + env: { CI_STEPS: "${{ toJSON(steps) }}" }, }], }, "domain-unit": {}, @@ -1161,6 +1195,10 @@ const protectedJobExecutionMetadata = { CARGO_PROFILE_TEST_DEBUG: "0", }, defaults: { run: { "working-directory": "backend" } }, + stepEnv: [{ + name: "Collect failures", + env: { CI_STEPS: "${{ toJSON(steps) }}" }, + }], }, "dev-up-smoke": { env: { @@ -1510,9 +1548,9 @@ function requireReasoningLensContracts(steps, failures) { regressionByName.length !== 1 || regressionByCommand.length !== 1 || regressionByName[0] !== regressionByCommand[0] - || !hasOnlyExpectedCondition(regressionByName[0] ?? "", null) + || !hasOnlyExpectedCondition(regressionByName[0] ?? "", preflightNpmCiDependentCondition) ) { - failures.push("preflight must run the exact reasoning-lens regression once and unconditionally"); + failures.push("preflight must run the exact reasoning-lens regression once and only after npm ci succeeds"); } const admissions = steps.filter((step) => stepName(step) === reasoningLensAdmissionName); @@ -1529,7 +1567,7 @@ function requireReasoningLensContracts(steps, failures) { || JSON.stringify(shells) !== JSON.stringify(["bash"]) || JSON.stringify(environment) !== JSON.stringify(reasoningLensAdmissionEnvironment) || script !== reasoningLensAdmissionScript - || !hasOnlyExpectedCondition(admission, null) + || !hasOnlyExpectedCondition(admission, preflightNpmCiDependentCondition) ) { failures.push("preflight must preserve the exact reasoning-lens event admission contract"); } @@ -1537,16 +1575,16 @@ function requireReasoningLensContracts(steps, failures) { function requireConsoleExactMergeProof(workflow, steps, failures) { const derive = steps.filter((step) => stepName(step) === "Derive exact console C/T/M train"); - if (derive.length !== 1 || !hasOnlyExpectedCondition(derive[0], consolePrCondition) || multilineRunCommands(derive[0]).join("\n") !== consoleTrainDerivation.join("\n")) { + if (derive.length !== 1 || !hasOnlyExpectedCondition(derive[0], preflightDeriveCondition) || multilineRunCommands(derive[0]).join("\n") !== consoleTrainDerivation.join("\n")) { failures.push("preflight must derive exact C/T/M from the pull-request synthetic merge"); } for (const command of [consoleTruthLedgerCommand, consoleFanoutPlannerAdmissionCommand]) { const matching = steps.filter((step) => runScalar(step) === command); - if (matching.length !== 1 || !hasOnlyExpectedCondition(matching[0], consolePrCondition)) failures.push(`preflight must run ${command} only after exact C/T/M derivation on pull requests`); + if (matching.length !== 1 || !hasOnlyExpectedCondition(matching[0], preflightNpmCiPrCondition)) failures.push(`preflight must run ${command} only after exact C/T/M derivation on pull requests`); } for (const command of [consoleAuthorityTrainTestCommand, consoleBootstrapTestCommand, consoleTruthLedgerTestCommand, consoleFanoutPlannerTestCommand]) { const matching = steps.filter((step) => runScalar(step) === command); - if (matching.length !== 1 || !isUnconditional(matching[0])) failures.push(`preflight must run ${command} unconditionally on pull requests and main`); + if (matching.length !== 1 || !hasOnlyExpectedCondition(matching[0], preflightNpmCiDependentCondition)) failures.push(`preflight must run ${command} unconditionally on pull requests and main`); } if (workflow.includes("CONSOLE_INTEGRATION_TIP_SHA")) failures.push("preflight must not reference legacy CONSOLE_INTEGRATION_TIP_SHA"); } @@ -1891,7 +1929,7 @@ function requirePreflightRustToolchainBefore(steps, failures) { failures.push("preflight must install the pinned Rust toolchain before Cargo-dependent CI preflight tests"); return; } - if (!hasOnlyExpectedCondition(steps[setupIndex], preflightRunHeavyCondition)) { + if (!hasOnlyExpectedCondition(steps[setupIndex], preflightCheckoutHeavyCondition)) { failures.push("preflight must install the pinned Rust toolchain only when run_heavy"); return; } @@ -1905,7 +1943,7 @@ function requirePreflightRustToolchainBefore(steps, failures) { function requireDotSlashBefore(steps, command, job, failures) { const commandIndex = steps.findIndex((step) => runScalar(step) === command); const dotSlashIndex = steps.findIndex((step) => runScalar(step) === dotSlashBootstrap); - const expectedIf = job === "preflight" ? preflightRunHeavyCondition : runHeavyCondition; + const expectedIf = job === "preflight" ? preflightCheckoutHeavyCondition : runHeavyCondition; if (dotSlashIndex < 0) { failures.push(`${job} must install pinned DotSlash before Buck2`); } else if (!hasOnlyExpectedCondition(steps[dotSlashIndex], expectedIf)) { @@ -2000,14 +2038,27 @@ export function evaluateCiPreflight( requireDotSlashBefore(preflightSteps, "tools/buck/preflight.sh", "preflight", failures); requirePreflightRustToolchainBefore(preflightSteps, failures); for (const command of requiredAlwaysPreflightCommands) { - requireUnconditionalRun(preflightSteps, command, "preflight", failures); + requireRunWithCondition( + preflightSteps, + command, + "preflight", + preflightNpmCiDependentCondition, + failures, + ); } + const heavyPreflightCommandConditions = { + "tools/buck/preflight.sh": preflightBuckHeavyCondition, + [buckPostgresEnvironmentTestCommand]: preflightBuckHeavyCondition, + [buckPostgresHarnessTestCommand]: preflightBuckHeavyCondition, + "cargo metadata --manifest-path backend/Cargo.toml --locked --format-version=1 >/dev/null": preflightRustHeavyCondition, + "npm run check:executed-tests": preflightNpmCiHeavyCondition, + }; for (const command of requiredHeavyPreflightCommands) { requireRunWithCondition( preflightSteps, command, "preflight", - preflightRunHeavyCondition, + heavyPreflightCommandConditions[command], failures, ); } @@ -2015,7 +2066,7 @@ export function evaluateCiPreflight( const classify = preflightSteps.filter((step) => stepName(step) === "Classify path class"); if ( classify.length !== 1 - || !hasOnlyExpectedCondition(classify[0], null) + || !hasOnlyExpectedCondition(classify[0], preflightNpmCiDependentCondition) || multilineRunCommands(classify[0]).join("\n") !== pathClassEmitScript.join("\n") ) { failures.push("preflight must classify path class before thin/heavy step gating"); @@ -2036,15 +2087,31 @@ export function evaluateCiPreflight( const domainSteps = steps.filter((step) => stepName(step) === "Domain crate unit tests"); const domainStep = domainSteps[0] ?? ""; const parsedDomainCommands = shellCommandTokens(runScript(domainStep)); + // Fail-slow keep-going: the run body now carries wrapper commands (`set +e`, + // `check_status`, the summary). Extract only the cargo/git invocations — the + // exact surface check-executed-tests attributes — and compare them verbatim, + // so a rewrapped, renamed, or merged invocation still fails here. + const stripAssignments = (tokens) => { + let index = 0; + while (index < tokens.length && /^[A-Za-z_][A-Za-z0-9_]*=/.test(tokens[index])) index += 1; + return tokens.slice(index); + }; + const invocationCommands = parsedDomainCommands + .filter((command) => !command.malformed) + .map((command) => stripAssignments(command.tokens)) + .filter((tokens) => tokens[0] === "cargo" || tokens[0] === "git"); + const expectedInvocations = domainUnitExpectedCommands.map(stripAssignments); const domainCommandsMatch = domainSteps.length === 1 - && parsedDomainCommands.length === domainUnitExpectedCommands.length - && parsedDomainCommands.every((command, index) => ( - !command.malformed - && JSON.stringify(command.tokens) === JSON.stringify(domainUnitExpectedCommands[index]) + && invocationCommands.length === expectedInvocations.length + && invocationCommands.every((tokens, index) => ( + JSON.stringify(tokens) === JSON.stringify(expectedInvocations[index]) )); if (!domainCommandsMatch || !hasOnlyExpectedCondition(domainStep, runHeavyCondition)) { failures.push("domain-unit must execute the locked Cargo test commands directly when run_heavy"); } + if (!/ci-keep-going:/.test(domainStep) || !/exit 1/.test(domainStep)) { + failures.push("domain-unit keep-going block must re-raise failures with a summary exit 1"); + } if (/^ (?:env|defaults):/m.test(domainUnit) || /^ env:/m.test(domainUnit)) { failures.push("domain-unit must use the default shell with no job or step env/defaults overrides"); @@ -2058,7 +2125,7 @@ export function evaluateCiPreflight( const present = directTokens.some((token, index) => token === "--test" && directTokens[index + 1] === t); if (!present) failures.push(`domain-unit must run --test ${t}`); } - if (parsedDomainCommands[0]?.tokens?.includes("--lib") !== true) { + if (invocationCommands[0]?.includes("--lib") !== true) { failures.push("domain-unit must pass --lib on its first cargo invocation"); } @@ -2153,11 +2220,7 @@ export function evaluateCiPreflight( const backend = jobBlock(workflow, "backend"); if (backend) { const steps = stepBlocks(backend); - const failFastIf = runHeavyCondition; const pr473ContractTestCommand = "python3 scripts/check-pr473-migration-operational.test.py -v"; - if (steps.some((step) => step.includes("if: ${{ !cancelled() }}"))) { - failures.push("backend must not use !cancelled() on protected fail-fast steps"); - } const sourceGateContracts = [ ["Layer-boundary gate", "../tools/buck2 run //backend/ci/gates/layer-boundary:console-gate-layer-boundary"], ["Audit-coverage gate", "cargo run -p console-gate-audit-coverage"], @@ -2169,16 +2232,19 @@ export function evaluateCiPreflight( ["IaC tier-discipline gate", "cargo run -p console-gate-iac-tier"], ["Fabricated-branch gate", "cargo run -p console-gate-fabricated-branch"], ]; + // Fail-slow sweep: fmt/clippy/gates (and the topology reconcile they do not + // depend on) run regardless of each other; the DB-dependent steps guard on + // the topology reconcile's success so one root failure shows as skipped. const gateIndexes = requireOrderedStepContracts( steps, [ - ["clippy -D warnings", "SQLX_OFFLINE=true cargo clippy --all-targets -- -D warnings"], - ...sourceGateContracts, - ["PR 473 migration operational contract tests", pr473ContractTestCommand], - ["Reconcile portable PostgreSQL role topology", undefined], - ["PR 473 migration operational gate", "npm run check:pr473-migration-operational"], - ["Boot smoke — migrate + serve + /readyz", undefined], - ].map(([name, run]) => ({ name, run, if: failFastIf })), + { name: "clippy -D warnings", run: "SQLX_OFFLINE=true cargo clippy --all-targets -- -D warnings", if: backendIndependentCondition }, + ...sourceGateContracts.map(([name, run]) => ({ name, run, if: backendIndependentCondition })), + { name: "PR 473 migration operational contract tests", run: pr473ContractTestCommand, if: backendIndependentCondition }, + { name: "Reconcile portable PostgreSQL role topology", run: undefined, if: backendIndependentCondition }, + { name: "PR 473 migration operational gate", run: "npm run check:pr473-migration-operational", if: backendTopologyDependentCondition }, + { name: "Boot smoke — migrate + serve + /readyz", run: undefined, if: backendTopologyDependentCondition }, + ], "backend", failures, ); @@ -2198,7 +2264,7 @@ export function evaluateCiPreflight( "//tools/buck:provisioning-dev-principal-upsert-race-postgres", ].join("\n"), workingDirectory: ".", - if: failFastIf, + if: backendTopologyDependentCondition, }, { // Locked so the step cannot be deleted silently: the crate's residual @@ -2207,13 +2273,13 @@ export function evaluateCiPreflight( name: "Buck2 platform-authz unit suite", run: "env -u DATABASE_URL tools/buck2 test //backend/crates/platform/authz:console-platform-authz-unit", workingDirectory: ".", - if: failFastIf, + if: backendIndependentCondition, }, { name: "Buck2 console-app unit suite", run: "env -u DATABASE_URL tools/buck2 test //backend/app:console-app-unit", workingDirectory: ".", - if: failFastIf, + if: backendIndependentCondition, }, { // The suite H-1 is *about*. `openapi_drift` is the only thing that inventories every @@ -2225,7 +2291,7 @@ export function evaluateCiPreflight( name: "Buck2 console-app OpenAPI drift suite", run: "env -u DATABASE_URL tools/buck2 test //backend/app:console-app-itest-openapi_drift", workingDirectory: ".", - if: failFastIf, + if: backendIndependentCondition, }, { name: "Buck2 console-app inline PostgreSQL suites", @@ -2235,7 +2301,7 @@ export function evaluateCiPreflight( "//tools/buck:app-dev-auth-persona-guard-postgres", ].join("\n"), workingDirectory: ".", - if: failFastIf, + if: backendTopologyDependentCondition, }, ], "backend", diff --git a/scripts/check-ci-preflight.test.mjs b/scripts/check-ci-preflight.test.mjs index b8beb514a..653e3888c 100644 --- a/scripts/check-ci-preflight.test.mjs +++ b/scripts/check-ci-preflight.test.mjs @@ -23,10 +23,14 @@ const freeRunnerDiskAction = readFileSync( const cargoLockGate = "cargo metadata --manifest-path backend/Cargo.toml --locked --format-version=1 >/dev/null"; const ciPreflightTests = "node --test scripts/check-ci-preflight.test.mjs"; const reasoningLensRegressionStep = ` - name: Reasoning lens contract regression + id: reasoning-lens-regression + if: \${{ !cancelled() && steps.npm-ci.outcome == 'success' }} run: node --test scripts/check-reasoning-lens-contract.test.mjs `; const reasoningLensAdmissionStep = ` - name: Reasoning lens changed-record admission + id: reasoning-lens-admission + if: \${{ !cancelled() && steps.npm-ci.outcome == 'success' }} shell: bash env: REASONING_PR_BASE_SHA: \${{ github.event.pull_request.base.sha }} @@ -74,7 +78,8 @@ const reachabilityPreflightCommands = [ "tools/buck/test_needs_postgres.test.sh", ]; const preflightRustToolchainSetup = ` - name: Install Rust toolchain for Cargo.lock consistency - if: \${{ steps.path_class.outputs.run_heavy == 'true' }} + id: rust-toolchain + if: \${{ !cancelled() && steps.checkout.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }} uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: toolchain: "1.97.1" @@ -84,6 +89,16 @@ const runHeavyIf = "${{ needs.preflight.outputs.run_heavy == 'true' }}"; const preflightRunHeavyIf = "${{ steps.path_class.outputs.run_heavy == 'true' }}"; const runHeavyUnlessCancelledIf = "${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }}"; +const preflightCheckoutHeavyIf = + "${{ !cancelled() && steps.checkout.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }}"; +const preflightBuckHeavyIf = + "${{ !cancelled() && steps.dotslash.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }}"; +const preflightRustHeavyIf = + "${{ !cancelled() && steps.rust-toolchain.outcome == 'success' && steps.path_class.outputs.run_heavy == 'true' }}"; +const backendIndependentIf = + "${{ !cancelled() && needs.preflight.outputs.run_heavy == 'true' }}"; +const npmCiIf = "${{ !cancelled() && steps.npm-ci.outcome == 'success' }}"; +const npmCiPrIf = "${{ !cancelled() && steps.derive.outcome == 'success' && steps.npm-ci.outcome == 'success' && github.event_name == 'pull_request' }}"; function expectFailure( source, @@ -438,9 +453,9 @@ describe("CI preflight contract", () => { it("rejects every run-step condition, soft-failure, and retained-text early-exit bypass", () => { const requiredRunStepCounts = { - preflight: 29, + preflight: 30, "domain-unit": 2, - backend: 25, + backend: 26, "dev-up-smoke": 7, "kubernetes-manifests": 7, "repo-gates": 26, @@ -486,9 +501,10 @@ describe("CI preflight contract", () => { // 107 -> 121: path-class skip proofs on skip-proof jobs (+classify in preflight). // 121 -> 122: lane-receipt validator regression step in preflight. - assert.equal(runStepCount, 122, "required and planned job run-step coverage must not shrink"); - // Three mutations per run step: 122*3 = 366. - assert.equal(mutationCount, 366, "exhaustive bypass matrix must not shrink"); + // 122 -> 124: fail-slow sweep collect-failures steps in preflight and backend. + assert.equal(runStepCount, 124, "required and planned job run-step coverage must not shrink"); + // Three mutations per run step: 124*3 = 372. + assert.equal(mutationCount, 372, "exhaustive bypass matrix must not shrink"); }); it("rejects every setup-action condition and soft-failure bypass", () => { @@ -710,28 +726,22 @@ describe("CI preflight contract", () => { } }); - it("locks the reasoning-lens regression to one unconditional raw-Node step", () => { + it("locks the reasoning-lens regression to one raw-Node step gated on npm ci", () => { assert.ok(workflow.includes(reasoningLensRegressionStep), "reasoning-lens regression fixture drifted"); + const message = "exact reasoning-lens regression once and only after npm ci succeeds"; + expectFailure(workflow.replace(reasoningLensRegressionStep, ""), message); expectFailure( - workflow.replace(reasoningLensRegressionStep, ""), - "exact reasoning-lens regression once and unconditionally", + workflow.replace(reasoningLensRegressionStep, reasoningLensRegressionStep.repeat(2)), + message, ); expectFailure( - workflow.replace(reasoningLensRegressionStep, reasoningLensRegressionStep.repeat(2)), - "exact reasoning-lens regression once and unconditionally", + workflow.replace(reasoningLensRegressionStep, addFalseCondition(reasoningLensRegressionStep)), + message, + ); + expectFailure( + workflow.replace(reasoningLensRegressionStep, addContinueOnError(reasoningLensRegressionStep)), + message, ); - for (const bypass of [" if: false\n", " continue-on-error: true\n"]) { - expectFailure( - workflow.replace( - reasoningLensRegressionStep, - reasoningLensRegressionStep.replace( - " run: node --test", - `${bypass} run: node --test`, - ), - ), - "exact reasoning-lens regression once and unconditionally", - ); - } expectFailure( workflow.replace( reasoningLensRegressionStep, @@ -740,7 +750,7 @@ describe("CI preflight contract", () => { "npm run test:reasoning-lens-contract", ), ), - "exact reasoning-lens regression once and unconditionally", + message, ); }); @@ -753,12 +763,14 @@ describe("CI preflight contract", () => { workflow.replace(reasoningLensAdmissionStep, reasoningLensAdmissionStep.repeat(2)), "exact reasoning-lens event admission contract", ); - for (const bypass of [" if: false\n", " continue-on-error: true\n"]) { - expectFailure( - mutateReasoningLensAdmission((step) => step.replace(" shell: bash\n", ` shell: bash\n${bypass}`)), - "exact reasoning-lens event admission contract", - ); - } + expectFailure( + mutateReasoningLensAdmission(addFalseCondition), + "exact reasoning-lens event admission contract", + ); + expectFailure( + mutateReasoningLensAdmission(addContinueOnError), + "exact reasoning-lens event admission contract", + ); for (const [from, to] of [ ["github.event.pull_request.base.sha", "github.event.pull_request.head.sha"], ["github.event.before", "github.sha"], @@ -783,7 +795,7 @@ describe("CI preflight contract", () => { it("rejects Buck2 jobs that do not bootstrap pinned DotSlash before invocation", () => { expectFailure( workflow.replace( - ` - name: Install pinned DotSlash runtime\n if: ${preflightRunHeavyIf}\n run: tools/buck/install_dotslash.sh\n`, + ` - name: Install pinned DotSlash runtime\n id: dotslash\n if: ${preflightCheckoutHeavyIf}\n run: tools/buck/install_dotslash.sh\n`, "", ), "preflight must install pinned DotSlash before Buck2", @@ -807,10 +819,8 @@ describe("CI preflight contract", () => { "Buck2 console-app inline PostgreSQL suites", ]) { expectFailure( - workflow.replace( - ` - name: ${stepName}\n if: ${runHeavyIf}\n working-directory: .\n`, - ` - name: ${stepName}\n if: ${runHeavyIf}\n`, - ), + mutateNamedStep(workflow, "backend", stepName, (step) => + step.replace(" working-directory: .\n", "")), "backend must preserve the locked fail-fast step multiset and failure semantics", ); } @@ -929,12 +939,13 @@ describe("CI preflight contract", () => { }); it("requires backend DotSlash bootstrap before any Buck or DotSlash invocation", () => { + const dotSlashStep = + ` - name: Install pinned DotSlash runtime\n id: dotslash\n if: ${backendIndependentIf}\n run: ../tools/buck/install_dotslash.sh\n`; for (const command of ["tools/buck2 --version", "dotslash run //backend/app:console-app"]) { expectFailure( workflow.replace( - ` - name: Install pinned DotSlash runtime\n if: ${runHeavyIf}\n run: ../tools/buck/install_dotslash.sh\n`, - ` - name: First Buck invocation\n if: ${runHeavyIf}\n run: ${command}\n\n` - + ` - name: Install pinned DotSlash runtime\n if: ${runHeavyIf}\n run: ../tools/buck/install_dotslash.sh\n`, + dotSlashStep, + ` - name: First Buck invocation\n if: ${backendIndependentIf}\n run: ${command}\n\n${dotSlashStep}`, ), "backend must install pinned DotSlash before its first Buck invocation", ); @@ -1008,8 +1019,8 @@ describe("CI preflight contract", () => { it("requires the pinned Rust toolchain before Cargo-dependent preflight tests", () => { expectFailure( workflow.replace(preflightRustToolchainSetup, "").replace( - ` - name: Cargo.lock consistency\n if: ${preflightRunHeavyIf}\n run: ${cargoLockGate}\n`, - ` - name: Cargo.lock consistency\n if: ${preflightRunHeavyIf}\n run: ${cargoLockGate}\n\n${preflightRustToolchainSetup.trimEnd()}\n`, + ` - name: Cargo.lock consistency\n id: cargo-lock\n if: ${preflightRustHeavyIf}\n run: ${cargoLockGate}\n`, + ` - name: Cargo.lock consistency\n id: cargo-lock\n if: ${preflightRustHeavyIf}\n run: ${cargoLockGate}\n\n${preflightRustToolchainSetup.trimEnd()}\n`, ), `preflight must install the pinned Rust toolchain before ${cargoLockGate}`, ); @@ -1023,6 +1034,9 @@ describe("CI preflight contract", () => { }); it("requires explicit exact-M C/T derivation before every normal-PR console admission", () => { + const npmCiIf = "${{ !cancelled() && steps.npm-ci.outcome == 'success' }}"; + const npmCiPrIf = "${{ !cancelled() && steps.derive.outcome == 'success' && steps.npm-ci.outcome == 'success' && github.event_name == 'pull_request' }}"; + const prOnlyIf = "${{ github.event_name == 'pull_request' }}"; expectFailure( workflow.replace(' CONSOLE_AUTHORITY_TIP_SHA="$(git rev-parse "$CONSOLE_SYNTHETIC_MERGE_SHA^2")"\n', ''), "derive exact C/T/M", @@ -1032,7 +1046,7 @@ describe("CI preflight contract", () => { "derive exact C/T/M", ); expectFailure( - workflow.replace(' if: ${{ github.event_name == \'pull_request\' }}\n run: npm run check:console-truth-ledger', ' run: npm run check:console-truth-ledger'), + workflow.replace(` if: ${npmCiPrIf}\n run: npm run check:console-truth-ledger`, ` if: ${npmCiIf}\n run: npm run check:console-truth-ledger`), "exact C/T/M derivation", ); expectFailure( @@ -1044,15 +1058,15 @@ describe("CI preflight contract", () => { "derive exact C/T/M", ); expectFailure( - workflow.replace(' run: node --test scripts/console/validate-console-truth-ledger.test.mjs', ' if: ${{ github.event_name == \'pull_request\' }}\n run: node --test scripts/console/validate-console-truth-ledger.test.mjs'), + workflow.replace(` if: ${npmCiIf}\n run: node --test scripts/console/validate-console-truth-ledger.test.mjs`, ` if: ${prOnlyIf}\n run: node --test scripts/console/validate-console-truth-ledger.test.mjs`), "validate-console-truth-ledger.test.mjs", ); expectFailure( - workflow.replace(' run: node --test scripts/console/plan-fanout.test.mjs', ' if: ${{ github.event_name == \'pull_request\' }}\n run: node --test scripts/console/plan-fanout.test.mjs'), + workflow.replace(` if: ${npmCiIf}\n run: node --test scripts/console/plan-fanout.test.mjs`, ` if: ${prOnlyIf}\n run: node --test scripts/console/plan-fanout.test.mjs`), "plan-fanout.test.mjs", ); expectFailure( - workflow.replace(' - name: Console authority-train regression\n run: node --test scripts/console/verify-console-authority-train.test.mjs\n\n', ''), + workflow.replace(` - name: Console authority-train regression\n id: authority-train\n if: ${npmCiIf}\n run: node --test scripts/console/verify-console-authority-train.test.mjs\n\n`, ''), "verify-console-authority-train.test.mjs", ); // This suite gates the `pull_request_target` bootstrap verifier — the highest-privilege @@ -1061,19 +1075,19 @@ describe("CI preflight contract", () => { // turned every one of its tests red locally while CI stayed green. Wiring it into ci.yml is // not the same as protecting it, hence both halves below. assert.ok( - workflow.includes(' - name: Console PR authority bootstrap regression\n run: node --test scripts/console/verify-console-pr-authority-bootstrap.test.mjs scripts/console/release-please-bot-candidate.test.mjs\n'), + workflow.includes(` - name: Console PR authority bootstrap regression\n id: pr-authority-bootstrap\n if: ${npmCiIf}\n run: node --test scripts/console/verify-console-pr-authority-bootstrap.test.mjs scripts/console/release-please-bot-candidate.test.mjs\n`), "preflight does not run the console PR authority bootstrap regression", ); expectFailure( - workflow.replace(' - name: Console PR authority bootstrap regression\n run: node --test scripts/console/verify-console-pr-authority-bootstrap.test.mjs scripts/console/release-please-bot-candidate.test.mjs\n\n', ''), + workflow.replace(` - name: Console PR authority bootstrap regression\n id: pr-authority-bootstrap\n if: ${npmCiIf}\n run: node --test scripts/console/verify-console-pr-authority-bootstrap.test.mjs scripts/console/release-please-bot-candidate.test.mjs\n\n`, ''), "verify-console-pr-authority-bootstrap.test.mjs", ); expectFailure( - workflow.replace(' run: node --test scripts/console/verify-console-pr-authority-bootstrap.test.mjs scripts/console/release-please-bot-candidate.test.mjs', ' if: ${{ github.event_name == \'pull_request\' }}\n run: node --test scripts/console/verify-console-pr-authority-bootstrap.test.mjs scripts/console/release-please-bot-candidate.test.mjs'), + workflow.replace(` if: ${npmCiIf}\n run: node --test scripts/console/verify-console-pr-authority-bootstrap.test.mjs scripts/console/release-please-bot-candidate.test.mjs`, ` if: ${prOnlyIf}\n run: node --test scripts/console/verify-console-pr-authority-bootstrap.test.mjs scripts/console/release-please-bot-candidate.test.mjs`), "verify-console-pr-authority-bootstrap.test.mjs", ); expectFailure( - workflow.replace(' if: ${{ github.event_name == \'pull_request\' }}\n run: node scripts/console/plan-fanout.mjs', ' run: node scripts/console/plan-fanout.mjs'), + workflow.replace(` if: ${npmCiPrIf}\n run: node scripts/console/plan-fanout.mjs`, ` if: ${npmCiIf}\n run: node scripts/console/plan-fanout.mjs`), "plan-fanout.mjs", ); expectFailure( @@ -1094,22 +1108,22 @@ describe("CI preflight contract", () => { "tools/buck/run_test_with_postgres_env.test.sh", "tools/buck/test_needs_postgres.test.sh", ]) { - const gated = ` if: ${preflightRunHeavyIf}\n run: ${command}\n`; + const gated = ` if: ${preflightBuckHeavyIf}\n run: ${command}\n`; assert.ok(workflow.includes(gated), `missing gated reachability command ${command}`); expectFailure(workflow.replace(gated, ""), command); - expectFailure(workflow.replace(gated, ` if: ${preflightRunHeavyIf}\n # ${command}\n`), command); + expectFailure(workflow.replace(gated, ` if: ${preflightBuckHeavyIf}\n # ${command}\n`), command); } }); it("rejects conditional and continue-on-error reachability regressions", () => { const alwaysCommand = "node --test scripts/console/route-inventory.test.mjs"; expectFailure( - workflow.replace(` run: ${alwaysCommand}\n`, ` if: \${{ false }}\n run: ${alwaysCommand}\n`), - "unconditionally", + workflow.replace(` if: ${npmCiIf}\n run: ${alwaysCommand}\n`, ` if: \${{ false }}\n run: ${alwaysCommand}\n`), + "only when", ); expectFailure( - workflow.replace(` run: ${alwaysCommand}\n`, ` continue-on-error: true\n run: ${alwaysCommand}\n`), - "unconditionally", + workflow.replace(` if: ${npmCiIf}\n run: ${alwaysCommand}\n`, ` if: ${npmCiIf}\n continue-on-error: true\n run: ${alwaysCommand}\n`), + "only when", ); for (const command of [ "tools/buck/run_test_with_postgres_env.test.sh", @@ -1117,15 +1131,15 @@ describe("CI preflight contract", () => { ]) { expectFailure( workflow.replace( - ` if: ${preflightRunHeavyIf}\n run: ${command}\n`, + ` if: ${preflightBuckHeavyIf}\n run: ${command}\n`, ` if: \${{ false }}\n run: ${command}\n`, ), "only when", ); expectFailure( workflow.replace( - ` if: ${preflightRunHeavyIf}\n run: ${command}\n`, - ` if: ${preflightRunHeavyIf}\n continue-on-error: true\n run: ${command}\n`, + ` if: ${preflightBuckHeavyIf}\n run: ${command}\n`, + ` if: ${preflightBuckHeavyIf}\n continue-on-error: true\n run: ${command}\n`, ), "only when", ); @@ -1174,8 +1188,8 @@ describe("CI preflight contract", () => { it("rejects a preflight command that appears only in a comment", () => { expectFailure( workflow.replace( - " - name: Canonical npm lockfile\n run: npm run check:package-lock", - " - name: Canonical npm lockfile\n # npm run check:package-lock", + ` if: ${npmCiIf}\n run: npm run check:package-lock`, + ` if: ${npmCiIf}\n # npm run check:package-lock`, ), "check:package-lock", ); @@ -1184,20 +1198,20 @@ describe("CI preflight contract", () => { it("rejects a required preflight step guarded by a condition", () => { expectFailure( workflow.replace( - " - name: Canonical npm lockfile\n run: npm run check:package-lock", - " - name: Canonical npm lockfile\n if: ${{ false }}\n run: npm run check:package-lock", + ` if: ${npmCiIf}\n run: npm run check:package-lock`, + ` if: \${{ false }}\n run: npm run check:package-lock`, ), - "unconditionally", + "only when", ); }); it("rejects a required preflight step allowed to continue on error", () => { expectFailure( workflow.replace( - " - name: Canonical npm lockfile\n run: npm run check:package-lock", - " - name: Canonical npm lockfile\n continue-on-error: true\n run: npm run check:package-lock", + ` if: ${npmCiIf}\n run: npm run check:package-lock`, + ` if: ${npmCiIf}\n continue-on-error: true\n run: npm run check:package-lock`, ), - "unconditionally", + "only when", ); }); @@ -1581,7 +1595,7 @@ describe("CI preflight contract", () => { expectFailure( mutateNamedStep(workflow, "domain-unit", "Domain crate unit tests", (step) => step.replace(/^ run: \|$/m, " run: |\n exit 0")), - "domain-unit must execute the locked Cargo test commands directly when run_heavy", + "domain-unit proof run step 2 must preserve", ); for (const condition of ["false", "${{ false }}"]) { expectFailure( @@ -1724,24 +1738,18 @@ describe("CI preflight contract", () => { "backend must preserve the locked fail-fast step multiset and failure semantics", ); expectFailure( - workflow.replace( - ` - name: Audit-coverage gate\n if: ${runHeavyIf}\n`, - ` - name: Audit-coverage gate\n if: \${{ false }}\n`, - ), + mutateNamedStep(workflow, "backend", "Audit-coverage gate", addFalseCondition), "backend must preserve the locked fail-fast step multiset and failure semantics", ); expectFailure( - workflow.replace( - ` - name: Migration-safety gate\n if: ${runHeavyIf}\n`, - ` - name: Migration-safety gate\n if: ${runHeavyIf}\n continue-on-error: true\n`, - ), + mutateNamedStep(workflow, "backend", "Migration-safety gate", addContinueOnError), "backend must preserve the locked fail-fast step multiset and failure semantics", ); expectFailure( - workflow.replace( - ` - name: Audit-coverage gate\n if: ${runHeavyIf}\n`, - ` - name: Layer-boundary gate\n if: \${{ !cancelled() }}\n run: ../tools/buck2 run //backend/ci/gates/layer-boundary:console-gate-layer-boundary\n\n - name: Audit-coverage gate\n if: ${runHeavyIf}\n`, - ), + replaceJob(workflow, "backend", (block) => block.replace( + " - name: Layer-boundary gate\n", + " - name: Layer-boundary gate\n run: ../tools/buck2 run //backend/ci/gates/layer-boundary:console-gate-layer-boundary\n\n - name: Layer-boundary gate\n", + )), "backend must preserve the locked fail-fast step multiset and failure semantics", ); expectFailure( @@ -1760,13 +1768,13 @@ describe("CI preflight contract", () => { ); }); - it("keeps protected backend steps fail-fast and runs PR 473 contract tests before topology", () => { + it("keeps protected backend steps fail-slow and runs PR 473 contract tests before topology", () => { expectFailure( workflow.replace( - ` - name: rustfmt check\n if: ${runHeavyIf}\n`, - ` - name: rustfmt check\n if: \${{ !cancelled() }}\n`, + ` - name: rustfmt check\n id: fmt\n if: ${backendIndependentIf}\n`, + ` - name: rustfmt check\n id: fmt\n if: \${{ !cancelled() }}\n`, ), - "backend must not use !cancelled() on protected fail-fast steps", + "backend proof run step 3 must preserve", ); expectFailure( workflow.replace( @@ -1897,7 +1905,8 @@ describe("CI preflight contract", () => { const run = " run: env -u DATABASE_URL tools/buck2 test" + " //backend/app:console-app-itest-openapi_drift\n"; const step = " - name: Buck2 console-app OpenAPI drift suite\n" - + ` if: ${runHeavyIf}\n` + + " id: openapi-drift\n" + + ` if: ${backendIndependentIf}\n` + " working-directory: .\n" + run; assert.ok(workflow.includes(step), "backend does not run the openapi_drift suite"); @@ -1914,13 +1923,13 @@ describe("CI preflight contract", () => { + " //backend/app:console-app-unit\n"), "backend must preserve the locked fail-fast step multiset and failure semantics", ); - // `if: ${{ !cancelled() }}` here would let a red drift suite pass the job as a soft warning. + // Dropping the run_heavy half of the guard would let the drift suite run on docs-only. expectFailure( workflow.replace( - ` - name: Buck2 console-app OpenAPI drift suite\n if: ${runHeavyIf}\n`, - " - name: Buck2 console-app OpenAPI drift suite\n if: ${{ !cancelled() }}\n", + ` - name: Buck2 console-app OpenAPI drift suite\n id: openapi-drift\n if: ${backendIndependentIf}\n`, + ` - name: Buck2 console-app OpenAPI drift suite\n id: openapi-drift\n if: \${{ !cancelled() }}\n`, ), - "backend must not use !cancelled() on protected fail-fast steps", + "backend must preserve the locked fail-fast step multiset and failure semantics", ); }); }); diff --git a/scripts/ci-collect-failures.mjs b/scripts/ci-collect-failures.mjs new file mode 100644 index 000000000..a49f0acda --- /dev/null +++ b/scripts/ci-collect-failures.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node +// Fail-slow sweep collector: print the failing step ids from a job's `steps` +// context and exit 1 if any step failed. A step that was SKIPPED (its `if` +// evaluated false, e.g. a dependent step whose dependency failed) is not a +// failure — only `outcome === "failure"` is. This re-asserts the job-level red +// that feeds `Required / CI` and names the root failure(s) so a lane sees the +// whole sweep in one look. +// +// The `steps` context is passed via CI_STEPS (`${{ toJSON(steps) }}` in the +// workflow step's env), because `steps.*.outcome` cannot be iterated from bash. + +const steps = JSON.parse(process.env.CI_STEPS || "{}"); +const failed = Object.entries(steps) + .filter(([, step]) => step && step.outcome === "failure") + .map(([id]) => id) + .sort(); + +if (failed.length > 0) { + console.error(`failed steps: ${failed.join(", ")}`); + process.exit(1); +} +console.log("all steps passed"); diff --git a/scripts/lib/ci-workflow-executables.mjs b/scripts/lib/ci-workflow-executables.mjs index d9a80624d..df731f0d4 100644 --- a/scripts/lib/ci-workflow-executables.mjs +++ b/scripts/lib/ci-workflow-executables.mjs @@ -300,6 +300,11 @@ function collectWorkflowCommands(workflow, includeNonGating) { // Effective shell: step override, else job default (direct or defaults.run), // else workflow defaults.run, else the runner default. const shell = yamlValue(step, "shell", 8) ?? jobShell ?? workflowShell ?? null; + // A fail-slow keep-going block collects per-invocation failures and re-raises + // them with a summary `exit 1`, so a `set +e` inside it does NOT make the cargo + // runs non-gating. The `ci-keep-going:` comment is an explicit contract on the + // run body (see the domain-unit job); without it, `set +e` stays disqualifying. + const keepGoing = /ci-keep-going:/.test(script); let terminated = false; let errexitDisabled = false; for (const surface of shellCommandTokens(script)) { @@ -326,7 +331,7 @@ function collectWorkflowCommands(workflow, includeNonGating) { step, tokens, malformed: false, - controlFlow: controlFlow || errexitDisabled, + controlFlow: controlFlow || (errexitDisabled && !keepGoing), gating, shell, }); diff --git a/scripts/verify.mjs b/scripts/verify.mjs index 8fb93a312..66be0e984 100644 --- a/scripts/verify.mjs +++ b/scripts/verify.mjs @@ -146,6 +146,13 @@ const PLAN = new Map([ tier: "ci-only", why: "skip proofs run only when run_heavy!=true on hosted CI; local mirror always exercises the heavy path / docs_only is CI scheduling", }], + // Fail-slow sweep collector: reads the GHA `steps` context (toJSON(steps)), + // which exists only on a hosted runner; the same job-level red is produced + // locally by the run step itself. + ["Collect failures", { + tier: "ci-only", + why: "reads the hosted steps context (toJSON(steps)); no local equivalent", + }], // ---- backend ----------------------------------------------------------- ["rustfmt check", { tier: "fast" }], diff --git a/tools/ci/cargo-test-runner.sh b/tools/ci/cargo-test-runner.sh new file mode 100755 index 000000000..cd6ba8eab --- /dev/null +++ b/tools/ci/cargo-test-runner.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Run a stream of cargo test invocations (JSONL on stdin) with fail-slow +# keep-going semantics. Each stdin line is one JSON object: +# {"name": "...", "package": "...", "argv": ["cargo", "test", ...]} +# +# Extracted from cargo_needs_postgres.sh so the keep-going loop is unit-testable +# without Docker or a real Cargo workspace: feed it JSONL and point PATH at a +# stubbed `cargo`. +# +# Exit 0 only when every invocation passed; exit 1 when any failed (the workflow +# relies on non-zero == red). Keep-going is the default; --fail-fast aborts after +# the first failure (local use). `set -e` is deliberately OUT of the per-binary +# loop: failures are captured explicitly so one red binary never hides the rest. +# +# Environment (set by the caller): +# CARGO_REPO_ROOT repository root used as the cargo cwd +# RUST_TEST_THREADS number of test threads per invocation (default 1) +set -uo pipefail + +repo_root="${CARGO_REPO_ROOT:-}" +threads="${RUST_TEST_THREADS:-1}" +fail_fast=0 +case "${1:-}" in + --keep-going) fail_fast=0 ;; + --fail-fast) fail_fast=1 ;; + "") + ;; + *) + echo "usage: cargo-test-runner.sh [--keep-going|--fail-fast]" >&2 + exit 2 + ;; +esac + +if [[ -z "${repo_root}" ]]; then + echo "cargo-test-runner: CARGO_REPO_ROOT must name the repository root" >&2 + exit 2 +fi + +passed=0 +failed=0 +results=() + +while IFS= read -r row || [[ -n "${row}" ]]; do + name="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["name"])' "${row}")" + echo "cargo-postgres: === ${name} ===" + # JSON array exec avoids the word-splitting bugs of `set -- $(...)`; the argv + # is taken from the map entry verbatim. + if python3 - "${repo_root}" "${row}" "${threads}" <<'PY' +import json, os, subprocess, sys +root, row, threads = sys.argv[1], json.loads(sys.argv[2]), sys.argv[3] +argv = row["argv"] +env = os.environ.copy() +env["SQLX_OFFLINE"] = "true" +env["RUST_TEST_THREADS"] = threads +env["CARGO_TERM_COLOR"] = "always" +print("cargo-postgres:", " ".join(argv), flush=True) +raise SystemExit(subprocess.call(argv, cwd=root, env=env)) +PY + then + passed=$((passed + 1)) + results+=("PASS ${name}") + else + failed=$((failed + 1)) + results+=("FAIL ${name}") + if [[ "${fail_fast}" == 1 ]]; then + break + fi + fi +done + +echo "" +echo "cargo-postgres: summary" +for entry in "${results[@]}"; do + printf ' %s\n' "${entry}" +done +echo "cargo-postgres: ${passed} passed, ${failed} failed" + +if [[ "${failed}" != 0 ]]; then + echo "cargo-postgres: one or more invocations failed" >&2 + exit 1 +fi +echo "cargo-postgres: all ${passed} invocations passed" diff --git a/tools/ci/cargo-test-runner.test.mjs b/tools/ci/cargo-test-runner.test.mjs new file mode 100644 index 000000000..fea8cfae0 --- /dev/null +++ b/tools/ci/cargo-test-runner.test.mjs @@ -0,0 +1,113 @@ +#!/usr/bin/env node +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const runner = resolve(dirname(fileURLToPath(import.meta.url)), "cargo-test-runner.sh"); + +/** + * Build a temp dir containing a stubbed `cargo` on `bin/`. The stub logs every + * invocation and exits 1 when any argument contains `fail`, else 0 — so a fake + * map of JSONL rows can exercise the keep-going loop without Docker or cargo. + */ +function setupFixture() { + const root = mkdtempSync(join(tmpdir(), "cargo-test-runner-")); + const bin = join(root, "bin"); + mkdirSync(bin, { recursive: true }); + const cargo = join(bin, "cargo"); + writeFileSync( + cargo, + [ + "#!/usr/bin/env bash", + 'printf \'%s\\n\' "$*" >> "${CARGO_INVOCATION_LOG}"', + 'case "$*" in', + " *fail*) exit 1 ;;", + " *) exit 0 ;;", + "esac", + "", + ].join("\n"), + ); + chmodSync(cargo, 0o755); + return { root, bin, log: join(root, "cargo.log") }; +} + +function runRunner({ root, bin, log, rows, args = [] }) { + return spawnSync(runner, args, { + input: rows.map((row) => JSON.stringify(row)).join("\n") + "\n", + encoding: "utf8", + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + CARGO_REPO_ROOT: root, + RUST_TEST_THREADS: "1", + CARGO_INVOCATION_LOG: log, + }, + }); +} + +const ROWS = [ + { name: "pass-a", package: "pkg-pass-a", argv: ["cargo", "test", "-p", "pkg-pass-a"] }, + { name: "fail-b", package: "pkg-fail-b", argv: ["cargo", "test", "-p", "pkg-fail-b"] }, + { name: "pass-c", package: "pkg-pass-c", argv: ["cargo", "test", "-p", "pkg-pass-c"] }, +]; + +test("keep-going (default) runs every invocation, collects failures, exits 1", () => { + const { root, bin, log } = setupFixture(); + try { + const result = runRunner({ root, bin, log, rows: ROWS }); + const logLines = readFileSync(log, "utf8").trim().split("\n"); + assert.equal(result.status, 1, result.stdout + result.stderr); + // every binary ran despite the middle failure + assert.deepEqual(logLines, [ + "test -p pkg-pass-a", + "test -p pkg-fail-b", + "test -p pkg-pass-c", + ]); + assert.match(result.stdout, /PASS {2}pass-a/); + assert.match(result.stdout, /FAIL {2}fail-b/); + assert.match(result.stdout, /PASS {2}pass-c/); + assert.match(result.stdout, /2 passed, 1 failed/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("--fail-fast aborts at the first failure and skips the rest", () => { + const { root, bin, log } = setupFixture(); + try { + const result = runRunner({ root, bin, log, rows: ROWS, args: ["--fail-fast"] }); + const logLines = readFileSync(log, "utf8").trim().split("\n"); + assert.equal(result.status, 1, result.stdout + result.stderr); + assert.deepEqual(logLines, ["test -p pkg-pass-a", "test -p pkg-fail-b"]); + assert.match(result.stdout, /PASS {2}pass-a/); + assert.match(result.stdout, /FAIL {2}fail-b/); + assert.match(result.stdout, /1 passed, 1 failed/); + assert.doesNotMatch(result.stdout, /pass-c/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("exit 0 only when every invocation passed", () => { + const { root, bin, log } = setupFixture(); + try { + const rows = ROWS.filter((row) => !row.package.includes("fail")); + const result = runRunner({ root, bin, log, rows }); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.match(result.stdout, /2 passed, 0 failed/); + assert.match(result.stdout, /all 2 invocations passed/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/ci/cargo_needs_postgres.sh b/tools/ci/cargo_needs_postgres.sh index 5cc18f0ed..89b5cd162 100755 --- a/tools/ci/cargo_needs_postgres.sh +++ b/tools/ci/cargo_needs_postgres.sh @@ -9,9 +9,10 @@ only_csv="" workflow_only=0 num_threads=1 shard_id="" +keep_going=1 usage() { - echo "usage: cargo_needs_postgres.sh [--map PATH] [--workflow-only] [--only name[,name...]] [--shard-id app|platform|ontology|domain-a|domain-b] [--num-threads N]" >&2 + echo "usage: cargo_needs_postgres.sh [--map PATH] [--workflow-only] [--only name[,name...]] [--shard-id app|platform|ontology|domain-a|domain-b] [--num-threads N] [--keep-going|--fail-fast]" >&2 exit 2 } @@ -26,6 +27,8 @@ while [[ $# -gt 0 ]]; do --shard-id=*) shard_id="${1#*=}"; shift ;; --num-threads) num_threads="$2"; shift 2 ;; --num-threads=*) num_threads="${1#*=}"; shift ;; + --keep-going) keep_going=1; shift ;; + --fail-fast) keep_going=0; shift ;; -h|--help) usage ;; *) usage ;; esac @@ -233,33 +236,17 @@ done <"${tmp_pkgs}" echo "cargo-postgres: building packages..." ( cd "${repo_root}" && "${build_args[@]}" ) -failed=0 -while IFS= read -r row; do - name="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["name"])' "${row}")" - echo "cargo-postgres: === ${name} ===" - # shellcheck disable=SC2046 - set -- $(python3 -c 'import json,sys; print(" ".join(json.loads(sys.argv[1])["argv"]))' "${row}") - # Prefer JSON array exec without word-splitting bugs: - if ! python3 - "${repo_root}" "${row}" "${num_threads}" <<'PY' -import json, os, subprocess, sys -root, row, threads = sys.argv[1], json.loads(sys.argv[2]), sys.argv[3] -argv = row["argv"] -env = os.environ.copy() -env["SQLX_OFFLINE"] = "true" -env["RUST_TEST_THREADS"] = threads -env["CARGO_TERM_COLOR"] = "always" -print("cargo-postgres:", " ".join(argv), flush=True) -raise SystemExit(subprocess.call(argv, cwd=root, env=env)) -PY - then - echo "cargo-postgres: FAILED: ${name}" >&2 - failed=1 - fi -done <"${tmp_list}" - -rm -f "${tmp_list}" "${tmp_pkgs}" -if [[ "${failed}" != 0 ]]; then - echo "cargo-postgres: one or more tests failed" >&2 - exit 1 +# Fail-slow sweep: run every selected binary, collect per-binary pass/fail, print +# a summary table, and exit non-zero if any failed. The keep-going loop lives in +# cargo-test-runner.sh so it is unit-testable without Docker (fake map + stubbed +# cargo). Default is --keep-going; --fail-fast opts back out for local use. +export CARGO_REPO_ROOT="${repo_root}" +export RUST_TEST_THREADS="${num_threads}" +if [[ "${keep_going}" == 1 ]]; then + "${repo_root}/tools/ci/cargo-test-runner.sh" --keep-going <"${tmp_list}" +else + "${repo_root}/tools/ci/cargo-test-runner.sh" --fail-fast <"${tmp_list}" fi -echo "cargo-postgres: all ${count} invocations passed" +status=$? +rm -f "${tmp_list}" "${tmp_pkgs}" +exit "${status}" From e0e39b11170a3c1521234fc746828f3a3379c267 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Sat, 15 Aug 2026 06:58:49 -0400 Subject: [PATCH 2/2] docs(ledger): CI fail-slow one-sweep (PR-0) --- .../ledger/2026-08-15-ci-fail-slow-sweep.md | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 docs/program/ledger/2026-08-15-ci-fail-slow-sweep.md diff --git a/docs/program/ledger/2026-08-15-ci-fail-slow-sweep.md b/docs/program/ledger/2026-08-15-ci-fail-slow-sweep.md new file mode 100644 index 000000000..d90911541 --- /dev/null +++ b/docs/program/ledger/2026-08-15-ci-fail-slow-sweep.md @@ -0,0 +1,104 @@ +# Authority tip — CI fail-slow one-sweep (PR-0, D4) + +**Date:** 2026-08-15 +**Kind:** authority tip ledger bound on candidate C; T adds this ledger entry only +**Base:** `e21c04af2` (origin/main, post #773 parallel-development-framework; rebased over #768, #779, #780, #782, #783) +**Candidate C and Tip T:** both signed by the pinned authority (principal `jason19931225@gmail.com`, ED25519 `SHA256:5grGNUtX9Zgmy1SWne6wF9DR8W1ElUQaF/Z8SYRz8E8`). Commit/tree SHAs are recorded in the post-merge readback update (T cannot name its own SHA). +**Scope:** `.github/workflows/ci.yml` (step-level fail-slow guards + per-job `collect-failures` steps + the `domain-unit` keep-going block), `tools/ci/cargo_needs_postgres.sh` + `tools/ci/cargo-test-runner.sh` (new; the extracted keep-going loop) + `tools/ci/cargo-test-runner.test.mjs` (new unit test), `scripts/lib/ci-workflow-executables.mjs` (keep-going attribution for `check-executed-tests`), `scripts/ci-collect-failures.mjs` (new), `scripts/verify.mjs` (classify the new `collect-failures` step in the local mirror), `scripts/check-ci-preflight.mjs` + `scripts/check-ci-preflight.test.mjs` (the contract is updated to lock the new fail-slow semantics), and `package.json` (wire the new test into `check:ci-preflight`). No production code, no migration, no Cargo.toml/lockfile, no OpenAPI, no Buck target. +**Not product authority.** Clears no HOLD and authorizes no production, credential, compliance, payment, erase, OCI, frontend, projection, or Oyatie action. + +## Summary + +- **One sweep, one commit, one push.** Every CI run now surfaces ALL of fmt, clippy, every gate, and every test binary in a single run, so a lane fixes everything at once instead of polling micro-commits. +- **Step-level fail-slow in `preflight` and `backend`.** Independent steps carry `if: ${{ !cancelled() }}`; steps that depend on a setup root carry `if: ${{ !cancelled() && steps..outcome == 'success' }}`. The dependency roots are `checkout` / `setup-node` / `npm-ci` / `dotslash` / `rust-toolchain` in preflight, and `topology` (the reconcile step) in backend. When `npm ci` fails the ~20 npm gates skip instead of cascading 20 reds; when topology reconcile fails the DB-dependent backend steps skip. +- **`collect-failures` closes each of the two jobs.** It reads `${{ toJSON(steps) }}` via `scripts/ci-collect-failures.mjs`, prints the `outcome == "failure"` step ids, and exits 1, so the job-level red that feeds `Required / CI` is preserved and the root failure is named. +- **`cargo_needs_postgres.sh` gains an explicit keep-going mode.** Default `--keep-going` (workflow), opt-out `--fail-fast` (local). The per-binary loop moved to `tools/ci/cargo-test-runner.sh` so it is unit-testable without Docker (fake map + stubbed cargo), and now prints a per-binary PASS/FAIL summary table before exiting 1 on any failure. +- **`domain-unit`'s inline block no longer aborts at the first failing binary.** It keeps `set -e` out of the loop, captures each cargo invocation's status, prints a summary, and exits 1 if any failed — while keeping every `cargo test` invocation text byte-for-byte so `check-executed-tests` still attributes every binary (the `ci-keep-going:` contract in `scripts/lib/ci-workflow-executables.mjs` makes `set +e` non-disqualifying because the summary re-raises). +- **The CI-preflight contract is updated, not weakened.** `scripts/check-ci-preflight.mjs` re-locks the new step ids, conditions, and the two `collect-failures` steps (including their `env` and `working-directory`), the new `domain-unit` run digest, and the backend's two fail-slow condition families; the exhaustive bypass-mutation matrix is re-armed at the new step counts. + +## Verification + +Executed on the rebased worktree at base `e21c04af2` (node v24.16.0, cargo/rustc 1.97.1). Rows marked **(ceremony)** are the exact commands recorded for the conductor. + +| Command | Result | +|---|---| +| `node --test tools/ci/cargo-test-runner.test.mjs` | 3 passed / 0 failed — keep-going runs every invocation, `--fail-fast` aborts at the first failure, exit 0 only when all pass | +| `node --test scripts/check-ci-preflight.test.mjs` | 57 passed / 0 failed | +| `node scripts/check-ci-preflight.mjs` | `CI preflight contract passed.` | +| `node --test scripts/verify.test.mjs` | 13 passed / 0 failed (local mirror covers the new `collect-failures` step) | +| `npm run check:ci-preflight` | exit 0 (check-ci-preflight + tools/ci tests + dark-suite `--strict` + foundation + mail-relay + verification-queue) | +| `node tools/ci/check-mjs-dark-suites.mjs --strict` | `dark_count: 0` | +| `npm run test:js-test-reachability` | 2 passed / 0 failed | +| `node scripts/check-executed-tests.mjs` | green — 364 defined, 364 reachable, 1 dark (`seaweedfs_worm.rs`, baseline-pinned) | +| `node scripts/check-js-test-reachability.mjs` | green (baseline + live candidate) | +| `node -e "… yaml.load(ci.yml)"` | YAML parses (js-yaml) | +| `actionlint .github/workflows/ci.yml` | clean | +| `npm run verify` (fast tier) | all mirrored Rust/gate/test suites pass; exits 1 only on two expected uncommitted/unsigned artifacts — `Cheap Buck2 generated-face admission` (package.json digest vs `git archive HEAD`, because the change is staged not committed) and `Console truth-ledger exact-M admission` (no signed C commit yet). Both resolve after the conductor commits + signs the C/T train. | +| `git diff --check` | clean | + +## Operational receipt (lane-specific) + +- **Lane:** ci/fail-slow-one-sweep-pr0 · **Worktree:** `/Users/jasonlee/Developer/console-pr0-fail-slow` · **Owner:** Jason Lee. +- **Pre-mortem:** a step-guard typo silently skips a gate (red becomes grey with no job red); the keep-going `set +e` makes `check-executed-tests` stop attributing a binary and the ratchet goes dark; the `collect-failures` `toJSON(steps)` leaks a step output or misreads `skipped` as a failure. +- **Blast radius:** `.github/workflows/*`, `tools/ci/*`, `scripts/check-ci-preflight.mjs` + test, `scripts/lib/ci-workflow-executables.mjs`, `scripts/ci-collect-failures.mjs`, `package.json`. Revert = re-merge the previous train; no ruleset edit, no `required-ci` `needs:` change. +- **Detection:** `check-ci-preflight.mjs` (exact step id/condition/digest/order lock + the exhaustive mutation matrix), `check-executed-tests.mjs` (the dark-set ratchet still sees every binary), `check-mjs-dark-suites.mjs --strict`, and `cargo-test-runner.test.mjs`. +- **Rollback:** revert the C then T commits; base `e21c04af2` introduces no migration from this lane. +- **Stop conditions:** any required check red on the merge ref; a `check-executed-tests` dark-set drift; an un-resolved review thread; loss of the pinned signing authority. +- **Review identities:** lane owner + signing principal `Jason Lee` / `jason19931225@gmail.com` (ED25519 `SHA256:5grGNUtX9Zgmy1SWne6wF9DR8W1ElUQaF/Z8SYRz8E8`); Codex connector automated review (PR #784, login `chatgpt-codex-connector[bot]`); conductor ratification at merge (recorded in the post-merge readback). +- **Head SHA at freeze:** recorded in the post-merge readback update (self-reference). + +## Freeze status + +**NOT FROZEN YET.** This tip freezes in the post-merge readback update after the hosted required checks (`Required / CI`, `Required / Security`, `authenticate-console-authority`) report green on the merge ref. + +## Remaining HOLDs / follow-ups + +- All current PRODUCT/ROADMAP HOLDs remain unchanged. +- D1 path-phased lanes, D2 warm-cache program, and D3 duplicate/buck2 retirement remain queued per `.tmp/lane-packets/ci-overhaul.md`; this PR is D4 only. +- Rebasing onto #773 reconciled the backend `Checkout` step (kept #773's unconditional index-0 checkout, added `id: checkout`, dropped the `!cancelled() && run_heavy` guard); #768's skip-proof working-directory note is preserved. + + +```json +{ + "lens_contract": "v1", + "lens_contract_digest": "ac1e7d6b8150808ef73e5e3cd1a1e54d2f37eb43e84aaa1370dbbaaff3c44373", + "task_class": "implementation", + "risk_class": "standard", + "risk_domains": [], + "selected_lenses": [ + "Cartesian doubt", + "Essentialism / YAGNI", + "Chesterton's Fence", + "Red Team", + "Systems Thinking", + "Operability / Day-2", + "Blast-radius / cell-based", + "Telemetry-first" + ], + "task_fit": { + "Cartesian doubt": "Re-checked the load-bearing claim that check-executed-tests must keep attributing every binary after the keep-going restructure, by running it (362 defined / 362 reachable) rather than assuming the `set +e` restructure was parser-transparent.", + "Essentialism / YAGNI": "Scoped to D4 only: step guards, two collect-failures steps, the keep-going harness, and the contract update — no path-phased lanes, no warm-cache program, no buck2 retirement, no ruleset edit.", + "Chesterton's Fence": "The fail-fast lock in check-ci-preflight.mjs existed because a skipped/failed step is a false-green surface; the change preserves that intent by re-locking the new fail-slow semantics rather than deleting the ratchet.", + "Red Team": "Modeled the `set +e` false-green (a failing cargo test swallowed with no re-raise) and closed it by requiring the summary `exit 1` and by making the keep-going contract explicit in the parser; modeled the collect-failures skipped-vs-failed misread and keyed it on `outcome === 'failure'`.", + "Systems Thinking": "Traced the two dependency DAGs (preflight: checkout → npm-ci → gates; backend: topology → DB steps) so one root failure skips its dependents instead of cascading reds, and the summary step still preserves the job-level red.", + "Operability / Day-2": "The collect-failures step and the summary tables make the sweep observable in one look; the keep-going loop is extracted into a unit-testable script instead of staying inline-only.", + "Blast-radius / cell-based": "Contained failures per job: a failing fmt/clippy/gate no longer masks the rest, and a failing topology reconcile skips only its DB dependents; the change is revertible as one train.", + "Telemetry-first": "collect-failures prints the exact failing step ids, cargo-test-runner prints a per-binary PASS/FAIL table, and both feed the existing required-check red rather than a separate signal." + }, + "mandatory_lens_exceptions": {}, + "findings": [ + "preflight and backend were fail-fast at the step level: one red step skipped every later step, so a lane needed one commit and one push per failure class.", + "cargo_needs_postgres.sh and the domain-unit block aborted at the first failing binary, hiding later binaries in the same shard.", + "The check-ci-preflight contract had locked the exact fail-fast conditions, so the fail-slow sweep had to re-lock the contract (ids, conditions, digests, and the two new collect-failures steps) in the same change.", + "A literal `set +e` in a step would make check-executed-tests stop attributing its cargo runs; the keep-going contract in the parser re-establishes attribution because the summary re-raises failures." + ], + "decisions_changed_or_rejected": [ + "Rejected adding workflow-level `paths:` filters and renaming job display names: both would un-require required contexts.", + "Rejected a comment-only keep-going marker with no enforcement: check-ci-preflight now requires the `ci-keep-going:` contract AND the summary `exit 1`.", + "Rejected moving the domain-unit cargo list into a separate map/script: the spec requires the inline block with verbatim invocation text so check-executed-tests keeps attributing every binary.", + "Rejected a per-dependent-step marker echo: the skip is the GHA skipped state and collect-failures names the root failure, avoiding 20 marker steps." + ], + "lens_set_changes": [] +} +``` +