From 47a707ad433ec1873477c168f2aa0a2cd19834ca Mon Sep 17 00:00:00 2001 From: matthew Date: Sat, 8 Aug 2026 09:30:38 -0700 Subject: [PATCH 001/166] docs: the ADR link 404s for everyone reading on the web `docs/plugins.md` is synced to the published docs site. `docs/adr/` is not: it is a repository-only record. So a relative link from one to the other resolves in a checkout and 404s for a reader on getbusbar.com, which is what it was doing at /docs/plugins/. Made absolute so it resolves from both. Scope checked rather than assumed: the other files carrying relative `adr/` links (development.md, internals.md, testing.md, code-layout.md) are NOT synced to the site, so their links are correct where they live and were left alone. --- docs/plugins.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/plugins.md b/docs/plugins.md index ea120390..f0bb389a 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -627,7 +627,10 @@ key it expects) in that plugin's `settings:`, and let the plugin validate it on may be a `SecretRef`, in which case the core resolves it against the secret backend **before** the settings cross the ABI, so a raw key never has to sit in plaintext config, is never logged, and is never written back to the overlay (only the reference is persisted). An unresolvable ref fails the -load fail-closed. See [ADR-0010](adr/0010-plugin-licensing.md) for the full model. +load fail-closed. See +[ADR-0010](https://github.com/GetBusbar/busbar/blob/main/docs/adr/0010-plugin-licensing.md) for the +full model. The link is absolute because `docs/adr/` is a repository-only record and is not synced to +the published docs site, so a relative path resolves in a checkout and 404s for a reader on the web. ## Inspecting and validating From b288b55ce26d376e1b58a309fd2a42a1c4a58ec0 Mon Sep 17 00:00:00 2001 From: matthew Date: Sat, 8 Aug 2026 09:53:29 -0700 Subject: [PATCH 002/166] docker: actually publish `latest`, which the comment claimed and the tags did not `docker pull getbusbar/busbar` served 1.5.2 for the whole of the 1.5.3 release. The exact pin `getbusbar/busbar:1.5.3` published correctly; `latest` never moved, because the tag list only emitted `type=semver,pattern={{version}}`. docker/metadata-action does not imply `latest` from a semver pattern: it needs an explicit `type=raw,value=latest` or the `latest=true` flag, and neither was there. So `latest` was frozen at whatever it was last set to by hand. The comment directly above it said "X.Y.Z + latest". It had said so through at least two releases while being false, which is the part worth noticing: the thing that documented the behaviour and the thing that produced it had drifted apart with nothing comparing them. Added as an explicit tag, gated on the same resolved-version condition as the pin, so a bare dispatch still publishes only `test`. --- .github/workflows/docker.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 46bd749c..d59ad557 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -376,10 +376,16 @@ jobs: images: | ${{ env.DOCKERHUB_IMAGE }} ${{ env.GHCR_IMAGE }} - # version resolved → X.Y.Z + latest (exact pins only, no floating - # major/minor aliases — one consistent tag shape); bare dispatch → `test` only. + # version resolved -> X.Y.Z + latest (exact pins only, no floating major/minor aliases, + # one consistent tag shape); bare dispatch -> `test` only. + # + # `latest` is an EXPLICIT type=raw. It is not implied by type=semver, and this comment + # claimed the cascade produced it while the tag list did not: 1.5.3 published as an exact + # pin while `docker pull getbusbar/busbar` kept serving 1.5.2, because nothing had moved + # `latest` since someone last did it by hand. A comment is not a tag. tags: | type=semver,pattern={{version}},value=v${{ steps.ver.outputs.ver }},enable=${{ steps.ver.outputs.ver != '' }} + type=raw,value=latest,enable=${{ steps.ver.outputs.ver != '' }} type=raw,value=test,enable=${{ steps.ver.outputs.ver == '' }} labels: | org.opencontainers.image.title=busbar From d101624a7e13a18ee93d3189cb4c7b2314e284e1 Mon Sep 17 00:00:00 2001 From: matthew Date: Sat, 8 Aug 2026 10:36:33 -0700 Subject: [PATCH 003/166] docs: bump stale version references from 1.5.0/1.5.2 to 1.5.3 getting-started.md's Docker pin example and configuration.md's "three orthogonal axes" note were still citing pre-1.5.3 versions even though the described behavior is unchanged in 1.5.3. --- docs/configuration.md | 2 +- docs/getting-started.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 72ecce76..8e11bc4b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -323,7 +323,7 @@ auth: **Token extraction order (data plane):** `Authorization: Bearer`, then `x-api-key`, then `x-goog-api-key`. Blank values are treated as absent. -**Three orthogonal axes (as of 1.5.2).** Data-plane admission, admin-API access, and governance +**Three orthogonal axes (as of 1.5.3).** Data-plane admission, admin-API access, and governance enforcement are independent, each with one local source of truth: - **Data-plane admission** is decided **solely** by `auth.chain`: `[]` = open/anonymous, `[keys]` = diff --git a/docs/getting-started.md b/docs/getting-started.md index 1e24b5e6..64b412f9 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -75,7 +75,7 @@ docker run -d -p 8080:8080 \ getbusbar/busbar ``` -The provider catalog ships inside the image at `/etc/busbar/providers.yaml`, so you only mount `config.yaml` (written in [Step 2](#step-2-write-a-minimal-config)). Pin an exact version (`getbusbar/busbar:1.5.0`) or ride `latest`. If you use a durable store, give it a writable volume (e.g. `-v busbar-data:/var/lib/busbar` with `store.settings.db_path: /var/lib/busbar/governance.db`). +The provider catalog ships inside the image at `/etc/busbar/providers.yaml`, so you only mount `config.yaml` (written in [Step 2](#step-2-write-a-minimal-config)). Pin an exact version (`getbusbar/busbar:1.5.3`) or ride `latest`. If you use a durable store, give it a writable volume (e.g. `-v busbar-data:/var/lib/busbar` with `store.settings.db_path: /var/lib/busbar/governance.db`). **Or build from source** (requires Rust 1.97+): From b02d06d690b211cafb441985e064f5327f49ef2a Mon Sep 17 00:00:00 2001 From: Matthew Jackson Date: Sat, 8 Aug 2026 11:19:33 -0700 Subject: [PATCH 004/166] ci: close the gaps the 1.5.3 release found, at the root (#49) Every change here traces to something that actually went wrong shipping 1.5.3, and each one is proven RED before it is trusted GREEN. FEATURE BRANCHES WERE COMPLETELY UNGATED. ci.yml triggered on pushes to main/dev/qa and on pull_request, so work on a feature branch had no CI at all until someone opened a PR. Two 1.5.4 items were reported green from local runs, had never been CI-verified, and the gap was only found days later at PR time. Now `branches: ['**']`, with two tiers: every branch push runs structure lint, fmt/clippy/build/test, config-stability and public-hygiene; dev/qa/main, every pull request, and workflow_dispatch additionally run openapi-schema, migration-corpus, executable-config, no-default-features, no-plugins-gate, txn-guards, timing and windows. The tradeoff is stated in the file: a Windows-only regression is still caught at PR time, exactly where it was caught before, so the fast tier is pure addition. The cost is four duplicated jobs when a branch with an open PR is pushed. The two events deliberately stay in separate concurrency groups; unifying them would let the cheap push run cancel the full PR run the required checks depend on. A green check on a feature branch now means something weaker than a green check on a PR, so a gate-tier job states which tier ran and names the jobs it did not run. Its rule and the jobs' `if:` guards are separate implementations of the same predicate, and they were checked to agree on every event/ref combination. A FLAKY WINDOWS TEST BLOCKED A RELEASE. Root cause found, and it was not "a slow runner needs a longer timeout". In an_idempotency_key_survives_a_client_disconnect_mid_mint the client's 100ms timeout and the server's whole request path shared ONE single-threaded `#[tokio::test]` runtime. One OS-level deschedule of that worker longer than the client's budget makes tokio observe the timeout as expired BEFORE polling the server far enough to reach put_key. The client errors, `first.is_err()` is satisfied for entirely the wrong reason, and NOTHING is minted, so no amount of polling can observe a write that never started. Reproduced locally by shrinking the budget under heavy CPU oversubscription: 1 failure in 10, with precisely the CI message. Replaced the sleep-versus-timeout race with a three-signal rendezvous: the store announces it has entered put_key, parks until the test has taken the client away, then announces the write committed. The test now also ASSERTS the write had not landed yet, which the old one could not: it had no way to tell a disconnect mid-mint from one after it. 30/30 green under the same contention that broke the old test, and faster. Generous timeouts remain purely as liveness backstops, because a missing rendezvous would otherwise hang the suite rather than fail it, and each one states what it waited for and what it saw. VERIFY-ASSETS COULD NOT SEE A MISSING PLATFORM. It asserted `assets != 0`. 1.5.3 published FIVE assets where seven were expected, missing Apple Silicon Mac and x86_64 Linux; install.sh 404'd on Apple Silicon and two of five download links were dead. A count can never see a missing platform, only a name can. The platform list now lives once, in .github/release-targets.json; release.yml builds both the upload matrix and the expected asset names from it, and verify-deploy fetches the same file AT THE TAG UNDER TEST, so the verifier and the release cannot disagree about what was supposed to ship. Worse, verify-assets was SKIPPED on 1.5.3, because `needs:` on a failed job skips the dependent: the one guard that would have noticed was skipped precisely because the release was broken, taking notify-downstream with it. It now runs under `!cancelled()`, so a partial matrix produces a red job naming the missing platforms instead of a grey one naming nothing. POST-DEPLOY VERIFICATION DID NOT RUN WHEN IT MATTERED. verify-deploy's workflow_run trigger required the triggering run to have SUCCEEDED, so when Release failed it never ran, on the release that most needed it, and every defect it exists to catch was found by hand. It now also triggers on `release: [published]`, which fires regardless of how the rest of the run goes. Two new checks: (m) the documented `gh attestation verify` command passes on real downloaded bytes, and (n) numbers the site presents as live actually track their authority. Also documented, rather than left as folklore, why verify-deploy needs no qa-gate-style dispatcher: it has zero `uses:` steps, no checkout, reads no repository file, and takes its version from the event payload, so the workflow_run default-branch trap has nothing to bite. THE PROMOTION PUSH RACED. `git push origin qa:main` was rejected as non-fast-forward while main was a strict ancestor of qa with zero divergent commits; a retry seconds later worked. origin/main is a local photograph of the remote; the SERVER re-checks against whatever replica it lands on, and checking harder locally cannot fix a disagreement between two machines. scripts/promote.sh verifies strict ancestry, verifies CI on the exact SHA rather than the branch name, retries a rejection only after RE-VERIFYING ancestry, and reads the remote back afterwards because a push's exit code is a claim and the remote ref is the fact. Its selftest drives all six paths against throwaway repos, including a transient rejection that succeeds on retry and a push that reports success without moving the remote. THE GATE COULD NOT SAY WHY IT WAS RED. The 1.5.3 plugin gate went red twice and found zero product defects; both were infrastructure, and both printed "DO NOT TAG THIS RELEASE". That is right for a product defect and wrong for an unset environment variable, and a gate that cries wolf twice per release teaches people to rerun it rather than read it. release-check now has two verdicts: PRODUCT keeps the old alarming message, HARNESS / ENVIRONMENT says the gate never got far enough to ask. Deliberately not a classifier over error text, which would add silent misclassification as a third failure mode; call sites know which kind they are. THE RELEASE BODY NAMED NOTHING. It was a one-line compare link, so 1.5.3 did not mention that --validate now resolves env:/file: secret references and exits 1 when one cannot. The body now leads with the CHANGELOG section for the version being cut, fail-soft so a missing section never blocks a release. docker.yml's header claimed a tag cascade the code has never produced. Corrected, and the claim is now guarded rather than merely fixed: verify-deploy asserts latest and the version pin resolve to the same digest, which is the assertion whose absence let latest sit frozen at 1.5.2 through the whole of the 1.5.3 release. Co-authored-by: Matthew --- .github/release-targets.json | 45 ++++ .github/workflows/ci.yml | 112 ++++++++- .github/workflows/docker.yml | 9 +- .github/workflows/release.yml | 204 +++++++++++++---- .github/workflows/verify-deploy.yml | 229 ++++++++++++++++++- crates/busbar/src/admin/tests/tests.rs | 232 +++++++++++++------ scripts/promote.sh | 302 +++++++++++++++++++++++++ scripts/release-check.sh | 70 +++++- 8 files changed, 1073 insertions(+), 130 deletions(-) create mode 100644 .github/release-targets.json create mode 100755 scripts/promote.sh diff --git a/.github/release-targets.json b/.github/release-targets.json new file mode 100644 index 00000000..f7710838 --- /dev/null +++ b/.github/release-targets.json @@ -0,0 +1,45 @@ +{ + "_comment": [ + "THE PLATFORM LIST, IN EXACTLY ONE PLACE. Every consumer derives from this file; nothing", + "hardcodes a parallel copy.", + "", + " release.yml `targets` job reads it to build the upload matrix AND to compute the exact set", + " of asset filenames that matrix owes the Release. `verify-assets` then asserts", + " every one of those names is present and plausibly sized.", + " verify-deploy.yml fetches it over raw.githubusercontent AT THE TAG UNDER TEST (it checks out", + " nothing on purpose) and derives the same expected names, so the post-deploy", + " verifier and the release that produced the assets can never disagree about what", + " was supposed to ship.", + "", + "WHY IT IS A FILE AND NOT A LITERAL IN EACH WORKFLOW. v1.5.3 published FIVE assets where seven", + "were expected: aarch64-apple-darwin and x86_64-unknown-linux-gnu were both missing, which is", + "Apple Silicon Mac and x86_64 Linux. install.sh 404'd on Apple Silicon and two of five links on", + "the live /download/ page were dead. The guard in place asserted `assets != 0`, which five assets", + "pass comfortably: a COUNT can never see a missing platform, only a NAME can. Fixing that by", + "writing an expected-names list into the verifier would have created a second place to forget a", + "platform, which is the same defect one level up.", + "", + "Adding a platform is one entry here. Its build leg and both of its verifications follow.", + "", + "`pgo: true` routes a HOST-NATIVE target through scripts/pgo-build.sh (PGO is MANDATORY for", + "release: the same fail-closed build docker.yml ships). PGO needs the instrumented binary to run", + "ON THE RUNNER to train, so only targets whose arch matches the runner's are PGO'd. The cross", + "targets (aarch64-linux on an x86_64 host, x86_64-darwin on an arm64 host) and windows cannot", + "self-train, so they keep the plain --release build.", + "", + "`archive` is the extension the packaging step produces: the PGO path tars by hand, the plain", + "path hands `archive: busbar-$target` to upload-rust-binary-action, which emits .tar.gz on unix", + "and .zip on windows. Same base name either way." + ], + "targets": [ + { "target": "x86_64-unknown-linux-gnu", "os": "ubuntu-latest", "pgo": true, "archive": "tar.gz" }, + { "target": "aarch64-unknown-linux-gnu", "os": "ubuntu-latest", "pgo": false, "archive": "tar.gz" }, + { "target": "x86_64-apple-darwin", "os": "macos-latest", "pgo": false, "archive": "tar.gz" }, + { "target": "aarch64-apple-darwin", "os": "macos-latest", "pgo": true, "archive": "tar.gz" }, + { "target": "x86_64-pc-windows-msvc", "os": "windows-latest", "pgo": false, "archive": "zip" } + ], + "metadata_assets": [ + "busbar-{tag}.cdx.json", + "busbar-openapi-{tag}.json" + ] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80b17f85..de519cd6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,51 @@ name: CI +# TWO TIERS, AND WHY. +# +# This workflow used to trigger on `push` to main/dev/qa ONLY, plus `pull_request`. That left every +# feature branch COMPLETELY UNGATED until someone opened a PR. During 1.5.4 development two work +# items were reported green from local runs and had never been CI-verified at all; the gap was only +# discovered when the PR was opened, days of work later. "It passed locally" is not a CI result, and +# a branch nobody has opened a PR for is exactly where that mistake is cheapest to make and most +# expensive to discover. +# +# So: `branches: ['**']`, and EVERY branch push now runs CI. But the full set is 12 jobs including a +# Windows runner, a release-mode timing gate, a loom model and a full-history migration corpus, and +# paying all of that on every intermediate commit of a long-lived feature branch is real money for +# very little marginal signal. So pushes to NON-PROMOTION branches run the FAST TIER: +# +# FAST TIER (every branch push) structure lint, fmt/clippy/build/test, config-stability, +# public-hygiene. This answers the only questions that matter +# mid-feature: does it build, does it lint, do the tests pass, is +# the layout/config-grammar/public-prose still clean. +# FULL TIER (dev, qa, main, and everything above PLUS openapi-schema, migration-corpus, +# EVERY pull request, and executable-config, no-default-features, no-plugins-gate, +# workflow_dispatch) txn-guards, timing, windows. +# +# THE TRADEOFF, STATED PLAINLY: a Windows-only or featureless-build-only regression is now caught at +# PR time rather than at push time. That is exactly where it was caught before this change, so the +# fast tier costs nothing anyone had; it is pure addition. The cost paid is that a branch with an +# open PR runs the fast tier (push) alongside the full tier (pull_request) for the same commit. The +# two events land in DIFFERENT concurrency groups (`github.ref` is `refs/heads/` for push and +# `refs/pull/N/merge` for pull_request) and they are deliberately NOT unified: a shared group would +# let the cheap push run cancel the full PR run that the required checks depend on. Four duplicated +# jobs is the accepted price of never again shipping an unverified branch. +# +# ESCAPE HATCH: `workflow_dispatch` forces the FULL tier on any ref, so a feature branch can buy the +# whole gate before opening a PR without waiting for one: +# gh workflow run ci.yml -R GetBusbar/busbar --ref feat/my-branch +# +# `branches:` never matches tag pushes, so the release tag (v1.5.3 et al) still does not run CI here. on: push: - branches: [main, dev, qa] + branches: ['**'] pull_request: + workflow_dispatch: # Without this, every push queues a fully independent run that competes for the same runners # instead of cancelling the one it superseded -- qa-gate.yml already has this; ci.yml never did, # so a burst of rapid pushes (e.g. iterating on a CI fix) pays for N full runs instead of 1. +# Doubly load-bearing now that every branch push triggers a run. concurrency: group: ci-${{ github.ref }} cancel-in-progress: true @@ -17,6 +55,54 @@ env: RUSTFLAGS: "-D warnings" jobs: + # WHICH TIER RAN, SAID OUT LOUD. A green check mark is read as "the gate passed", and after this + # change a green check on a feature branch means something WEAKER than a green check on a PR. + # An operator who cannot see the difference will assume the strong one, which is the precise + # mistake this workflow's two tiers otherwise re-introduce in a new place. So every run states its + # own tier in the job summary and names the jobs it did NOT run. Costs one echo; buys the run's + # verdict being self-describing instead of needing someone to remember the rule. + gate-tier: + name: gate tier (what this run actually proved) + runs-on: ubuntu-latest + steps: + - name: Declare the tier + # Ref and event come in as ENV, not as `${{ }}` spliced into the script text. A branch name + # is attacker-influenced data (anyone who can open a PR picks it) and GitHub allows plenty of + # shell metacharacters in one; splicing it into a `[ ... ]` test is a script-injection seam + # that happens to be quiet until someone names a branch to exploit it. + env: + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + REF_NAME: ${{ github.ref_name }} + run: | + set -euo pipefail + if [ "$EVENT" != "push" ] || \ + [ "$REF" = "refs/heads/main" ] || \ + [ "$REF" = "refs/heads/dev" ] || \ + [ "$REF" = "refs/heads/qa" ]; then + { + echo "### CI tier: FULL" + echo "" + echo "Every CI job ran. Event \`$EVENT\`, ref \`$REF\`." + } >> "$GITHUB_STEP_SUMMARY" + else + { + echo "### CI tier: FAST" + echo "" + echo "Push to a non-promotion branch (\`$REF\`), so this run proved:" + echo "structure lint, fmt/clippy/build/test, config-stability, public-hygiene." + echo "" + echo "It did NOT run: openapi-schema, migration-corpus, executable-config," + echo "no-default-features, no-plugins-gate, txn-guards, timing, windows." + echo "" + echo "Those run on every pull request, on dev/qa/main, and on demand via" + echo "\`gh workflow run ci.yml -R GetBusbar/busbar --ref $REF_NAME\`." + echo "" + echo "A green check here is NOT the full gate. Do not promote on it." + } >> "$GITHUB_STEP_SUMMARY" + fi + cat "$GITHUB_STEP_SUMMARY" + # LAYOUT GATE — its OWN job, deliberately. It used to be the first step of `check`, ahead of # clippy/build/test, which made it a MASK: any layout violation aborted the job before a single # test ran, and the `Test` step is the only place BUSBAR_TEST_POSTGRES_URL / VALKEY_URL are set @@ -198,6 +284,9 @@ jobs: openapi-schema: name: openapi-schema clippy · drift · coverage runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable @@ -230,6 +319,9 @@ jobs: migration-corpus: name: migration corpus (every shipped config still migrates) runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 with: @@ -321,6 +413,9 @@ jobs: executable-config-lint: name: executable-config gate (heredocs · test literals · shipped yaml) runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - name: Install Rust toolchain @@ -350,6 +445,9 @@ jobs: no-default-features: name: no-default-features build · clippy · test runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable @@ -385,6 +483,9 @@ jobs: no-plugins-gate: name: no-plugins gate (compiled out · not installed) runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable @@ -402,6 +503,9 @@ jobs: txn-guards: name: txn compile fence · loom model runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable @@ -418,6 +522,9 @@ jobs: timing: name: timing gate (release) runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable @@ -429,6 +536,9 @@ jobs: windows: name: windows build · test runs-on: windows-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d59ad557..1f0a3d1c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,7 +4,14 @@ name: Docker # manifest (linux/amd64 + linux/arm64), FROM scratch over static musl binaries. # # Triggers: -# - v* tags: publish with the semver tag cascade (X.Y.Z, X.Y, X, latest) +# - v* tags: publish the EXACT semver pin plus `latest` (X.Y.Z and latest) -- no floating +# X.Y or X aliases. This header used to claim an "X.Y.Z, X.Y, X, latest" cascade that the +# `tags:` block has never produced; the code is right about the shape and the header was stale. +# Both halves of that drift are now GUARDED rather than merely corrected: verify-deploy.yml +# asserts that `latest` and the version pin resolve to the SAME manifest digest on BOTH +# registries, which is the assertion whose absence let `latest` sit frozen at 1.5.2 through the +# whole of the 1.5.3 release. GetBusbar/helm-charts pins `getbusbar/busbar:`, the +# exact pin, so this tag shape is load-bearing downstream and not cosmetic. # - workflow_dispatch: publish a `test` tag only — end-to-end pipeline check # without cutting a release. # diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8e92ff7a..dbd3bf2f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -81,15 +81,60 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + # THE BODY NAMES WHAT CHANGED. `--generate-notes` alone produced a one-line compare link, so + # v1.5.3's published body did not mention that `--validate` now RESOLVES `env:`/`file:` secret + # references and exits 1 when one cannot resolve (boot behaviour unchanged). An operator + # deciding whether to take a config-affecting release could not learn that from the release. + # The CHANGELOG section for the version being cut is already written, already reviewed, and + # already the canonical answer, so the release body is now that section followed by the + # auto-generated commit/contributor notes rather than the notes alone. + # + # FAIL-SOFT, DELIBERATELY: a missing or unparseable CHANGELOG section must never block a + # release that is otherwise good. It degrades to the previous behaviour and says so in the log. + - name: Extract this version's CHANGELOG section for the release body + id: notes + run: | + set -euo pipefail + V="${GITHUB_REF_NAME#v}" + python3 - "$V" > /tmp/relnotes.md <<'PY' + import re, sys + version = sys.argv[1] + try: + text = open("CHANGELOG.md", encoding="utf-8").read() + except OSError: + sys.exit(0) + # Headings look like `## [1.5.3], 2026-08-08`. Capture through to the next `## `. + m = re.search( + r"^##\s*\[%s\][^\n]*\n(.*?)(?=^##\s|\Z)" % re.escape(version), + text, re.M | re.S) + if m: + sys.stdout.write(m.group(1).strip() + "\n") + PY + if [ -s /tmp/relnotes.md ]; then + echo "found=1" >> "$GITHUB_OUTPUT" + echo "CHANGELOG section for ${V} ($(wc -l < /tmp/relnotes.md) lines) will lead the release body." + else + echo "found=0" >> "$GITHUB_OUTPUT" + echo "::warning::No CHANGELOG section found for ${V}; falling back to generated notes only." \ + "The release will still publish, but its body will not say what changed." + fi - name: Create GitHub Release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh release create "${GITHUB_REF_NAME}" \ - --repo "${GITHUB_REPOSITORY}" \ - --title "busbar ${GITHUB_REF_NAME}" \ - --verify-tag --generate-notes \ - || gh release view "${GITHUB_REF_NAME}" --repo "${GITHUB_REPOSITORY}" + if [ "${{ steps.notes.outputs.found }}" = "1" ]; then + gh release create "${GITHUB_REF_NAME}" \ + --repo "${GITHUB_REPOSITORY}" \ + --title "busbar ${GITHUB_REF_NAME}" \ + --verify-tag --generate-notes --notes-file /tmp/relnotes.md \ + || gh release view "${GITHUB_REF_NAME}" --repo "${GITHUB_REPOSITORY}" + else + gh release create "${GITHUB_REF_NAME}" \ + --repo "${GITHUB_REPOSITORY}" \ + --title "busbar ${GITHUB_REF_NAME}" \ + --verify-tag --generate-notes \ + || gh release view "${GITHUB_REF_NAME}" --repo "${GITHUB_REPOSITORY}" + fi # Generate a CycloneDX Software Bill of Materials (every dependency + version + # license) and attach it to the Release. Lets downstream users answer "is the @@ -152,32 +197,54 @@ jobs: gh release upload "${GITHUB_REF_NAME}" "${GITHUB_WORKSPACE}/busbar-openapi-${GITHUB_REF_NAME}.json" \ --repo "${GITHUB_REPOSITORY}" --clobber + # THE PLATFORM LIST, IN EXACTLY ONE PLACE. This job emits the build matrix AND the exact set of + # asset filenames that matrix is contractually obliged to produce, from one literal. `upload-assets` + # consumes the first; `verify-assets` consumes the second. + # + # WHY IT IS A JOB AND NOT A LITERAL MATRIX. v1.5.3 published with FIVE assets where seven were + # expected: `busbar-aarch64-apple-darwin.tar.gz` and `busbar-x86_64-unknown-linux-gnu.tar.gz` were + # both missing, which is Apple Silicon Mac and x86_64 Linux, the two most common platforms there + # are. `curl -fsSL https://getbusbar.com/install.sh | sh` returned a 404 on Apple Silicon (the + # documented one-line install, dead on the most common developer machine) and two of five download + # links on the live /download/ page 404'd. The guard below asserted `assets != 0`, which a + # five-asset release passes comfortably. A count can never see a MISSING platform; only a name can. + # + # Deriving both lists here means adding a platform is one edit and its verification comes along + # automatically. A hardcoded expected-names list in the verifier would be a second place to forget, + # which is the same defect one level up. + targets: + name: release target matrix (single source of truth) + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.emit.outputs.matrix }} + assets: ${{ steps.emit.outputs.assets }} + steps: + - uses: actions/checkout@v7 + - name: Emit the target matrix and the asset names it must produce + id: emit + env: + TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import json, os + spec = json.load(open(".github/release-targets.json")) + tag = os.environ["TAG"] + inc = [{k: t[k] for k in ("target", "os", "pgo")} for t in spec["targets"]] + assets = ["busbar-%s.%s" % (t["target"], t["archive"]) for t in spec["targets"]] + assets += [a.replace("{tag}", tag) for a in spec["metadata_assets"]] + print("matrix=" + json.dumps({"include": inc})) + print("assets=" + json.dumps(assets)) + PY + cat "$GITHUB_OUTPUT" + upload-assets: - needs: create-release + needs: [create-release, targets] name: ${{ matrix.target }} runs-on: ${{ matrix.os }} strategy: fail-fast: false - matrix: - # `pgo: true` routes a HOST-NATIVE target through scripts/pgo-build.sh (PGO is MANDATORY for - # release: the same fail-closed build docker.yml ships). PGO needs the instrumented binary - # to run ON THE RUNNER to train, so only the targets whose arch matches the runner's are - # PGO'd: x86_64-linux on the x86_64 ubuntu runner, and aarch64-darwin on the arm64 macos - # runner. The cross targets (aarch64-linux built on an x86_64 host, x86_64-darwin on an arm64 - # host) and windows cannot self-train, so they keep the plain --release build. - include: - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - pgo: true - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - - target: x86_64-apple-darwin - os: macos-latest - - target: aarch64-apple-darwin - os: macos-latest - pgo: true - - target: x86_64-pc-windows-msvc - os: windows-latest + matrix: ${{ fromJSON(needs.targets.outputs.matrix) }} steps: - uses: actions/checkout@v7 @@ -320,26 +387,83 @@ jobs: # matrix but does NOT inherit its fail-fast:false — one green target is enough to have assets, but # zero across the board must hard-fail here. verify-assets: - needs: [upload-assets] + needs: [targets, upload-assets, sbom, openapi] runs-on: ubuntu-latest + # `!cancelled()` IS THE POINT OF THIS LINE, and it is the second half of the 1.5.3 defect. + # `upload-assets` runs `fail-fast: false`, so when two of its five legs failed the JOB failed -- + # and a `needs:` on a failed job SKIPS the dependent by default. So the one guard that existed to + # notice a broken release was skipped precisely because the release was broken, and + # `notify-downstream` was skipped with it. A verifier that only runs when everything already + # worked is not a verifier. This one runs whenever the run was not cancelled, so a partial matrix + # produces a RED verify-assets naming the platforms that are missing, instead of a grey one + # naming nothing. + if: ${{ !cancelled() }} steps: - - name: Assert the Release has at least one asset + - name: Assert the Release carries every asset the matrix owes it env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EXPECTED: ${{ needs.targets.outputs.assets }} run: | set -euo pipefail - count="$(gh release view "${GITHUB_REF_NAME}" \ - --repo "${GITHUB_REPOSITORY}" \ - --json assets --jq '.assets | length')" - echo "Release ${GITHUB_REF_NAME} has ${count} asset(s)." - if [ "${count}" -eq 0 ]; then - echo "::error::PHANTOM RELEASE: ${GITHUB_REF_NAME} was published with 0 assets." \ - "Every build/pack target failed to upload a tarball. Failing the release run so this" \ - "tag is not mistaken for a real release by busbar's plugin-registry-gate. Fix the" \ - "build (check Cargo.lock freshness vs --locked and the plugin cdylib build step)," \ - "delete this tag+release, and re-cut." >&2 - exit 1 - fi + gh release view "${GITHUB_REF_NAME}" --repo "${GITHUB_REPOSITORY}" \ + --json assets --jq '.assets[] | "\(.name)\t\(.size)"' > /tmp/got.tsv || : > /tmp/got.tsv + echo "Release ${GITHUB_REF_NAME} published these assets:" + cat /tmp/got.tsv + python3 - <<'PY' + import json, os, sys + expected = json.loads(os.environ["EXPECTED"]) + got = {} + for line in open("/tmp/got.tsv"): + line = line.rstrip("\n") + if not line: + continue + name, _, size = line.partition("\t") + got[name] = int(size or 0) + + missing = [a for a in expected if a not in got] + # A NAME IN THE ASSET LIST IS NOT A USABLE ARTIFACT: GitHub creates the row as soon as the + # upload starts, so a 0-byte or truncated upload lists identically to a good one. 1 KiB is + # far below any real busbar tarball and far above an empty or header-only file. + empty = [a for a in expected if a in got and got[a] < 1024] + + summary = os.environ.get("GITHUB_STEP_SUMMARY") + lines = ["### Release asset verification", "", "| asset | bytes | verdict |", "| --- | --- | --- |"] + for a in expected: + if a not in got: + lines.append("| `%s` | - | MISSING |" % a) + elif got[a] < 1024: + lines.append("| `%s` | %d | TOO SMALL |" % (a, got[a])) + else: + lines.append("| `%s` | %d | ok |" % (a, got[a])) + extra = sorted(set(got) - set(expected)) + if extra: + lines += ["", "Also present (not required): " + ", ".join("`%s`" % e for e in extra)] + if summary: + open(summary, "a").write("\n".join(lines) + "\n") + print("\n".join(lines)) + + if not got: + print("::error::PHANTOM RELEASE: %s was published with 0 assets. Every build/pack " + "target failed to upload. Failing the release run so this tag is not mistaken " + "for a real release by busbar's plugin-registry-gate. Fix the build (check " + "Cargo.lock freshness vs --locked and the plugin cdylib build step), delete " + "this tag+release, and re-cut." % os.environ["GITHUB_REF_NAME"], file=sys.stderr) + sys.exit(1) + if missing or empty: + if missing: + print("::error::INCOMPLETE RELEASE: %s is missing %d of %d required asset(s): %s. " + "Each missing name is a platform whose users get a 404 from install.sh and " + "from the /download/ page. Find that target's leg in `upload-assets`, fix " + "it, then delete this tag+release and re-cut." % + (os.environ["GITHUB_REF_NAME"], len(missing), len(expected), + ", ".join(missing)), file=sys.stderr) + if empty: + print("::error::TRUNCATED RELEASE: %s uploaded these assets at under 1 KiB, which " + "means the upload was cut short: %s" % + (os.environ["GITHUB_REF_NAME"], ", ".join(empty)), file=sys.stderr) + sys.exit(1) + print("All %d required assets present and plausibly sized." % len(expected)) + PY # ── Fan out an instant "a real release just happened" signal to every downstream repo that # self-heals off busbar's releases (Homebrew tap, Helm chart, and whatever else gets added to diff --git a/.github/workflows/verify-deploy.yml b/.github/workflows/verify-deploy.yml index 61f67ca9..052ce866 100644 --- a/.github/workflows/verify-deploy.yml +++ b/.github/workflows/verify-deploy.yml @@ -53,6 +53,20 @@ name: Verify deploy # these back the live numbers on getbusbar.com and are currently unmonitored. # (l) the DOCUMENTED quickstart actually runs: both the binary quickstart and the docker # one-liner from docs/getting-started.md boot and answer on /healthz. +# (m) `gh attestation verify --repo GetBusbar/busbar` -- the exact command the docs tell +# users to run before trusting a download -- passes on real downloaded bytes. Previously "the +# release is attested" meant a workflow step exited 0, never that a user could verify it. +# (n) numbers getbusbar.com presents as LIVE actually track their authority (Docker pull count vs +# Docker Hub, star count vs GitHub). The download page served a pull count baked into static +# HTML at build time, beside a working Worker nothing called, so it aged until a redeploy. +# +# THE PATTERN THESE LAST TWO EXIST FOR. Three defects in the 1.5.3 release were one shape: something +# DECLARED a behaviour and nothing CHECKED it. docker.yml's comment claimed a `latest` tag its tag +# list never emitted, so `docker pull getbusbar/busbar` served 1.5.2 for the whole release. The site +# claimed a live pull count that was a build-time snapshot. A "zero broken links" claim turned out +# to be unreproducible, and there is still no link checker anywhere. Check (a) already guards the +# first of those; (n) guards the second. A claim with no assertion behind it decays invisibly, which +# is worse than not making it. # # Triggers: # - workflow_run: fires when release.yml ("Release") or docker.yml ("Docker") completes on a tag @@ -73,7 +87,45 @@ name: Verify deploy # instead of just killing the job, and the job carries an overall `timeout-minutes: 60`. This # matters more on the daily schedule than on release: a 6-hour hung run would still be "in # progress" when the next day's run fires. +# +# WHEN THIS RUNS, AND THE HOLE THAT WAS IN IT. The `workflow_run` trigger below is gated on the +# triggering run having SUCCEEDED. During the 1.5.3 release the Release workflow FAILED (two of five +# build legs died), so this verifier never ran at all -- on precisely the release that most needed +# verifying. Every defect it exists to catch was then found BY HAND: install.sh 404ing on Apple +# Silicon, two dead links on the live /download/ page, `docker pull getbusbar/busbar` serving the +# previous version. A verifier that only runs when the build already went well is not a verifier. +# +# THE `workflow_run` DEFAULT-BRANCH TRAP, AND WHY THIS FILE DOES NOT NEED A DISPATCHER. +# `workflow_run` ALWAYS loads the workflow YAML from the repository's DEFAULT BRANCH, whatever +# branch the triggering run was on. That bit qa-gate.yml hard: `main` was over a hundred commits +# behind, so every auto-fired gate ran stale logic, and the repair was to rewrite it as a thin +# DISPATCHER that checks out the triggering SHA and runs the gate scripts from THAT checkout. +# +# This workflow is structurally IMMUNE to the same trap, and the reason is worth stating so nobody +# has to re-derive it or "fix" it by adding a checkout: +# * It has ZERO `uses:` steps, therefore NO `actions/checkout` at all, therefore no working copy +# that could be stale. That is deliberate, and check (h) depends on it: reading ./install.sh +# from a checkout would hide a fix that merged but never deployed. +# * It reads NO repository file. Every path it touches is either created on the runner (the +# digest helper, the quickstart config, both written inline here) or fetched over HTTPS. The one +# repository file it consults, .github/release-targets.json, is fetched from raw.githubusercontent +# AT THE TAG UNDER TEST, not read from a checkout, precisely so it tracks the release rather +# than the default branch. +# * The VERSION under test comes from the event payload (`release.tag_name` / +# `workflow_run.head_branch`) or, on the schedule, from the live /releases/latest redirect. It +# never comes from crates/busbar/Cargo.toml, so a stale default branch cannot make it verify the +# wrong version. +# So the only thing loaded from `main` is this file's own text, and this file's own text has no +# repo-state dependency to go stale against. A dispatcher would add a checkout whose entire purpose +# is to be read, in a workflow whose entire purpose is to read nothing local. +# +# So `release: [published]` is now a trigger in its own right. `create-release` publishes the Release +# object BEFORE the build matrix runs, and that event fires regardless of what the rest of the run +# does, which is exactly the property needed: if a release object exists in public, it gets verified, +# whether or not the workflow that made it finished happy. on: + release: + types: [published] workflow_run: workflows: ["Release", "Docker"] types: [completed] @@ -106,6 +158,7 @@ jobs: if: >- github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || + github.event_name == 'release' || (github.event.workflow_run.conclusion == 'success' && startsWith(github.event.workflow_run.head_branch, 'v')) env: @@ -135,6 +188,9 @@ jobs: exit 1 ;; esac + elif [ "${{ github.event_name }}" = "release" ]; then + V="${{ github.event.release.tag_name }}" + V="${V#v}" else V="${{ github.event.workflow_run.head_branch }}" V="${V#v}" @@ -243,15 +299,55 @@ jobs: exit 1 fi echo "PASS: GitHub Release ${TAG} exists" - expected=( - "busbar-x86_64-unknown-linux-gnu.tar.gz" - "busbar-aarch64-unknown-linux-gnu.tar.gz" - "busbar-x86_64-apple-darwin.tar.gz" - "busbar-aarch64-apple-darwin.tar.gz" - "busbar-x86_64-pc-windows-msvc.zip" - "busbar-${TAG}.cdx.json" - "busbar-openapi-${TAG}.json" + # THE EXPECTED NAMES ARE DERIVED, NOT TYPED. This list used to be five platform names + # written out by hand here, a second copy of the list release.yml builds from. Two copies + # of a platform list is one place to forget a platform, and forgetting it here means the + # verifier cheerfully passes a release that is missing exactly the platform nobody + # remembered. `.github/release-targets.json` is the single source of truth; release.yml's + # `targets` job builds the upload matrix from it and this fetches the SAME file AT THE TAG + # UNDER TEST over raw.githubusercontent, so the verifier and the release that produced the + # assets cannot disagree about what was supposed to ship. Fetched rather than checked out + # because this job deliberately checks nothing out. + mani="https://raw.githubusercontent.com/GetBusbar/busbar/${TAG}/.github/release-targets.json" + if ! curl -fsS --max-time 30 "$mani" -o /tmp/release-targets.json; then + # Tags cut BEFORE this manifest existed do not carry it. Falling back to main's copy is + # correct for them and honest about what it is doing: the platform set is what changes + # rarely, so main's list is the right answer for an old tag, and the alternative + # (hardcoding a list here) is the exact defect the manifest replaced. A release cut from + # now on always carries its own, so the fallback quietly stops being used. + echo "note: ${TAG} predates .github/release-targets.json; falling back to main's copy." + if ! curl -fsS --max-time 30 \ + "https://raw.githubusercontent.com/GetBusbar/busbar/main/.github/release-targets.json" \ + -o /tmp/release-targets.json; then + echo "FAIL: no release-target manifest at ${TAG} and none on main either." + echo "Without it this check cannot know which platforms ${TAG} owed, and guessing is" + echo "how a missing platform got waved through in the first place." + exit 1 + fi + fi + mapfile -t expected < <(python3 - "$TAG" <<'PY' + import json, sys + tag = sys.argv[1] + spec = json.load(open("/tmp/release-targets.json")) + for t in spec["targets"]: + print("busbar-%s.%s" % (t["target"], t["archive"])) + for a in spec["metadata_assets"]: + print(a.replace("{tag}", tag)) + PY ) + # A `mapfile` whose process substitution died leaves an EMPTY array and does not trip + # `set -e`, so the loop below would iterate over nothing and this whole check would PASS + # vacuously. That is the same shape as a gate that has quietly stopped gating, which is + # the failure this check exists to prevent, so it is asserted rather than assumed. The + # floor is 5 because busbar has never shipped fewer platform archives than that plus its + # two metadata assets; a manifest that produced fewer means the manifest is wrong. + if [ "${#expected[@]}" -lt 5 ]; then + echo "FAIL: derived only ${#expected[@]} expected asset name(s) from the manifest." + echo "Refusing to 'verify' a release against an empty or truncated expectation list:" + echo "that passes for any release at all, including one that published nothing." + exit 1 + fi + echo "manifest at ${TAG} requires ${#expected[@]} asset(s): ${expected[*]}" # A NAME IN THE ASSET LIST IS NOT A USABLE ARTIFACT. GitHub lists an asset as soon as the # upload row is created, so a 0-byte or truncated upload appears here exactly like a good # one. install.sh (check (h)) only ever exercises the ONE tarball matching the runner's @@ -790,3 +886,120 @@ jobs: fi docker rm -f busbar-quickstart >/dev/null 2>&1 || true [ "$fail" = 0 ] + + # (m) BUILD PROVENANCE ACTUALLY VERIFIES. release.yml records a keyless Sigstore attestation + # for every archive, and docs tell users to check it with `gh attestation verify`. Nothing has + # ever run that command against a published asset, so "the release is attested" was a claim + # about a workflow step succeeding, not about what a user can verify. A user who runs the + # documented command and gets a failure concludes the artifact is tampered with; that has to + # be tested from the outside, on the real downloaded bytes, like everything else here. + - name: (m) the documented attestation check passes on the real published assets + timeout-minutes: 10 + run: | + set -euo pipefail + V="${{ steps.ver.outputs.version }}"; TAG="${{ steps.ver.outputs.tag }}" + fail=0 + work="$(mktemp -d)" + # Two platforms rather than all five: attestation is a property of the release run, not of + # the target, so a second one is a cheap independent sample and five is just slower. + for a in "busbar-x86_64-unknown-linux-gnu.tar.gz" "busbar-aarch64-apple-darwin.tar.gz"; do + if ! gh release download "$TAG" --repo GetBusbar/busbar --pattern "$a" --dir "$work" --clobber; then + echo "FAIL: (m) could not download $a to verify its attestation" + fail=1; continue + fi + if gh attestation verify "$work/$a" --repo GetBusbar/busbar; then + echo "PASS: (m) $a verifies against its build provenance" + else + echo "::error::(m) 'gh attestation verify $a --repo GetBusbar/busbar' FAILED for ${TAG}. This is the exact command the docs tell users to run before trusting a download, so every user who follows that advice sees a tamper warning. Fix: confirm release.yml's attest-build-provenance step ran for this target and that the uploaded bytes are the attested ones (a re-upload after attestation breaks the digest binding)." + fail=1 + fi + done + rm -rf "$work" + [ "$fail" = 0 ] + + # (n) NUMBERS THE SITE PRESENTS AS LIVE MUST BE LIVE. Four defects in the 1.5.3 release were + # the same shape: something DECLARED a behaviour and nothing CHECKED it. + # * docker.yml's comment claimed a `latest` tag its tag list never emitted, so + # `docker pull getbusbar/busbar` served 1.5.2 through the whole release. + # * The download page presented a Docker pull count baked into static HTML at BUILD time. + # * "Zero broken links" was unreproducible, and one was broken in production. + # * And the sharpest one: the Cloudflare Worker behind /api/pulls and /api/stars has a + # COMMITTED fix for staleness -- its own commit subject says the stars stop going + # stale -- that was never deployed. marketing's deploy.yml runs `wrangler pages deploy`, + # which is Pages ONLY; the Worker needs a separate `wrangler deploy` that lives in a + # README as a MANUAL step and + # runs in no CI anywhere. The code that keeps these numbers fresh has been sitting in git + # while the deployed Worker serves frozen values. + # + # THE GENERALISATION, which is the reason this check is worth its runtime: WHAT IS DEPLOYED IS + # NOT WHAT IS COMMITTED unless something proves it. A green build, a merged fix and a passing + # test suite all say things about the repository. None of them says anything about production. + # + # This asserts it from the DATA side, which is what can be checked from outside without the + # Worker's cooperation. The stronger form is to have the Worker expose its build marker on a + # health route and compare that to the commit under test; when it grows one, add that here as + # a direct deployed-equals-committed assertion and this becomes the backstop rather than the + # primary. Deliberately not stubbed in now: a check that can only go green once another repo + # ships a feature nobody has scheduled is a known-red gate with no owner, which is how gates + # get switched off. + # + # TOLERANCES ARE TIGHT ON PURPOSE. An earlier revision of this check used 20%, calibrated so it + # would be green on the day it was written, on the assumption the ~10% gap was a legitimate + # cache TTL. That assumption was wrong and the calibration was the mistake: the counter is not + # lagging, it is FROZEN, and a frozen counter's gap grows without bound. A tolerance wide + # enough to pass a permanently frozen number is a guard that will never fire. These are set to + # catch a frozen number and nothing looser. Expect this check to be RED until the Worker is + # actually deployed. That is the check doing its job, not a miscalibration. + - name: (n) the live numbers on getbusbar.com really are live + timeout-minutes: 10 + run: | + set -euo pipefail + fail=0 + # Docker pull count: authority is Docker Hub's own repository endpoint. + hub="$(curl -fsS --max-time 30 "https://hub.docker.com/v2/repositories/getbusbar/busbar/" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin).get("pull_count",-1))' || echo -1)" + site="$(curl -fsS --max-time 30 "https://getbusbar.com/api/pulls" \ + | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("pull_count", d.get("count",-1)))' || echo -1)" + echo "docker pulls: hub=${hub} site-api=${site}" + if [ "$hub" -lt 0 ] || [ "$site" -lt 0 ]; then + echo "::error::(n) could not read the Docker pull count from both Docker Hub and getbusbar.com/api/pulls (hub=${hub}, site=${site}). The site advertises this number as live, so an endpoint that cannot answer is a broken promise to every visitor." + fail=1 + else + # 1% or 250, whichever is larger: comfortably absorbs a real cache TTL (the Worker's + # own refresh is hourly) while a frozen counter blows past it within a day. + tol=$(( hub / 100 )); [ "$tol" -lt 250 ] && tol=250 + d=$(( hub > site ? hub - site : site - hub )) + if [ "$d" -gt "$tol" ]; then + echo "::error::(n) the Docker pull count getbusbar.com/api/pulls serves (${site}) is ${d} away from Docker Hub's actual count (${hub}), exceeding the ${tol} tolerance. This is a FROZEN counter, not a stale cache: the gap only grows. Root cause: the Cloudflare Worker that refreshes these values is deployed by nobody. marketing/.github/workflows/deploy.yml runs 'wrangler pages deploy', which is Pages only; the Worker needs a separate 'wrangler deploy' documented in website/workers/README.md as a MANUAL step that no CI runs. The committed Worker source already contains the staleness fix. Fix: add a gated 'wrangler deploy' step for website/workers/** to that workflow so what is committed is what is deployed." + fail=1 + else + echo "PASS: (n) the served Docker pull count tracks Docker Hub (delta ${d} <= ${tol})" + fi + fi + # GitHub star count: authority is the repo's own API. + gh_stars="$(gh api repos/GetBusbar/busbar --jq .stargazers_count 2>/dev/null || echo -1)" + site_stars="$(curl -fsS --max-time 30 "https://getbusbar.com/api/stars" \ + | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("stargazers_count", d.get("stars", d.get("count",-1))))' || echo -1)" + echo "stars: github=${gh_stars} site-api=${site_stars}" + if [ "$gh_stars" -lt 0 ] || [ "$site_stars" -lt 0 ]; then + echo "::error::(n) could not read the star count from both api.github.com and getbusbar.com/api/stars (github=${gh_stars}, site=${site_stars})." + fail=1 + else + # Stars: a LIVE read equals the truth. 2 absorbs a genuine sub-minute race and nothing + # else. And a served value that EXCEEDS GitHub's is decisive on its own -- it can only + # be a value captured before someone unstarred, which is a frozen snapshot by + # definition, so it is called out separately below. + stol=2 + sd=$(( gh_stars > site_stars ? gh_stars - site_stars : site_stars - gh_stars )) + if [ "$sd" -gt "$stol" ]; then + if [ "$site_stars" -gt "$gh_stars" ]; then + echo "::error::(n) getbusbar.com/api/stars serves ${site_stars} while GitHub reports ${gh_stars}. A live read CANNOT exceed the truth: this value was captured before someone unstarred and has not been rewritten since, which is proof the refresh is dead rather than merely lagging. Same root cause as the pull count: the Worker is never deployed by CI (marketing deploy.yml runs 'wrangler pages deploy' only; the Worker's 'wrangler deploy' is a manual step in website/workers/README.md)." + else + echo "::error::(n) the star count getbusbar.com serves (${site_stars}) is ${sd} away from GitHub's (${gh_stars}), exceeding the ${stol} tolerance. Same failure shape as the Docker pull count: a number presented as live that is not." + fi + fail=1 + else + echo "PASS: (n) the served star count tracks GitHub (delta ${sd} <= ${stol})" + fi + fi + [ "$fail" = 0 ] diff --git a/crates/busbar/src/admin/tests/tests.rs b/crates/busbar/src/admin/tests/tests.rs index 9c8e2bfe..f5d0c5ec 100644 --- a/crates/busbar/src/admin/tests/tests.rs +++ b/crates/busbar/src/admin/tests/tests.rs @@ -1882,21 +1882,63 @@ async fn test_admin_v1_idempotency_reservation_frees_on_failure() { handle.abort(); } -/// A `Store` wrapper that sleeps on the FIRST `put_key` only, then runs at full speed — a stand-in -/// for a slow durable store's write round-trip, used to widen the window between "the mint has been -/// handed to the uncancellable blocking task" and "the mint actually lands" so a client disconnect -/// can be landed deterministically inside it. -struct SlowKeyStore { +/// A `Store` wrapper that PARKS the FIRST `put_key` on a rendezvous until the test explicitly +/// releases it, then runs at full speed. It stands in for a slow durable store's write round-trip, +/// but unlike a `sleep` it does not race a clock: it makes the window between "the mint has been +/// handed to the uncancellable blocking task" and "the mint actually lands" as wide as the test +/// needs and not one microsecond wider, and it tells the test EXACTLY when each edge is crossed. +/// +/// WHY NOT A SLEEP. The predecessor slept 500ms here and the test raced a 100ms client timeout +/// against it. Both the client's timer and the server's whole request path live on the SAME +/// single-threaded `#[tokio::test]` runtime, so one OS-level deschedule of that single worker +/// thread longer than the client's budget makes tokio observe the timeout as expired BEFORE it ever +/// polls the server far enough to reach `put_key`. The client errors, `first.is_err()` is satisfied +/// for entirely the wrong reason, and NOTHING is ever minted -- so no amount of polling afterwards +/// can observe a write that never started. That is what failed on a contended Windows runner during +/// the 1.5.3 release while the identical SHA passed on the same job minutes earlier. Reproduced +/// locally by shrinking the client budget under heavy CPU oversubscription; it fails with precisely +/// the old "never landed" message. The repair is not a longer timeout, which only makes the same +/// race rarer and slower. It is to stop having a race at all. +/// +/// THREE SIGNALS, no clocks in the happy path: +/// `entered` (store -> test) the mint is INSIDE `put_key` and has not written yet. +/// `release` (test -> store) the test is done arranging the disconnect; finish the write. +/// `landed` (store -> test) the write has committed. +/// +/// `release` blocks a BLOCKING-POOL thread, never a runtime worker: `put_key` is only ever reached +/// from inside the mint's `spawn_blocking` closure. If that ever stopped being true this would +/// block a runtime worker instead, which is itself the invariant this test exists to defend. +/// +/// The `recv_timeout` on `release` is a DEADLOCK BACKSTOP, not synchronisation. The mint holds the +/// process-wide `EXISTENCE_GATE` across this park, so a test that panicked between `entered` and +/// `release` would otherwise wedge every other key-mutating test in the binary forever. The backstop +/// turns that into a bounded, legible failure instead of a hung suite. Nothing in a passing run ever +/// waits on it. +struct GatedKeyStore { inner: Arc, - delay: std::time::Duration, + entered: tokio::sync::mpsc::Sender<()>, + release: std::sync::Mutex>>, + landed: tokio::sync::mpsc::Sender<()>, fired: std::sync::atomic::AtomicBool, } -impl crate::governance::Store for SlowKeyStore { +impl crate::governance::Store for GatedKeyStore { fn put_key(&self, key: &busbar_api::VirtualKey) -> crate::governance::StoreResult<()> { - if !self.fired.swap(true, std::sync::atomic::Ordering::SeqCst) { - std::thread::sleep(self.delay); + if self.fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + return self.inner.put_key(key); } - self.inner.put_key(key) + // FIRST `put_key` only: this is the mint the test wants to catch in flight. + let _ = self.entered.try_send(()); + if let Some(rx) = self + .release + .lock() + .unwrap_or_else(|e| e.into_inner()) + .take() + { + let _ = rx.recv_timeout(std::time::Duration::from_secs(30)); + } + let out = self.inner.put_key(key); + let _ = self.landed.try_send(()); + out } fn get_key(&self, id: &str) -> crate::governance::StoreResult> { self.inner.get_key(id) @@ -1992,91 +2034,139 @@ impl crate::governance::Store for SlowNthPutKeyStore { } } -/// The double-mint reachable via an ordinary client-side timeout shorter than a slow store's write: -/// the handler future is DROPPED mid-`config_transaction` (a client disconnect/timeout), but the -/// mint keeps running to completion on the uncancellable blocking task and DOES write the key. A -/// retry with the same `Idempotency-Key` must see the reservation still held (not an empty slot it -/// can double-mint into). +/// The double-mint reachable when a client goes away mid-mint: the handler future is DROPPED +/// mid-`config_transaction` (a client disconnect, whether from a timeout, a ^C or a closed laptop), +/// but the mint keeps running to completion on the uncancellable blocking task and DOES write the +/// key. A retry with the same `Idempotency-Key` must see the reservation still held (not an empty +/// slot it can double-mint into). +/// +/// FULLY DETERMINISTIC. Every ordering this test depends on is established by a rendezvous with +/// `GatedKeyStore` (see its doc comment for the wall-clock race this replaced), never by racing one +/// duration against another: +/// 1. the mint is provably INSIDE `put_key` before the client is taken away, +/// 2. the client is provably gone before the write is allowed to proceed, +/// 3. the write has provably committed before the retry is issued. +/// Step 2 also lets the test ASSERT the write had not landed yet, which the timeout-racing version +/// could not do: it had no way to tell a disconnect mid-mint from a disconnect after one. #[tokio::test] async fn an_idempotency_key_survives_a_client_disconnect_mid_mint() { crate::metrics::init(); let inner = Arc::new(MemoryStore::new()); - let slow_store: Arc = Arc::new(SlowKeyStore { + let (entered_tx, mut entered_rx) = tokio::sync::mpsc::channel::<()>(1); + let (landed_tx, mut landed_rx) = tokio::sync::mpsc::channel::<()>(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel::<()>(1); + let gated_store: Arc = Arc::new(GatedKeyStore { inner, - delay: std::time::Duration::from_millis(500), + entered: entered_tx, + release: std::sync::Mutex::new(Some(release_rx)), + landed: landed_tx, fired: std::sync::atomic::AtomicBool::new(false), }); - let gov = gov_with_signer(slow_store, Some("admintok".to_string())); + let gov = gov_with_signer(gated_store, Some("admintok".to_string())); let app = TestApp::new().governance(gov).build(); let router = crate::build_router(app); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let handle = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - // A short client-side timeout (100ms) vs. the store's 500ms write — a 5x margin, both real - // sleeps. - let short_timeout_client = reqwest::Client::builder() - .timeout(std::time::Duration::from_millis(100)) - .build() - .unwrap(); - let post = |client: &reqwest::Client| { + let keys_url = format!("http://{addr}/api/v1/admin/keys"); + let post = |client: &reqwest::Client, url: &str| { client - .post(format!("http://{addr}/api/v1/admin/keys")) + .post(url) .header("x-admin-token", "admintok") .header("content-type", "application/json") .header("idempotency-key", "dc-1") .body(r#"{"name": "disconnector"}"#) .send() }; + let count_disconnector = |v: &serde_json::Value| -> usize { + v["items"] + .as_array() + .map(|a| { + a.iter() + .filter(|k| k["name"].as_str() == Some("disconnector")) + .count() + }) + .unwrap_or(0) + }; - // First request: the client times out and errors BEFORE the 500ms store write completes, - // dropping the server-side handler future mid-mint. - let first = post(&short_timeout_client).await; - assert!(first.is_err(), "the short client timeout must fire first"); + // The request that will be abandoned. NO client timeout: the disconnect below is caused + // deliberately, so nothing here depends on a clock. + let doomed_url = keys_url.clone(); + let doomed = tokio::spawn(async move { + let client = reqwest::Client::new(); + post(&client, &doomed_url).await + }); - // Wait for the (uncancellable) blocking mint to actually land. POLL, do not sleep a fixed - // amount: a flat 800ms against a 500ms store write leaves only 300ms of slack, and a contended - // Windows runner eats that (observed: this test failed on `qa` with `got: []` while the SAME - // commit passed on `dev`). Polling keeps the assertion exact while making the WAIT adaptive, so - // a slow runner costs time instead of a false red. - { - let probe = reqwest::Client::new(); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); - loop { - let landed = probe - .get(format!("http://{addr}/api/v1/admin/keys")) - .header("x-admin-token", "admintok") - .send() - .await - .ok(); - if let Some(resp) = landed { - if let Ok(v) = resp.json::().await { - let n = v["items"] - .as_array() - .map(|a| { - a.iter() - .filter(|k| k["name"].as_str() == Some("disconnector")) - .count() - }) - .unwrap_or(0); - // Stop as soon as the mint is observable. If a SECOND key ever appears the - // assertions below still catch it, so this loop cannot mask the real defect. - if n >= 1 { - break; - } - } - } - assert!( - std::time::Instant::now() < deadline, - "the mid-mint blocking write never landed within 20s" - ); - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - } - } + // RENDEZVOUS 1: the mint is inside the store's `put_key` and has not written yet. + // + // THE `timeout` IS A LIVENESS BACKSTOP, NOT THE SYNCHRONISATION. The ordering this test needs is + // established by the rendezvous itself; the timeout only bounds how long a BROKEN harness may + // hang. It has to be here: if the mint never reaches `put_key`, the sender is still alive (the + // store outlives the test body), so `recv()` would block forever and the run would burn the CI + // job's whole timeout while saying nothing at all. The bound is deliberately enormous relative + // to the microseconds the happy path takes, so no amount of runner contention can reach it -- + // that asymmetry is exactly what the 500ms-sleep-versus-100ms-timeout version did not have. + // Every backstop below states what it was waiting for and what it saw instead. + const RENDEZVOUS_BACKSTOP: std::time::Duration = std::time::Duration::from_secs(60); + tokio::time::timeout(RENDEZVOUS_BACKSTOP, entered_rx.recv()) + .await + .expect( + "waited 60s for the mint to enter the store's put_key and it never did. The request \ + did not get as far as the blocking mint, so there is no mid-mint window to disconnect \ + inside and this test would be asserting nothing.", + ) + .expect("the gated store was dropped before the mint entered put_key"); + + // THE DISCONNECT. Dropping the in-flight request closes the connection, which is precisely what + // a client-side timeout does to the server, minus the race. The handler future is dropped; the + // already-scheduled `spawn_blocking` mint is not. + doomed.abort(); + let joined = doomed.await; + assert!( + joined.as_ref().is_err_and(|e| e.is_cancelled()), + "the doomed request must have been cancelled mid-flight; it instead completed, which means \ + the store gate did not hold the mint open and there was no mid-mint disconnect: {:?}", + joined.map(|r| r.map(|resp| resp.status())) + ); + + // A full probe round trip: it forces the runtime to poll the server task (reaping the closed + // connection) AND proves the write really has not landed yet, i.e. the disconnect above was + // genuinely mid-mint rather than after it. Note this read does not take `EXISTENCE_GATE`, which + // the parked mint is holding, so it cannot deadlock against it. + let probe = reqwest::Client::new(); + let mid: serde_json::Value = probe + .get(&keys_url) + .header("x-admin-token", "admintok") + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!( + count_disconnector(&mid), + 0, + "the mint must still be parked in put_key at this point, so no key should exist yet; \ + found one already committed, so the disconnect was not mid-mint: {mid:?}" + ); + + // RENDEZVOUS 2 and 3: let the abandoned mint finish, and wait for it to actually commit. + release_tx + .send(()) + .expect("the parked mint hung up on its release channel"); + tokio::time::timeout(RENDEZVOUS_BACKSTOP, landed_rx.recv()) + .await + .expect( + "waited 60s after releasing the parked mint and its write never committed. A mint \ + whose client disconnected must still run to completion on the uncancellable blocking \ + task; observed instead: released, then silence.", + ) + .expect("the gated store was dropped before the released mint could commit"); - // Retry with the SAME Idempotency-Key, no timeout this time. + // Retry with the SAME Idempotency-Key. let normal_client = reqwest::Client::new(); - let retry = post(&normal_client).await.unwrap(); + let retry = post(&normal_client, &keys_url).await.unwrap(); // Either outcome is correct under the fix: a 409 "already in flight" (if `dc-1`'s TTL window is // still open and the reservation was never cleared) or — since the mint already landed and the // cache slot may have been replaced by the real committed body before this retry runs — a replay diff --git a/scripts/promote.sh b/scripts/promote.sh new file mode 100755 index 00000000..dfeafb5d --- /dev/null +++ b/scripts/promote.sh @@ -0,0 +1,302 @@ +#!/usr/bin/env bash +# promote.sh -- move a promotion branch forward by fast-forward, safely and repeatably. +# +# scripts/promote.sh dev qa # promote dev to qa +# scripts/promote.sh qa main # promote qa to main (this is what cuts the release) +# scripts/promote.sh --selftest # prove this script's RED and GREEN paths before trusting it +# +# WHY THIS IS A SCRIPT AND NOT THREE COMMANDS TYPED AT RELEASE TIME. +# +# During the 1.5.3 promotion, `git push origin qa:main` was REJECTED as non-fast-forward while +# `main` was verifiably a strict ancestor of `qa` with ZERO divergent commits. A retry seconds later +# succeeded with nothing having changed in between. So the guard in use at the time -- +# `git merge-base --is-ancestor origin/main origin/qa` followed immediately by the push -- is not +# sufficient on its own, for a reason worth writing down: +# +# `origin/main` is a LOCAL remote-tracking ref. It is a photograph of the remote taken at fetch +# time. `git push` does not consult it; the SERVER re-checks fast-forwardness against whatever +# that particular backend replica currently believes. Between the fetch and the push those two can +# disagree, and the push loses. Nothing local is wrong, nothing has diverged, and the identical +# command works moments later. Checking harder locally cannot fix this, because the check and the +# thing being checked are on different machines. +# +# So this script does four things the ad-hoc commands did not: +# 1. Verifies STRICT ANCESTRY, and refuses loudly (never force-pushes) when it genuinely fails. +# 2. Verifies CI is green ON THE EXACT SHA being promoted, not on the branch NAME. A branch name +# resolves to whatever it points at now; a release is cut from a commit. Those differ precisely +# when someone pushed while you were reading the checks. +# 3. RETRIES a non-fast-forward rejection, re-fetching and RE-VERIFYING ancestry each time, so a +# transient replica disagreement is absorbed while a real divergence still stops the promotion. +# 4. VERIFIES THE REMOTE ACTUALLY MOVED afterwards, by reading it back with `git ls-remote`, +# rather than trusting the push's exit code. A zero exit is a claim; the remote ref is the fact. +# +# Idempotent: promoting a branch that is already at the target SHA is a no-op that exits 0. +set -euo pipefail + +REMOTE="${PROMOTE_REMOTE:-origin}" +PUSH_ATTEMPTS="${PROMOTE_PUSH_ATTEMPTS:-5}" +VERIFY_ATTEMPTS="${PROMOTE_VERIFY_ATTEMPTS:-5}" +BACKOFF="${PROMOTE_BACKOFF:-3}" +CHECK_CI=1 + +die() { echo "promote: $*" >&2; exit 1; } +note() { echo "promote: $*"; } + +# --- CI verdict for an exact SHA ------------------------------------------------------------- +# Branch names are not commits. Asking "is CI green on qa" answers a question about a name; asking +# "is CI green on 8f80889" answers the question a release actually depends on. +ci_is_green_for_sha() { + local sha="$1" json fails + command -v gh >/dev/null 2>&1 || { echo "promote: gh not installed" >&2; return 2; } + json="$(gh run list --commit "$sha" --json workflowName,status,conclusion --limit 100 2>/dev/null)" || return 2 + python3 - "$sha" <<'PY' <<<"$json" +import json, sys +runs = json.load(sys.stdin) +sha = sys.argv[1] +if not runs: + print("promote: NO workflow runs at all for %s." % sha, file=sys.stderr) + print("promote: an unverified commit must not be promoted. If CI genuinely did not run for this", + file=sys.stderr) + print("promote: commit, push it to a branch and let CI run, or re-run CI on it explicitly.", + file=sys.stderr) + sys.exit(1) +bad = [r for r in runs if r["status"] != "completed" + or r["conclusion"] not in ("success", "skipped", "neutral")] +running = [r for r in bad if r["status"] != "completed"] +failed = [r for r in bad if r["status"] == "completed"] +for r in sorted(runs, key=lambda r: r["workflowName"]): + print("promote: %-24s %s/%s" % (r["workflowName"], r["status"], r["conclusion"])) +if running: + print("promote: STILL RUNNING on %s: %s" % (sha, ", ".join(r["workflowName"] for r in running)), + file=sys.stderr) + print("promote: wait for it. Promoting on a run that has not finished is promoting on a guess.", + file=sys.stderr) + sys.exit(1) +if failed: + print("promote: FAILED on %s: %s" % (sha, ", ".join(r["workflowName"] for r in failed)), + file=sys.stderr) + sys.exit(1) +print("promote: every workflow run for %s is green (%d run(s))." % (sha, len(runs))) +PY +} + +# --- ancestry -------------------------------------------------------------------------------- +assert_strict_ancestor() { + local dst_sha="$1" src_sha="$2" dst="$3" src="$4" + if git merge-base --is-ancestor "$dst_sha" "$src_sha"; then + return 0 + fi + echo "promote: REFUSING. ${dst} (${dst_sha}) is NOT an ancestor of ${src} (${src_sha})." >&2 + echo "promote: these branches have genuinely diverged, so a fast-forward is not possible and" >&2 + echo "promote: this script will not force anything. Commits on ${dst} that are not on ${src}:" >&2 + git --no-pager log --oneline "${src_sha}..${dst_sha}" | sed 's/^/promote: /' >&2 || true + echo "promote: resolve the divergence (merge or rebase ${dst} into ${src}), then re-run." >&2 + return 1 +} + +remote_sha() { git ls-remote "$REMOTE" "refs/heads/$1" | awk '{print $1}'; } + +promote() { + local src="$1" dst="$2" i src_sha dst_sha pushed=0 + + note "fetching ${REMOTE}" + git fetch "$REMOTE" --prune --quiet + + src_sha="$(remote_sha "$src")" + [ -n "$src_sha" ] || die "${REMOTE}/${src} does not exist" + dst_sha="$(remote_sha "$dst")" + [ -n "$dst_sha" ] || die "${REMOTE}/${dst} does not exist" + + note "${src} = ${src_sha}" + note "${dst} = ${dst_sha}" + + if [ "$src_sha" = "$dst_sha" ]; then + note "${dst} is already at ${src_sha}; nothing to promote (idempotent no-op)." + return 0 + fi + + assert_strict_ancestor "$dst_sha" "$src_sha" "$dst" "$src" || return 1 + note "ancestry OK: ${dst} is a strict ancestor of ${src}, so this is a fast-forward." + note "commits this promotion moves onto ${dst}:" + git --no-pager log --oneline "${dst_sha}..${src_sha}" | sed 's/^/promote: /' + + if [ "$CHECK_CI" = 1 ]; then + note "checking CI on the exact SHA being promoted (${src_sha})" + ci_is_green_for_sha "$src_sha" || die "CI is not green on ${src_sha}; refusing to promote." + else + echo "promote: WARNING: --no-ci-check given. Promoting ${src_sha} to ${dst} WITHOUT" >&2 + echo "promote: verifying any workflow result. Use this only when you already know why." >&2 + fi + + for (( i = 1; i <= PUSH_ATTEMPTS; i++ )); do + note "push attempt ${i}/${PUSH_ATTEMPTS}: ${src_sha} -> ${REMOTE}/${dst}" + if git push "$REMOTE" "${src_sha}:refs/heads/${dst}"; then + pushed=1 + break + fi + if [ "$i" -eq "$PUSH_ATTEMPTS" ]; then break; fi + # A rejection is only ever retried after RE-ESTABLISHING that the fast-forward is still legal. + # If someone really did push to the destination in the meantime, ancestry now fails and we + # refuse instead of hammering. This is the difference between absorbing a replica disagreement + # and papering over a divergence. + note "rejected; re-fetching and re-verifying before retrying in ${BACKOFF}s" + sleep "$BACKOFF" + git fetch "$REMOTE" --prune --quiet + dst_sha="$(remote_sha "$dst")" + if [ "$dst_sha" = "$src_sha" ]; then + note "${dst} is now already at ${src_sha}: a concurrent promoter won the race. Nothing to do." + pushed=1 + break + fi + assert_strict_ancestor "$dst_sha" "$src_sha" "$dst" "$src" || return 1 + done + [ "$pushed" = 1 ] || die "push to ${dst} still rejected after ${PUSH_ATTEMPTS} attempts." + + # THE PUSH'S EXIT CODE IS A CLAIM. The remote ref is the fact. Read it back. + for (( i = 1; i <= VERIFY_ATTEMPTS; i++ )); do + dst_sha="$(remote_sha "$dst")" + if [ "$dst_sha" = "$src_sha" ]; then + note "VERIFIED: ${REMOTE}/${dst} now reads ${dst_sha}." + note "promoted ${src} -> ${dst}." + return 0 + fi + note "post-push read-back ${i}/${VERIFY_ATTEMPTS}: ${dst} reads ${dst_sha:-}, want ${src_sha}" + [ "$i" -lt "$VERIFY_ATTEMPTS" ] && sleep "$BACKOFF" + done + echo "promote: PUSH REPORTED SUCCESS BUT ${REMOTE}/${dst} DID NOT MOVE." >&2 + echo "promote: wanted ${src_sha}, remote still reads ${dst_sha:-}. Do NOT assume the" >&2 + echo "promote: promotion happened: check the remote by hand before doing anything else." >&2 + return 1 +} + +# --- selftest -------------------------------------------------------------------------------- +# A promotion gate nobody has watched REFUSE is a promotion gate nobody should trust with a release. +# This builds throwaway local repos and drives every path: the fast-forward, the idempotent no-op, +# the divergence refusal, the transient-rejection retry (the exact 1.5.3 symptom), and the +# read-back verification that catches a push which did not land. +selftest() { + local root rc=0 + root="$(mktemp -d)" + trap 'rm -rf "$root"' RETURN + export GIT_AUTHOR_NAME=selftest GIT_AUTHOR_EMAIL=selftest@example.com + export GIT_COMMITTER_NAME=selftest GIT_COMMITTER_EMAIL=selftest@example.com + + fresh() { + rm -rf "$root/bare" "$root/work" + git init --quiet --bare "$root/bare" + git init --quiet "$root/work" + ( + cd "$root/work" + git remote add origin "$root/bare" + echo a > f; git add f; git commit --quiet -m base + git branch -M dev + git push --quiet origin dev:refs/heads/dev dev:refs/heads/qa + echo b > f; git commit --quiet -am second + git push --quiet origin dev:refs/heads/dev + ) + } + + check() { # check