diff --git a/.github/artifact-contract.json b/.github/artifact-contract.json new file mode 100644 index 00000000..2f11430c --- /dev/null +++ b/.github/artifact-contract.json @@ -0,0 +1,232 @@ +{ + "_comment": [ + "THE ARTIFACT CONTRACT: what must be true of EVERY shipped busbar binary, as DATA.", + "", + "Owner, 2026-08-08: \"should just be 1 build pipeline that takes what its building: arm, windows,", + "mac, but it does the same thing for each no way for 1 to be different\", and \"100% or 0% for", + "each build\".", + "", + "WHY THIS IS A FILE AND NOT A SEQUENCE OF `run:` STEPS. Through 1.5.3 every property a release", + "binary was supposed to have was asserted on the INPUT side -- an env var set on a build step --", + "and never on the OUTPUT. `BUSBAR_RELEASE_PUBKEY` was set on both build steps, the workflow was", + "green, a comment in release.yml stated the key was baked in, and", + "busbar-aarch64-unknown-linux-gnu 1.5.3 shipped with no key at all: `option_env!` is a", + "COMPILE-time read that fails silently to `None`. Setting an env var says what was intended.", + "Only inspecting the shipped bytes says what happened.", + "", + "Because the rows are data, adding a property is ONE ENTRY here plus its check function in", + "scripts/verify-artifact.py, and it then applies to every target automatically -- there is no", + "per-target list to forget a platform in, which is how the aarch64 leg went three releases", + "without a key. scripts/verify-artifact.py asserts SET EQUALITY between the ids declared here", + "and the checks it implements, so a row added here with no implementation is a hard failure", + "rather than a silently-skipped property, and an implementation with no row here is too.", + "scripts/tests/test_verify_artifact.py additionally requires every row to own BOTH a RED", + "fixture it flags and a GREEN twin it passes: a row nobody has watched fail is not a row.", + "", + "`applies_when` is the ONLY way a row may be conditional, it is matched against the target's", + "own declaration in .github/release-targets.json, and a row that applies to a target CANNOT be", + "skipped for any other reason -- there is no waiver field and no exception list. Every target", + "now builds and verifies on a NATIVE runner, so `exec: true` rows run for every platform,", + "Windows included; nothing is statically approximated because a runner could not execute it.", + "", + "`min_rows` is a floor. A loop over a discovered set with no floor is a check that passes", + "without executing: an empty or truncated contract file would otherwise verify every artifact", + "against nothing and report green." + ], + "min_rows": 14, + "rows": [ + { + "id": "archive_shape", + "title": "the archive contains exactly the declared executable, at a plausible size", + "exec": false, + "why": [ + "GitHub creates an asset row the moment an upload starts, so a truncated or empty upload", + "lists identically to a good one, and an archive that unpacks to the wrong name (or to a", + "directory) breaks install.sh and the documented `tar -xzf && ./busbar` in the same silent", + "way a missing asset does. Assert the members, not the presence." + ], + "applies_when": { + "kind": "binary" + } + }, + { + "id": "binary_format", + "title": "architecture, object format and linkage are what the target declares", + "exec": false, + "why": [ + "An asset named `aarch64` that contains x86_64 bytes is 100% broken for everyone who", + "downloads it, and it is invisible to every check that only executes the binary on the", + "runner that built it. Parsed straight out of the ELF/Mach-O/PE headers so the check is one", + "implementation on all three operating systems rather than `file` here and `ldd` there.", + "The linkage half folds in docker.yml's hard failure on a dynamically-linked binary where a", + "static one was intended (FROM scratch has no loader): linkage is now a declared property of", + "the target and a row that applies to every artifact, not an `ldd | grep` in one workflow." + ], + "applies_when": { + "kind": "binary" + } + }, + { + "id": "release_pubkey", + "title": "the release public key is embedded in the shipped bytes", + "exec": false, + "why": [ + "THE 1.5.3 DEFECT, ASSERTED ON THE ARTIFACT. busbar-aarch64-unknown-linux-gnu shipped with", + "no embedded key in 1.5.1, 1.5.2 and 1.5.3 while the other four assets had one. On ARM Linux", + "-- Graviton, Raspberry Pi, ARM containers -- every correctly-signed first-party plugin was", + "refused, and the only workaround was `plugins.trust.allow_unsigned: true`, which switches", + "the requirement off rather than trusting first-party selectively.", + "This row is the cheap static half: the 64-hex public key must appear verbatim in the binary.", + "`first_party_plugin` is the half that proves it is the RIGHT key and that it works." + ], + "applies_when": { + "kind": "binary" + } + }, + { + "id": "first_party_plugin", + "title": "a real signed first-party plugin verifies as first-party and is ready", + "exec": true, + "why": [ + "Functional, not a grep. The plugin is a REAL release of a REAL first-party plugin repo,", + "signed by the REAL private half of the release key, downloaded and handed to the SHIPPED", + "binary, which must report `SIGNATURE: first-party` and `STATUS: ready` from --list-plugins.", + "A grep proves 64 bytes are present; only this proves the embedded key verifies the", + "signatures the plugin repos actually produce. Run against the real 1.5.3 aarch64 asset it", + "reports, verbatim: `SIGNATURE: unsigned STATUS: SKIPPED: manifest claims first-party", + "publisher 'busbar' but this build embeds no busbar release key`.", + "The check also asserts `plugins.enabled: true` and a non-empty inventory in the output,", + "because --list-plugins falls back to the DEFAULT plugins block with a mere [warn] when the", + "config is unreadable, and a naive check would then read `no plugin tarballs found` and see", + "no failure text at all." + ], + "applies_when": { + "kind": "binary", + "published": true + } + }, + { + "id": "version_anchored", + "title": "--version reports exactly the version being released", + "exec": true, + "why": [ + "ANCHORED on both ends. A substring or prefix match lets `1.5.4` be satisfied by `1.5.40`,", + "and lets a stale binary from a previous matrix leg pass while claiming to be this release.", + "The bytes must run and identify themselves as this exact version, nothing more." + ], + "applies_when": { + "kind": "binary" + } + }, + { + "id": "quickstart_boots", + "title": "the documented quickstart boots this binary and answers on /healthz", + "exec": true, + "why": [ + "The minimal config.yaml from docs/getting-started.md, the documented default listen address,", + "and the documented `curl localhost:8080/healthz` -> `ok`. A new user's first five minutes,", + "run against the artifact they will actually download. This used to be checked for exactly", + "ONE platform -- verify-deploy.yml hardcodes busbar-x86_64-unknown-linux-gnu.tar.gz -- which", + "is the same shape of gap as the missing key: a property established on one artifact and", + "assumed for the rest." + ], + "applies_when": { + "kind": "binary" + } + }, + { + "id": "attestation", + "title": "the build-provenance attestation verifies against these exact bytes", + "exec": false, + "why": [ + "`gh attestation verify --repo GetBusbar/busbar` is the command the docs tell users", + "to run, so it is the command that must be proven to pass -- on the bytes downloaded from the", + "release, not on a local copy of the file the build step happened to still have. An", + "attestation that covers a different digest than the asset a user downloads is worse than", + "none, because it looks like provenance.", + "Gated on `published`: an artifact packaged into an image rather than", + "uploaded has no release asset to attest against, and a row that cannot", + "run must not be quietly satisfied." + ], + "applies_when": { + "kind": "binary", + "published": true + } + }, + { + "id": "build_evidence", + "title": "the build's own evidence is bound to the shipped bytes by digest", + "exec": false, + "why": [ + "The build step records the SHA-256 of the archive it produced. This row asserts that digest", + "equals the digest of the archive that was actually downloaded from the release. Without it", + "every evidence-based row below is unanchored: a build could report a verified PGO profile", + "for bytes that never reached the release, which is a certificate for the wrong artifact." + ], + "applies_when": { + "kind": "binary" + } + }, + { + "id": "pgo_applied", + "title": "PGO really was applied where the target declares it", + "exec": false, + "applies_when": { + "pgo": true, + "kind": "binary" + }, + "why": [ + "scripts/pgo-build.sh writes its proof marker ONLY after a non-empty merged profile fed a", + "successful -Cprofile-use build. This row turns that marker into a contract row rather than", + "an assertion living in one workflow step: the marker must exist, be marked verified, name", + "THIS target, and record a non-zero merged-profile size and .profraw count -- and it is bound", + "to the shipped archive's digest by `build_evidence`, so it certifies these bytes and not", + "some other build's.", + "`applies_when` is checked against .github/release-targets.json, and that file carries a", + "`pgo_floor` scripts/tests/test_release_contract.py enforces, so a target cannot quietly flip `pgo` to false to", + "make this row stop applying to it." + ] + }, + { + "id": "image_boots_documented_quickstart", + "applies_when": { + "kind": "image" + }, + "why": "busbar 1.5.3's image did NOT boot under any documented invocation: USER 65532:65532 against a root-owned /etc/busbar, so the overlay backend was unwritable and boot refused. The build was green and nothing ran the image. This row runs it." + }, + { + "id": "image_runs_as_nonroot", + "applies_when": { + "kind": "image" + }, + "why": "the image ships a non-root UID deliberately; a fix that restores boot by running as root would trade a boot failure for a privilege regression and look identical in CI." + }, + { + "id": "image_release_pubkey", + "applies_when": { + "kind": "image" + }, + "why": "the same embedded-key property the binaries owe. The image carries its own musl build, so a key present in the tarballs proves nothing about it." + }, + { + "id": "image_version_anchored", + "applies_when": { + "kind": "image" + }, + "why": "anchored so 1.5.4 cannot be satisfied by 1.5.40, and read from the running image rather than from the tag that labels it." + }, + { + "id": "image_matches_packaged_binary", + "applies_when": { + "kind": "image" + }, + "why": [ + "the binary INSIDE the image must be byte-identical to the musl artifact the single", + "build path produced for it. This is what makes the image a PACKAGE rather than a", + "second build: without it the image could be assembled from a differently-built", + "binary and every other row would still pass \u2014 which is how one artifact came to", + "embed the plugin release key while its sibling did not." + ] + } + ] +} \ No newline at end of file diff --git a/.github/release-targets.json b/.github/release-targets.json new file mode 100644 index 00000000..834d4609 --- /dev/null +++ b/.github/release-targets.json @@ -0,0 +1,217 @@ +{ + "_comment": [ + "THE PLATFORM LIST, IN EXACTLY ONE PLACE, AND NOW ALSO THE ONLY PLACE A TARGET MAY DIFFER.", + "", + "Every consumer derives from this file; nothing hardcodes a parallel copy.", + "", + " .github/workflows/build-artifact.yml builds ONE artifact for ONE target. Everything that", + " varies per target is a FIELD BELOW, passed in as a workflow input. There is no", + " per-target `if:`, no second build step and no second build path anywhere in it.", + " release.yml `targets` job reads it to build the build matrix, the VERIFY matrix, and the", + " exact set of asset filenames the release owes. `verify-assets` asserts every", + " name is present; `verify-artifact` runs the whole contract against each one.", + " verify-deploy.yml fetches it over raw.githubusercontent AT THE TAG UNDER TEST (it checks out", + " nothing on purpose) and derives the same expected names, so the post-deploy", + " verifier and the release that produced the assets can never disagree.", + "", + "WHY IT IS A FILE AND NOT A LITERAL IN EACH WORKFLOW. v1.5.3 published FIVE assets where seven", + "were expected: aarch64-apple-darwin and x86_64-unknown-linux-gnu were both missing, which is", + "Apple Silicon Mac and x86_64 Linux. A COUNT can never see a missing platform, only a NAME can.", + "", + "WHY EVERY TARGET IS NOW NATIVE, WHICH IS THE 1.5.4 FIX. Through 1.5.3 two of these targets were", + "CROSS-compiled (aarch64-linux on an x86_64 host, x86_64-darwin on an arm64 host) and therefore", + "could not run the PGO trainer, so they took a DIFFERENT build step from the PGO targets. Two", + "build steps meant two env blocks, and BUSBAR_RELEASE_PUBKEY was carried by both -- but nothing", + "compared their OUTPUTS, so an artifact could be produced on one path with a property the other", + "path's artifacts had, and nothing noticed. It did: busbar-aarch64-unknown-linux-gnu 1.5.3 (and", + "1.5.2, and 1.5.1) shipped with NO embedded release public key, so every correctly-signed", + "first-party plugin was refused on ARM Linux with `SIGNATURE: unsigned`. Verified against the", + "real 1.5.3 asset and the real signed store-sqlite 1.0.4 tarball.", + "", + "GitHub now offers a native runner for every one of these five targets -- `ubuntu-24.04-arm` for", + "ARM Linux and `macos-15-intel` for Intel macOS, both confirmed available to this org and green", + "on a probe run. So the cross-vs-native divergence is not mitigated, it is DELETED: every target", + "builds on its own architecture, every target runs its own PGO training, and every target is", + "VERIFIED on a runner that can execute it, so no contract row has to be skipped anywhere.", + "", + "FIELDS, ALL OF WHICH ARE INPUTS TO ONE IDENTICAL BUILD, NEVER SELECTORS FOR A DIFFERENT ONE:", + " target the rust target triple; the artifact is always `busbar-.`.", + " runner the GitHub-hosted runner label. MUST be native for `target` -- `native` below is", + " what asserts it, and scripts/tests/test_release_contract.py fails if any target claims", + " native on a runner of a different architecture.", + " pgo whether scripts/pgo-build.sh trains this target. Requires a native runner. See", + " `pgo_floor` below for why this cannot quietly drift to false.", + " archive the archive extension the packaging step produces.", + " exe the binary's filename inside the archive.", + " arch the machine architecture the shipped binary MUST report. This is a contract row:", + " an artifact named aarch64 that contains x86_64 bytes is a 100%-broken artifact.", + " format the object format the shipped binary MUST be (elf / macho / pe).", + " linkage `dynamic` or `static`. The release binaries are glibc/libSystem/MSVC dynamic; the", + " docker.yml musl binaries are static because FROM scratch has no loader. Declaring", + " it per target means the linkage assertion is one contract row rather than an ad-hoc", + " `ldd | grep` in one workflow and nowhere else.", + " plugin_asset the asset name of the REAL signed first-party plugin used to prove, on this", + " platform, that the shipped binary verifies a first-party signature. Functional, not", + " a grep: it is the check that would have caught the aarch64 defect on 1.5.1." + ], + "plugin_probe": { + "_comment": [ + "The known-good signed first-party plugin the `first_party_plugin` contract row runs against.", + "It is a REAL release of a REAL first-party plugin repo, signed by the REAL private half of", + "BUSBAR_RELEASE_PUBKEY -- which is the only thing that can prove the shipped busbar embeds the", + "matching public half. A self-packed fixture signed by an ephemeral key would prove only that", + "the signature code compiles (scripts/signing-gate.sh already proves that, on a binary it", + "builds itself); it would have gone green on the broken 1.5.3 aarch64 artifact." + ], + "repo": "GetBusbar/store-sqlite", + "tag": "v1.0.4", + "alias": "sqlite", + "expect_signature": "first-party", + "expect_status": "ready" + }, + "pgo_floor": 4, + "targets": [ + { + "target": "x86_64-unknown-linux-gnu", + "runner": "ubuntu-latest", + "pgo": true, + "archive": "tar.gz", + "exe": "busbar", + "arch": "x86_64", + "format": "elf", + "linkage": "dynamic", + "plugin_asset": "busbar-store-sqlite-1.0.4-x86_64-unknown-linux-gnu.tar.gz", + "kind": "binary", + "published": true, + "packaged_into": "" + }, + { + "target": "aarch64-unknown-linux-gnu", + "runner": "ubuntu-24.04-arm", + "pgo": true, + "archive": "tar.gz", + "exe": "busbar", + "arch": "aarch64", + "format": "elf", + "linkage": "dynamic", + "plugin_asset": "busbar-store-sqlite-1.0.4-aarch64-unknown-linux-gnu.tar.gz", + "kind": "binary", + "published": true, + "packaged_into": "" + }, + { + "target": "x86_64-apple-darwin", + "runner": "macos-15-intel", + "pgo": true, + "archive": "tar.gz", + "exe": "busbar", + "arch": "x86_64", + "format": "macho", + "linkage": "dynamic", + "plugin_asset": "busbar-store-sqlite-1.0.4-x86_64-apple-darwin.tar.gz", + "kind": "binary", + "published": true, + "packaged_into": "" + }, + { + "target": "aarch64-apple-darwin", + "runner": "macos-latest", + "pgo": true, + "archive": "tar.gz", + "exe": "busbar", + "arch": "aarch64", + "format": "macho", + "linkage": "dynamic", + "plugin_asset": "busbar-store-sqlite-1.0.4-aarch64-apple-darwin.tar.gz", + "kind": "binary", + "published": true, + "packaged_into": "" + }, + { + "_pgo_comment": [ + "The ONE target that does not train a profile, and the reason is written down rather than", + "implied. scripts/pgo-build.sh drives the trainer with a POSIX shell pipeline (background", + "jobs, `pkill -P`, an argv0 with no `.exe`); it has never been run on a Windows runner and", + "an untested trainer in the release path is a red release, not an optimisation. The runner", + "is still NATIVE, so this target takes the SAME single build step as the other four and the", + "same env block -- `pgo` is a parameter to that one step, not a second code path. Every", + "contract row except `pgo_applied` therefore applies here unchanged, and `pgo_floor` above", + "means a second target cannot join it without the count floor failing." + ], + "target": "x86_64-pc-windows-msvc", + "runner": "windows-latest", + "pgo": false, + "archive": "zip", + "exe": "busbar.exe", + "arch": "x86_64", + "format": "pe", + "linkage": "dynamic", + "plugin_asset": "busbar-store-sqlite-1.0.4-x86_64-pc-windows-msvc.tar.gz", + "kind": "binary", + "published": true, + "packaged_into": "" + }, + { + "target": "x86_64-unknown-linux-musl", + "runner": "ubuntu-latest", + "pgo": true, + "archive": "tar.gz", + "exe": "busbar", + "arch": "x86_64", + "format": "elf", + "linkage": "static", + "kind": "binary", + "published": false, + "packaged_into": "image-linux-amd64", + "plugin_asset": "" + }, + { + "target": "aarch64-unknown-linux-musl", + "runner": "ubuntu-24.04-arm", + "pgo": true, + "archive": "tar.gz", + "exe": "busbar", + "arch": "aarch64", + "format": "elf", + "linkage": "static", + "kind": "binary", + "published": false, + "packaged_into": "image-linux-arm64", + "plugin_asset": "" + }, + { + "target": "image-linux-amd64", + "kind": "image", + "platform": "linux/amd64", + "runner": "ubuntu-latest", + "arch": "x86_64", + "pgo": true, + "exe": "busbar", + "format": "elf", + "linkage": "static", + "archive": "oci", + "plugin_asset": "", + "published": false, + "packaged_into": "" + }, + { + "target": "image-linux-arm64", + "kind": "image", + "platform": "linux/arm64", + "runner": "ubuntu-24.04-arm", + "arch": "aarch64", + "pgo": true, + "exe": "busbar", + "format": "elf", + "linkage": "static", + "archive": "oci", + "plugin_asset": "", + "published": false, + "packaged_into": "" + } + ], + "metadata_assets": [ + "busbar-{tag}.cdx.json", + "busbar-openapi-{tag}.json" + ] +} \ No newline at end of file diff --git a/.github/workflows/a2a-conformance.yml b/.github/workflows/a2a-conformance.yml new file mode 100644 index 00000000..b04d05ac --- /dev/null +++ b/.github/workflows/a2a-conformance.yml @@ -0,0 +1,670 @@ +name: A2A conformance + +# TWO INDEPENDENT INSTRUMENTS, POINTED AT THE SAME PROTOCOL, PLUS A GOVERNANCE PROBE THAT IS +# DELIBERATELY NOT PART OF EITHER VERDICT. +# +# testing/a2a-harness/ an independent battery written from the published A2A specification +# alone, with adversarial and hostile-peer coverage. It found a real +# defect in a reference implementation. +# testing/a2a-tck/ a wrapper around a2aproject/a2a-tck, the publisher's OWN suite, which +# covers all three transports including gRPC across 36 test modules. +# Fetched at a pinned commit, never vendored (see its LICENSING.md). +# testing/a2a-governance/ budgets, quarantine, trust lifecycle. PRODUCT policy, not protocol. +# It imports the harness as a library and can never contribute to a +# conformance verdict -- the harness RAISES if a governance test is ever +# registered inside it. A perfectly conformant agent that ignores every +# budget and never quarantines anything scores 100% on conformance. +# +# WHY THIS WORKFLOW IS HERE AND NOT WHERE THE BATTERIES WERE WRITTEN. Two workflows once sat in the +# private design repository asking for `ubuntu-latest`. That repository is hosted on an internal +# Gitea instance with no registered runners AND no route from GitHub-hosted runners, so eight jobs +# failed permanently and the only two that went green had executed nothing. A standing red nobody +# can fix teaches everyone to ignore the signal, which is the same defect these batteries exist to +# catch, one level up. A conformance battery is a statement ABOUT busbar, so it belongs where +# busbar is built and where a red blocks the release it is about. Independence is a property of +# AUTHORSHIP, not of location: these were written without reading busbar's implementation, and the +# guard that keeps product knowledge out of the harness is enforced in code, not by filesystem +# distance. +# +# busbar is PUBLIC, so per the org rule (public -> GitHub-hosted, private -> busbar-selfhosted) +# every job here runs on `ubuntu-latest` at no cost, and nothing needs provisioning. There is no +# secret anywhere in this file, which is what makes "the control legs run ALWAYS" achievable rather +# than aspirational. +# +# THE CONTROL LEGS RUN ALWAYS. A battery that cannot judge a known-good peer cannot be trusted to +# judge ours, so every run re-establishes that both instruments still produce the pinned verdict +# against pinned third-party references. +# +# THE SUBJECT LEG IS ARMED OR RED. This is a REVERSAL of the previous policy and it is the point of +# this edit. The leg used to SKIP until `vars.BUSBAR_A2A_ENDPOINT` named a deployment, and it did +# not fail — the argument being that a job red for a reason that is not a defect is how red stops +# meaning defect. In practice the variable was NEVER SET: the check named `subject (busbar's own +# A2A endpoint)` reported `success` on every run with both of its real steps `skipped`, so there +# has never been an A2A conformance number of any kind. A leg that renders as the identical green +# tick whether it judged busbar or judged nothing is the exact false green the rest of this file is +# arranged to refuse. +# +# So the arm is no longer a URL. It is a busbar BINARY BUILT FROM THE COMMIT UNDER TEST, booted on +# loopback by `scripts/a2a-subject/boot.sh` — the same treatment, for the same reasons, that the +# sibling MCP battery already gives its subject: a release gate that depends on a live deployment +# produces two unreadable verdicts, a green meaning "the deployment was fine yesterday" and a red +# meaning "somebody redeployed", and neither is a statement about the commit under test. +# `vars.BUSBAR_A2A_ENDPOINT` survives as an OPTIONAL EXTRA leg for an operator who also wants a +# real deployment judged. +# +# AND `verdict` IS NOT OPTIONAL. Ten green ticks mean nothing if one of them is green because it +# never ran. The last job asserts, per leg, that the leg reached `success` -- a skipped or +# cancelled control leg is RED there. That is the only required check. + +on: + push: + branches: ['**'] + pull_request: + workflow_dispatch: + +concurrency: + group: a2a-conformance-${{ github.ref }} + cancel-in-progress: true + +env: + # NOT COSMETIC, AND NOT A PREFERENCE. a2a-go v2.4.0 serialises task status timestamps in the + # HOST's local zone instead of UTC, violating SPEC 5.6.1. On a UTC host the offset is zero, the + # bytes end in `Z` anyway, and the defect DISAPPEARS -- and CI runners are UTC. Running the + # control legs in UTC would silently retire a real third-party finding. The `tz-is-load-bearing` + # job re-runs the same control under TZ=UTC and REQUIRES the pinned baseline to break, so this + # line can never quietly become decoration. + TZ: America/New_York + A2AHT_CONTROL_BIN: /home/runner/.a2aht/bin + A2A_TCK_WORK: /home/runner/.a2a-tck + +jobs: + # ---------------------------------------------------------------- the battery's own machinery + harness-selftest: + name: harness selftest (before believing any verdict) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v5 + with: { python-version: '3.12' } + - name: The battery's own guards, each made to fail + run: | + set -euo pipefail + python3 testing/a2a-harness/scripts/harness-selftest.py + + # The comparator is machinery too, and it is the piece standing between us and the TCK's own + # `grpc: 0/72 (72 skipped)` tick. + - name: The TCK baseline comparator, each guard made to fail + run: | + set -euo pipefail + python3 testing/a2a-tck/check-baseline-selftest.py + + # The SUBJECT leg's own machinery, and it is machinery for exactly the reason the two above + # are: `NOT ARMED, SO NOT RUN` is now a RED state, and a rule whose enforcement is only ever + # exercised by the real thing is a rule nobody has watched work. This drives the arming + # transition in both directions, proves a non-existent subject binary does not count as an + # arm, proves the audience-boundary disproof fails against a peer that admits every + # credential, and proves the TCK number cannot be read from a run that reported nothing. + - name: The subject leg's arming rule and boundary proof must BITE + run: ./scripts/a2a-subject/boot.sh --selftest + + # The aggregator's `needs:` list is itself a hand-maintained enumeration, and the last + # enumeration in this tree that stopped covering what came after it did so silently. So the + # verdict's dependency set is held to SET EQUALITY with the workflow's job set, in both + # directions, and every leg it depends on must actually be read by its script. + - name: The verdict must depend on, and judge, every leg + run: | + set -euo pipefail + python3 -m pip install --quiet pyyaml + python3 testing/verdict-covers-every-leg.py + + # -------------------------------------------------------- instrument 1: the independent battery + control-a2a-go: + name: control a2a-go (${{ matrix.binding }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - binding: rest + transport: http_json + baseline: control-a2a-go-rest.json + - binding: jsonrpc + transport: jsonrpc + baseline: control-a2a-go-jsonrpc.json + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v5 + with: { python-version: '3.12' } + - uses: actions/setup-go@v5 + with: { go-version: '1.24' } + - name: Install the pinned control + run: | + set -euo pipefail + testing/a2a-harness/scripts/install-control.sh go + + - name: Run the battery against the control + working-directory: testing/a2a-harness + run: | + set -euo pipefail + mkdir -p reports + # `--allow-red` so the RUN's own exit code is not the verdict. The verdict is the + # baseline comparison in the next step: a control is allowed to have known deviations, + # it is not allowed to have DIFFERENT ones from yesterday. + python3 -m a2aht run \ + --launch "$A2AHT_CONTROL_BIN/a2a serve --echo --port 9099 --quiet \ + ${{ matrix.binding == 'jsonrpc' && '--transport jsonrpc' || '' }}" \ + --port 9099 \ + --label "control:a2a-go/${{ matrix.binding }}" \ + --tier pre-release \ + --client-drive "$A2AHT_CONTROL_BIN/a2a send {url} hello-from-harness" \ + --known-deviations baselines/known-deviations-a2a-go.json \ + --json reports/control.json --allow-red + + - name: The control must still produce its pinned verdict + working-directory: testing/a2a-harness + run: | + set -euo pipefail + python3 -m a2aht baseline \ + --report reports/control.json \ + --baseline "baselines/${{ matrix.baseline }}" + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: a2a-battery-control-${{ matrix.binding }} + path: testing/a2a-harness/reports/ + + control-a2a-python: + name: control a2a-python (the second, independent oracle) + runs-on: ubuntu-latest + # A HANG IS A FAILURE, AND IT MUST LOOK LIKE ONE WITHIN MINUTES. The battery has no overall + # deadline of its own: if a launched control never becomes ready, `--launch` waits, and the job + # sits amber for hours. Amber is not a verdict, and a leg nobody can read the result of is the + # same false signal as a leg that passed without executing. So the job is bounded, and the + # bound is well above the ~2 minutes the other control legs take. + timeout-minutes: 12 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v5 + with: { python-version: '3.12' } + - name: Install the pinned control + run: | + set -euo pipefail + testing/a2a-harness/scripts/install-control.sh python + - name: Run the battery against the control + working-directory: testing/a2a-harness + run: | + set -euo pipefail + mkdir -p reports + SRC="${A2AHT_CONTROL_SRC:-$HOME/.a2aht/src}" + # Port 41241 is the sample's own, and the sample path is the one pinned by + # install-control.sh's clone of tag v1.1.2. + # THE GRPC ACKNOWLEDGEMENT, AND WHY IT IS NOT A SKIP. This control's card declares a GRPC + # interface, and this battery drives JSON-RPC and HTTP+JSON only. Left unstated that is a + # red -- `card.every_declared_binding_is_exercised` fails on purpose, because a suite that + # goes green having never touched the transport that ships is worse than no suite. The + # flag does not silence it; it records the gap as ACKNOWLEDGED, by name, in the report. + # It is also the exact gap the official TCK leg exists to cover: the TCK drives all three + # transports, which is why wiring it beat writing a gRPC driver of our own. + # + # Started OUT OF BAND rather than through `--launch`, so its own stderr reaches the log. + # A control that fails to boot must say WHY here; `--launch` swallows it and the job then + # reports only "not reachable", which is true and useless. + "$SRC/venv/bin/python" "$SRC/a2a-python/samples/hello_world_agent.py" \ + > /tmp/a2a-python.log 2>&1 & + CONTROL_PID=$! + for _ in $(seq 1 60); do + curl -fsS -m 2 -o /dev/null \ + http://127.0.0.1:41241/.well-known/agent-card.json && break + sleep 1 + done + if ! curl -fsS -m 5 -o /dev/null \ + http://127.0.0.1:41241/.well-known/agent-card.json; then + echo "::error::the a2a-python control never served its agent card. Its own output:" + cat /tmp/a2a-python.log + kill $CONTROL_PID 2>/dev/null || true + exit 1 + fi + rc=0 + python3 -m a2aht run \ + --endpoint http://127.0.0.1:41241 \ + --label "control:a2a-python" --tier pre-release \ + --known-deviations baselines/known-deviations-a2a-python.json \ + --role server \ + --allow-undriven-bindings GRPC \ + --json reports/control-python.json --allow-red || rc=$? + kill $CONTROL_PID 2>/dev/null || true + echo " (battery exit $rc; the verdict is the baseline comparison)" + tail -40 /tmp/a2a-python.log + - name: The second control must still produce its pinned verdict + working-directory: testing/a2a-harness + run: | + set -euo pipefail + python3 -m a2aht baseline \ + --report reports/control-python.json \ + --baseline baselines/control-a2a-python.json + - uses: actions/upload-artifact@v4 + if: always() + with: { name: a2a-battery-control-python, path: testing/a2a-harness/reports/ } + + negative-control: + name: negative control (a broken peer MUST be red) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v5 + with: { python-version: '3.12' } + # A NEGATIVE CONTROL PROVES DISCRIMINATION, NOT MERELY DISPLEASURE. + # + # This leg used to boot the broken peer, `sleep 2`, and assert the battery exited 1. Both + # halves of that were wrong, and each was wrong in a way that had already fired. + # + # 1. `sleep 2` IS NOT READINESS, IT IS A GUESS, and on 2026-08-12 the guess lost: run + # 31565573446 booted the peer, found nothing listening two seconds later, and exited 3 — + # "never reached the peer, proved nothing" — while the run 51 minutes after it, on the + # same code, exited 1 and went green. A control leg whose verdict depends on how busy the + # runner is teaches people to re-run it, which is how a real red gets clicked past. The + # peer's readiness is now OBSERVED, on its own card, and the peer's own log is printed + # whatever happens — the old step redirected it to a file it never read back, so the one + # run that failed left no evidence of WHY the peer was absent. + # + # 2. `rc == 1` DOES NOT SAY THE INJECTED DEFECTS WERE CAUGHT. It says something failed. The + # harness's HONEST fake peer also exits 1 against this tier — it fails 7 tests that have + # nothing to do with the negative control — so the old assertion would have held with + # every one of the five deliberate MUST violations undetected. That is this battery's + # version of the sibling MCP suite's `isResponse()` blind spot: an instrument reporting a + # number that is true and does not mean what the check reads it to mean. + # + # So the peers are run in PAIR, and the claim is made per test, by name: for each violation + # deliberately injected into the broken peer there is a test that must FAIL against the + # broken peer and PASS against the honest one. A battery that always passes and a battery + # that always fails are equally useless, and only the pair can tell them apart. + - name: A broken peer MUST be red, an honest one MUST NOT be red for the same reasons + working-directory: testing/a2a-harness + run: | + set -euo pipefail + mkdir -p reports + + # READINESS BY OBSERVATION. Bounded well above the ~1s a local boot takes, so a slow + # runner is not a red, and a peer that never binds is a red that says so IN ITS OWN + # WORDS rather than as "nothing is listening" from the other side of the wire. + await_peer() { + local port="$1" log="$2" what="$3" waited=0 + until curl -fsS -m 2 -o /dev/null \ + "http://127.0.0.1:$port/.well-known/agent-card.json"; do + waited=$((waited + 1)) + if [ "$waited" -ge 30 ]; then + echo "::error::the $what peer never served a card on 127.0.0.1:$port within ${waited}s. Its own output follows; this leg proved NOTHING about the battery." + cat "$log" + return 1 + fi + sleep 1 + done + echo " the $what peer answered on 127.0.0.1:$port after ${waited}s" + } + + python3 -m a2aht fake-peer --port 9402 --broken > /tmp/broken.log 2>&1 & + BROKEN=$! + python3 -m a2aht fake-peer --port 9403 > /tmp/honest.log 2>&1 & + HONEST=$! + trap 'kill $BROKEN $HONEST 2>/dev/null || true' EXIT + await_peer 9402 /tmp/broken.log broken + await_peer 9403 /tmp/honest.log honest + echo "--- the broken peer's own account of what it violates:" + cat /tmp/broken.log + + run_battery() { + local port="$1" label="$2" out="$3" rc=0 + python3 -m a2aht run --endpoint "http://127.0.0.1:$port" \ + --label "$label" --tier pull-request --role server \ + --json "reports/$label.json" > "$out" 2>&1 || rc=$? + echo "$rc" + } + broken_rc=$(run_battery 9402 negative-control /tmp/negative.txt) + honest_rc=$(run_battery 9403 honest-control /tmp/honest-run.txt) + + echo "=== the battery against the BROKEN peer (exit $broken_rc)" + cat /tmp/negative.txt + echo "=== the battery against the HONEST peer (exit $honest_rc)" + cat /tmp/honest-run.txt + + # THE EXIT CODES, WHICH ARE THREE DIFFERENT STATEMENTS AND NOT A PASS/FAIL. + # 1 means tests ran and failed. 0 means the battery BLESSED a peer it was built to + # reject, which would invalidate every conformance number this workflow has ever + # produced. 3 means it never reached the peer, so the leg is a false green in waiting. + # They are reported apart because the remedy for each is a different one. + case "$broken_rc" in + 1) echo " the broken peer was rejected (exit 1: tests ran, tests failed)" ;; + 0) echo "::error::CATASTROPHIC: the battery exited 0 against the deliberately broken peer. It cannot tell a broken peer from a working one, and every A2A conformance number in this workflow is in question." + exit 1 ;; + 3) echo "::error::the battery exited 3 against the broken peer: it never reached it, so NOTHING WAS TESTED. This leg proved nothing; it did not prove the battery works." + exit 1 ;; + *) echo "::error::the broken peer produced exit $broken_rc, expected 1. A battery that cannot fail cannot pass." + exit 1 ;; + esac + if [ "$honest_rc" = "3" ]; then + echo "::error::the battery never reached the HONEST peer either, so the discrimination check below would be comparing two absences." + exit 1 + fi + + # THE DISCRIMINATION ITSELF. Each id here is the test that exists to catch ONE violation + # `cli.py::cmd_fake_peer` deliberately injects, and the pairing is asserted in both + # directions: a test that fails against everything catches nothing, and a test that + # passes against everything catches nothing either. + python3 - <<'PY' + import json, sys + + # test id -> the injected violation it exists to catch + PAIRS = { + "card.required_fields": "PROTO AgentCard: the REQUIRED `version` is absent", + "card.protocol_version_no_patch": "SPEC 3.6: the card advertises protocolVersion 1.0.3", + "core.task_state_is_defined_enum": "PROTO enum TaskState: emits TASK_STATE_MADE_UP", + "adv.concurrent_interleaved_tasks": "SPEC 3.4.2: one task id reused for every task", + "core.stream_opens_with_task_or_message": + "SPEC 3.1.2: streams an event for a task never created", + } + + def outcomes(path): + with open(path) as fh: + return {r["id"]: r["outcome"] for r in json.load(fh)["results"]} + + broken = outcomes("reports/negative-control.json") + honest = outcomes("reports/honest-control.json") + + print("\nDISCRIMINATION: every injected violation, caught on the broken peer and NOT " + "reported against the honest one.\n") + bad = 0 + for test_id, violation in sorted(PAIRS.items()): + b = broken.get(test_id, "ABSENT") + h = honest.get(test_id, "ABSENT") + ok = (b == "FAIL" and h == "PASS") + print(" %-6s %-42s broken=%-6s honest=%-6s %s" + % ("ok:" if ok else "BAD:", test_id, b, h, violation)) + if not ok: + bad += 1 + if bad: + sys.stdout.write( + "\n::error::%d of %d injected violations were not DISCRIMINATED. A test that " + "is red against both peers is not detecting the violation, and one that is " + "green against the broken peer is blind to it. Either way the battery's " + "verdict about busbar means less than it appears to.\n" % (bad, len(PAIRS))) + sys.exit(1) + print("\nnegative control behaved: %d injected violations, each caught on the broken " + "peer and each absent from the honest one." % len(PAIRS)) + PY + + swap-proof: + name: swap proof (six states the gate must tell apart) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v5 + with: { python-version: '3.12' } + - uses: actions/setup-go@v5 + with: { go-version: '1.24' } + - run: testing/a2a-harness/scripts/install-control.sh go + - name: Six states, six different correct verdicts + working-directory: testing/a2a-harness + run: | + set -euo pipefail + A2AHT_CONTROL_BIN="$A2AHT_CONTROL_BIN" ./scripts/swap-proof.sh + - uses: actions/upload-artifact@v4 + if: always() + with: { name: a2a-swap-proof, path: testing/a2a-harness/reports/ } + + tz-is-load-bearing: + name: the timezone pin still changes the answer + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v5 + with: { python-version: '3.12' } + - uses: actions/setup-go@v5 + with: { go-version: '1.24' } + - run: testing/a2a-harness/scripts/install-control.sh go + - name: Under TZ=UTC the pinned baseline MUST break, on the timestamp finding + run: | + set -euo pipefail + testing/a2a-harness/scripts/tz-is-load-bearing.sh + + # ------------------------------------------------------------------- instrument 2: official TCK + tck-control: + name: official TCK vs control (${{ matrix.leg }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + leg: [control-http-json, control-jsonrpc] + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v5 + with: { python-version: '3.12' } + - uses: actions/setup-go@v5 + with: { go-version: '1.24' } + - name: Fetch the pinned TCK, run it, and hold it to its pinned verdict + run: | + set -euo pipefail + testing/a2a-tck/run-tck.sh ${{ matrix.leg }} + - uses: actions/upload-artifact@v4 + if: always() + with: + name: a2a-tck-${{ matrix.leg }} + path: /home/runner/.a2a-tck/out/ + + # --------------------------------------------------------------- the governance tier, separate + governance-probe: + name: governance probe (NOT a conformance result) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v5 + with: { python-version: '3.12' } + - uses: actions/setup-go@v5 + with: { go-version: '1.24' } + - run: testing/a2a-harness/scripts/install-control.sh go + - name: Observe what a peer can see of governance + working-directory: testing/a2a-governance + run: | + set -euo pipefail + mkdir -p reports + python3 -m a2agov \ + --launch "$A2AHT_CONTROL_BIN/a2a serve --echo --port 9098 --quiet" \ + --port 9098 --label "control:a2a-go" \ + --client-drive "$A2AHT_CONTROL_BIN/a2a send {url} governance-probe" \ + --json reports/governance.json + - name: The probe must have OBSERVED something + # The probe never gates on pass/fail -- it reports observations. That is exactly the shape + # that can go green having done nothing, so the floor is on the observation count, and the + # separation is re-asserted here rather than assumed: a report that claims to be a + # conformance result would be a category error worth failing on. + working-directory: testing/a2a-governance + run: | + set -euo pipefail + python3 - <<'PY' + import json, sys + r = json.load(open("reports/governance.json")) + n = len(r.get("results", [])) + if n < 3: + sys.exit("governance probe recorded only %d results; it did not run." % n) + meta = r.get("meta", {}) + if not meta.get("not_a_conformance_result"): + sys.exit("the governance report does not mark itself as NOT a conformance result. " + "That flag is the thing stopping a governance run being read as a " + "conformance pass.") + print("governance probe: %d observations, correctly labelled non-conformance." % n) + PY + - uses: actions/upload-artifact@v4 + if: always() + with: { name: a2a-governance-probe, path: testing/a2a-governance/reports/ } + + # ------------------------------------------------------------------------------ the subject + subject: + name: subject (busbar, built from this commit — ARMED OR RED) + runs-on: ubuntu-latest + timeout-minutes: 45 + # PUBLISHED, not inferred. A job whose steps all skip still reports `success`, so the + # aggregator cannot tell "armed and passed" from "unarmed and did nothing" by looking at the + # job result -- which is precisely the shape of false green this workflow is built to refuse. + # The arm state is therefore an explicit output the verdict reads, and the verdict now treats + # `false` as RED. + outputs: + armed: ${{ steps.arm.outputs.armed }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v5 + with: { python-version: '3.12' } + # Node is the credential shim and the token minter, both shared with the MCP subject leg. + - uses: actions/setup-node@v4 + with: { node-version: '22' } + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + # THE SUBJECT IS BUILT FROM THIS COMMIT, and that is the change that makes this leg mean + # something at all. `CARGO_INCREMENTAL=0`: incremental artifacts are worthless on a fresh + # runner and a partially populated incremental cache has produced spurious "unable to copy" + # build failures, which is a red that is not a defect. + - name: Build the subject from this commit + env: + CARGO_INCREMENTAL: '0' + run: cargo build --bin busbar + + # This step RECORDS the arm state and does not judge it; the two instrument steps below fail + # on their own when unarmed (`boot.sh::require_armed`), and the verdict fails independently on + # this output. Two mechanisms for one fact, deliberately: a step re-run in isolation is still + # honest, and the verdict still catches a job cancelled before this step ever ran. + # + # NO SECRET AND NO REPOSITORY VARIABLE ARMS THIS ANY MORE. The arm is a FILE this job just + # built, so the leg cannot silently disarm because somebody deleted a variable or let a + # deployment lapse -- which is precisely how it came to be disarmed on every run since it was + # written. It can only disarm by the build failing, which is itself red. + - name: Record the arm state + id: arm + run: | + set -euo pipefail + if [ -x target/debug/busbar ]; then + echo "armed=true" >> "$GITHUB_OUTPUT" + else + echo "armed=false" >> "$GITHUB_OUTPUT" + { + echo "### A2A subject leg: NOT ARMED — this is RED" + echo "" + echo "The build produced no busbar binary, so the two instruments could only have run" + echo "against their pinned third-party controls. That proves the INSTRUMENTS work. It" + echo "proves nothing about busbar." + } >> "$GITHUB_STEP_SUMMARY" + fi + + # NOT conditional on the arm state. Unarmed, these steps FAIL — that is the transition A4.4 + # is about. `boot.sh` boots busbar on loopback with a fronted agent configured, mints a REAL + # audience-bound credential with the signing key this job generated, and PROVES the plane + # boundary is still intact (no credential / no audience / wrong audience / flipped signature + # must all be 401, and the right token must be admitted) before either instrument starts -- + # because a leg that reached the endpoint by weakening the thing under test would be reporting + # about a busbar nobody runs, which is worse than leaving it unarmed. + - name: Independent battery against the subject + env: + A2A_SUBJECT_BUSBAR_BIN: target/debug/busbar + run: ./scripts/a2a-subject/boot.sh --battery + + # `!cancelled()` rather than the default, so BOTH instruments report on every run. They are + # independent oracles and they fail for different reasons; letting the first red hide the + # second would mean bisecting the gate one instrument per run. + - name: Official TCK against the subject + if: ${{ !cancelled() && steps.arm.outcome == 'success' }} + env: + A2A_SUBJECT_BUSBAR_BIN: target/debug/busbar + run: ./scripts/a2a-subject/boot.sh --tck + + # THE OPTIONAL EXTRA LEG. If an operator also wants a real deployment judged, setting the + # variable adds a run against it -- it is never a substitute for the booted subject above, and + # it is never soft: a run that happens must pass. Absent, nothing here runs and nothing above + # depends on it. Note it judges whatever is deployed there, which may not be this commit. + - name: Also judge an external deployment, if one is configured + if: ${{ !cancelled() && vars.BUSBAR_A2A_ENDPOINT != '' }} + env: + BUSBAR_A2A_ENDPOINT: ${{ vars.BUSBAR_A2A_ENDPOINT }} + A2A_SUBJECT_TCK_LOG: .a2a-conformance/tck-external.txt + run: ./scripts/a2a-subject/boot.sh --tck + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: a2a-subject + path: | + .a2a-conformance + testing/a2a-harness/reports + if-no-files-found: warn + + # ----------------------------------------------------------------------------- the aggregator + verdict: + name: A2A conformance verdict + runs-on: ubuntu-latest + if: always() + needs: + - harness-selftest + - control-a2a-go + - control-a2a-python + - negative-control + - swap-proof + - tz-is-load-bearing + - tck-control + - governance-probe + - subject + steps: + # A ROW THAT CANNOT RUN IS RED, NEVER SKIPPED. Every leg above is required to have reached + # `success`. `skipped` and `cancelled` are failures here, because the failure mode this whole + # workflow exists to prevent is a tick over a job that executed nothing -- which is exactly + # what the eight-failures-two-vacuous-passes run in the old location looked like. + # + # `subject` USED TO BE the one leg allowed not to have tested anything, "only while it is + # unarmed by design". That exemption is DELETED. It was written when busbar served no A2A at + # all, and it decayed into exactly the hole it was shaped like: the leg was unarmed on every + # run for its entire life, reported `success` every time, and produced no conformance number + # of any kind. `armed=false` is now RED. The arm state still comes from the job's own + # published OUTPUT rather than from its result, because an all-steps-skipped job reports + # `success` and the two are otherwise indistinguishable from here. + - name: Every control leg must have EXECUTED + env: + SELFTEST: ${{ needs.harness-selftest.result }} + GO: ${{ needs.control-a2a-go.result }} + PY: ${{ needs.control-a2a-python.result }} + NEG: ${{ needs.negative-control.result }} + SWAP: ${{ needs.swap-proof.result }} + TZ_LEG: ${{ needs.tz-is-load-bearing.result }} + TCK: ${{ needs.tck-control.result }} + GOV: ${{ needs.governance-probe.result }} + SUBJECT: ${{ needs.subject.result }} + SUBJECT_ARMED: ${{ needs.subject.outputs.armed }} + run: | + set -euo pipefail + fail=0 + strict () { + if [ "$2" != "success" ]; then + echo "::error::$1 did not succeed (result: $2). A control leg that did not EXECUTE is red, not skipped." + fail=1 + else + echo " ok $1" + fi + } + strict harness-selftest "$SELFTEST" + strict control-a2a-go "$GO" + strict control-a2a-python "$PY" + strict negative-control "$NEG" + strict swap-proof "$SWAP" + strict tz-is-load-bearing "$TZ_LEG" + strict tck-control "$TCK" + strict governance-probe "$GOV" + + if [ "${SUBJECT_ARMED:-}" = "true" ]; then + strict subject "$SUBJECT" + elif [ "${SUBJECT_ARMED:-}" = "false" ]; then + echo "::error::subject published armed=false. NOT ARMED, SO NOT RUN is a RED state: the controls proved the instruments, not busbar. The arm is a busbar binary built from this commit, so an unarmed subject leg means the build produced nothing." + fail=1 + else + echo "::error::subject published no arm state (result: $SUBJECT, armed: '${SUBJECT_ARMED:-}'). The leg did not reach its own arming check, so whether busbar was tested is unknown -- and unknown is red." + fail=1 + fi + + [ "$fail" -eq 0 ] || exit 1 + echo + echo "A2A conformance verdict: every instrument executed and produced its pinned result." diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml new file mode 100644 index 00000000..7d0d57f1 --- /dev/null +++ b/.github/workflows/build-artifact.yml @@ -0,0 +1,138 @@ +# ONE BUILD PIPELINE, PARAMETERISED BY TARGET. It builds ONE artifact and it does the same thing +# for every target. +# +# THE DEFECT THAT MADE THIS NECESSARY. busbar 1.5.3 shipped four unix binaries; three embedded the +# plugin release public key and busbar-aarch64-unknown-linux-gnu did not. Not a 1.5.3 regression -- +# 1.5.1 and 1.5.2 shipped the same broken leg. On ARM Linux (Graviton, Raspberry Pi, ARM containers) +# every correctly-signed first-party plugin was refused, and the only workaround was +# `plugins.trust.allow_unsigned: true`, which switches the requirement off instead of trusting +# first-party selectively. +# +# The cause was structural, not a missing secret: the org variable was set the whole time. The +# release matrix had TWO build steps -- scripts/pgo-build.sh for host-native targets and +# upload-rust-binary-action for the cross targets -- and any property established on one of them was +# unproven on the other, silently and permanently. +# +# Owner, 2026-08-08: "should just be 1 build pipeline that takes what its building: arm, windows, +# mac, but it does the same thing for each no way for 1 to be different", and "100% or 0% for each +# build". +# +# SO: ONE STEP BUILDS, FOR EVERY TARGET. There is no `if:` on any step in this file, no per-target +# branch, and no second build action. Everything a target needs arrives as an INPUT, derived from +# .github/release-targets.json by release.yml's `targets` job -- the single source of truth that +# already produced the asset-name list. The env block that carries BUSBAR_RELEASE_PUBKEY is written +# once, on the one build step, so "one target missed it" is not a state this workflow can be in. +# +# EVERY TARGET IS NATIVE, WHICH IS WHAT MADE ONE PATH POSSIBLE. GitHub now offers a runner for every +# architecture busbar ships: `ubuntu-24.04-arm` for ARM Linux and `macos-15-intel` for Intel macOS, +# both confirmed available to this org and green on a probe run before this was written. So the +# cross-compilation that forced the second path is gone rather than worked around, four of five +# targets train their own PGO profile on their own architecture, and -- the part that matters for +# verification -- every artifact can be EXECUTED on a runner of its own platform, so no contract row +# has to be statically approximated anywhere, Windows included. +# +# WHAT IS DELIBERATELY NOT HERE: any assertion that the build worked. This workflow produces an +# artifact and the evidence about how it was produced; scripts/verify-artifact.py asserts the +# contract against the bytes downloaded back from the release. Asserting an output property here +# would be asserting it on the machine that had every opportunity to get it wrong, which is the +# 1.5.3 shape: the env var was set, the workflow was green, and the artifact had no key. +name: build one release artifact + +on: + workflow_call: + inputs: + target: + description: The rust target triple to build. THE parameter this pipeline takes. + required: true + type: string + runner: + description: >- + The GitHub-hosted runner label for this target. MUST be native for `target`: + .github/release-targets.json declares it and scripts/tests/test_release_contract.py fails the + build if a target claims a runner of a different architecture. + required: true + type: string + archive: + description: The archive extension this target's asset uses (tar.gz / zip). + required: true + type: string + tag: + description: The draft release tag to upload the asset to. There is no git tag yet. + required: true + type: string + +permissions: + contents: write + id-token: write + attestations: write + +jobs: + build: + name: ${{ inputs.target }} + runs-on: ${{ inputs.runner }} + steps: + - uses: actions/checkout@v7 + + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ inputs.target }} + + # THE BUILD. One step, one script, one env block, five targets. + # + # BUSBAR_RELEASE_PUBKEY is a repository/org VARIABLE, not a secret -- it is the PUBLIC half of + # the release keypair, embedded into the binary at compile time by plugin-sign's + # `option_env!`, so a busbar-signed plugin verifies as first-party with zero configuration. + # The matching PRIVATE half is the BUSBAR_SIGN_KEY secret each first-party plugin repo signs + # its own release with. + # + # scripts/release-build.sh REFUSES to compile when it is absent or malformed, for every + # target, in one place. That refusal is the build-time half of the fix; it cannot see a + # compiler that read the variable and dropped it, which is why the contract also asserts the + # key on the shipped BYTES afterwards. + - name: Build the artifact + shell: bash + env: + BUSBAR_RELEASE_PUBKEY: ${{ vars.BUSBAR_RELEASE_PUBKEY }} + run: scripts/release-build.sh "${{ inputs.target }}" --evidence-dir build-evidence + + # Upload to the DRAFT by tag. Under this release order there is no git tag yet -- the tag is an + # output of a verified run -- so the asset is placed on the draft explicitly rather than by a + # tool that resolves the release from the git ref. + - name: Upload the artifact to the draft + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + archive="busbar-${{ inputs.target }}.${{ inputs.archive }}" + [ -s "$archive" ] || { echo "::error::no archive at $archive; ${{ inputs.target }} would be MISSING from the release." >&2; exit 1; } + gh release upload "${{ inputs.tag }}" "$archive" --repo "$GITHUB_REPOSITORY" --clobber + + # Keyless (Sigstore/OIDC) build-provenance binding this archive's DIGEST to this run and + # commit, so a swapped artifact on the Release page fails `gh attestation verify`. It binds a + # digest, so promoting the draft later changes nothing. The contract's `attestation` row + # re-runs that exact user-facing command against the bytes downloaded back from the release. + - name: Attest build provenance + uses: actions/attest-build-provenance@v4 + with: + subject-path: busbar-${{ inputs.target }}.${{ inputs.archive }} + + # THE RECEIPT AND THE EVIDENCE, WHICH ARE TWO DIFFERENT JOBS OF THE SAME UPLOAD. + # + # EVIDENCE: `artifact.sha256` (and, where the target declares PGO, pgo-build.sh's proof + # marker) travels to the verify matrix, which binds it to the digest of the archive it + # downloads from the release. Without that binding a build could certify a PGO profile for + # bytes that never shipped. + # + # RECEIPT: the artifact's existence is how release.yml's `verify-set-equality` job learns that + # THIS target was actually produced. It reads the produced set and the verified set from + # receipts rather than from job conclusions, because a job that was skipped -- which is what + # `needs:` does to a dependent when an upstream leg fails -- leaves a grey square and no + # evidence, and a set derived from grey squares silently shrinks to fit. + - name: Upload build evidence and the produced-receipt + uses: actions/upload-artifact@v4 + with: + name: build-evidence-${{ inputs.target }} + path: build-evidence + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/ci-images-mirror.yml b/.github/workflows/ci-images-mirror.yml new file mode 100644 index 00000000..50190618 --- /dev/null +++ b/.github/workflows/ci-images-mirror.yml @@ -0,0 +1,307 @@ +name: CI images mirror + +# Mirror the pinned CI service-container images into GHCR, so CI does not depend on Docker Hub's +# shared anonymous quota. +# +# THE PROBLEM THIS HALF-CLOSES, AND WHY THE OTHER HALF COULD NOT BE DONE THE OBVIOUS WAY. +# +# `ci.yml` and `release.yml` boot Postgres and Valkey as service containers. Pinning them by digest +# (done) buys REPRODUCIBILITY: the same commit gets the same bytes on any day. It does nothing about +# RATE LIMITS, because the pull still goes to Docker Hub on the anonymous per-IP quota that every +# GitHub runner shares. A throttled pull there is a red nobody can clear by fixing busbar, and under +# release.yml's `branch-green` gate it stops a release. +# +# Authentication is what buys quota, and `release.yml`'s gate does authenticate: it only ever +# triggers on a push to `main` and on workflow_dispatch, so its secrets are always present. +# `ci.yml` CANNOT do the same. It runs on `pull_request`, GitHub withholds secrets from fork runs, +# and a service container's `credentials:` block cannot be made conditional. An unconditional block +# would hand every fork PR an empty username and password. GetBusbar/busbar is PUBLIC with forks, so +# that is a real breakage, not a theoretical one -- and it would break precisely the fork PRs the +# change was meant to protect. +# +# A PUBLIC GHCR PACKAGE NEEDS NO CREDENTIALS AT ALL, and that is the property this whole workflow +# rests on. Verified directly against an existing public package in this org before any of this was +# written: an anonymous token request to ghcr.io followed by a HEAD of the manifest returns 200 with +# no secrets anywhere. So a fork PR can pull a mirrored image with nothing configured, and GHCR's +# limits are not the Docker Hub anonymous pool. +# +# -- THE SEQUENCING, WHICH IS THE PART THAT CAN BREAK EVERYTHING --------------------------------- +# +# A NEWLY CREATED GHCR PACKAGE IS PRIVATE BY DEFAULT. So repointing `ci.yml` at these mirrors before +# they exist and are public breaks EVERY CI run, including the fork PRs the mirror exists to protect. +# That is a partial promote -- moving the consumer before the producer is proven -- and it gets the +# same treatment as the release promote: THE CONSUMER MOVES LAST, AND ONLY ON PROOF. +# +# 1. Land and run THIS workflow. It creates the packages and proves the bytes arrived. +# 2. ONE-TIME OWNER ACTION: flip both packages to public. There is no REST endpoint for container +# package visibility -- it is a UI action under the package's own settings -- so this cannot be +# scripted with any token. +# 3. Only then repoint `ci.yml`'s service containers at the mirrors. +# +# AND THE ANONYMOUS CHECK ARMS ITSELF AT STEP 3, RATHER THAN STANDING RED THROUGH STEP 2. +# +# An earlier draft of this file failed the anonymous-pull step from its very first run, on the +# grounds that a private package is an incomplete state and incomplete states should be loud. The +# reasoning was wrong, and the rule it broke is absolute here: "nothing should ever be released red", +# "or ignored". A KNOWN, EXPECTED red is the worst kind, because it trains everyone to read a red X +# on this workflow as normal -- and then the next red, a mirror that genuinely did not land or a +# package flipped back to private, looks exactly like the one everybody was told to expect. A good +# reason for a standing red does not make it a different thing. +# +# The distinguishing fact is whether anything CONSUMES the mirrors, and it is derived from the tree +# by `scripts/ci-images.py --consumer-state` rather than from a flag somebody remembers to set: +# +# * While `ci.yml` still pulls from Docker Hub, the packages being private breaks NOTHING, because +# nothing consumes them. The check reports the outstanding flip as a NOTICE and the job is +# GREEN. That is honest: at that moment the repository is in a correct state with a known next +# action, and green is what a correct state should look like. +# * The moment `ci.yml` is repointed, a private package genuinely does break CI -- including the +# fork PRs the mirror exists to protect -- and the check becomes a hard failure with no +# tolerance. +# +# So the guard arms itself AS A CONSEQUENCE of the consumer moving. There is no window in which a +# red on this workflow means "expected". +# +# THE NOTICE CANNOT BECOME PERMANENT. An indefinite notice is just a red with better manners. If +# `ci.yml` is still on Docker Hub GRACE_DAYS after this workflow first landed, the notice turns into +# a failure: at that point the mirror is unfinished work that has stopped being tracked, which is a +# real defect even though no image is broken. +# +# Until step 3, this workflow changes nothing about how CI runs. It is additive and safe to land. +on: + # RE-MIRROR WHENEVER A PIN CHANGES. This is the trigger that matters in normal operation: a re-pin + # (monthly-refresh.yml's sweep, or a hand bump) must be followed by a mirror of the NEW digest, or + # ci.yml would point at a mirrored image that no longer matches what the workflows pin. Keying it + # on the exact files that carry or derive the pins means the two can never drift apart by anyone + # forgetting a step. + # + # IT IS ALSO HOW THIS WORKFLOW FIRST RUNS, WITHOUT ANYONE PUSHING `main`. `workflow_dispatch` is + # only offered for workflows present on the DEFAULT branch, and `main` is the release branch: under + # this repository's model landing there cuts a release, and it currently sits 11 commits behind + # `dev`'s unreleased work. Waiting for the next release to make a CI-infrastructure workflow + # runnable would be backwards, and pushing `main` to shortcut it would be worse. On merge to `dev` + # this fires by itself and creates the packages, which is step 1 done with no release-branch + # involvement at all. + push: + branches: [main, dev] + paths: + - ".github/workflows/ci-images-mirror.yml" + - ".github/workflows/ci.yml" + - ".github/workflows/release.yml" + - "scripts/ci-images.py" + workflow_dispatch: + schedule: + # Monthly, a day after monthly-refresh.yml's dependency sweep (09:00 UTC on the 1st), so a + # re-pin from that PR is already on the branch when this runs. Off the top of the hour: :00 cron + # slots are the most contended on GitHub's shared scheduler. + - cron: "37 10 2 * *" + +permissions: + contents: read + packages: write + +jobs: + mirror: + name: mirror the pinned CI service images into GHCR + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + # How long the "ci.yml has not been repointed yet" NOTICE may stand before it becomes a + # failure. Long enough that the one-time visibility flip is not a fire drill, short enough + # that unfinished work cannot quietly become permanent. + GRACE_DAYS: "30" + steps: + - uses: actions/checkout@v7 + with: + # Full history so the grace window can be measured from the commit that ADDED this + # workflow. Deriving it from git rather than from a hardcoded date means it cannot drift. + fetch-depth: 0 + + # DERIVED, NEVER RESTATED. scripts/ci-images.py parses the pins out of ci.yml and release.yml + # and asserts the two agree, so this workflow holds no copy of a digest. A third copy of the + # same fact is the defect that published v1.5.3 with five assets where seven were expected. + # Its --selftest runs FIRST: never trust the derivation before proving the deriver still + # catches an unpinned image, a disagreement, and an empty list. + - name: ci-images self-test (prove the deriver still catches a real violation) + run: python3 scripts/ci-images.py --selftest + + - name: Derive the pinned images + id: images + run: | + set -euo pipefail + json="$(python3 scripts/ci-images.py --list)" + echo "$json" | python3 -m json.tool + echo "json=$json" >> "$GITHUB_OUTPUT" + + - uses: docker/setup-buildx-action@v4 + + - name: Log in to GHCR + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Manifest-only copy, exactly the primitive docker.yml's promote job uses: no rebuild, the + # multi-arch index is preserved, and the mirrored image is the same bytes rather than a second + # build that ought to match. + - name: Copy each pinned image into GHCR + env: + IMAGES: ${{ steps.images.outputs.json }} + run: | + set -euo pipefail + echo "$IMAGES" | python3 -c 'import json,sys;[print(i["source"], i["mirror"]) for i in json.load(sys.stdin)]' \ + | while read -r src dst; do + echo "== $src -> $dst" + docker buildx imagetools create -t "$dst" "$src" + done + + # THE COPY MUST BE PROVEN TO HAVE LANDED, NOT ASSUMED FROM AN EXIT CODE. + # + # `imagetools create` exiting 0 says the API call was accepted. It does not say the registry + # now serves those bytes under that name. A mirror job that reports success while having + # copied nothing is the exact failure class that cost this project three days on the marketing + # side: the counts Worker logged every failure and returned normally, so a refresh that updated + # nothing reported success. So both mirrors are re-derived from the registry and compared to + # the PINNED SOURCE DIGEST. + # + # INDEX DIGEST FIRST, CHILD DIGESTS AS THE FALLBACK. For a single-source copy buildx normally + # preserves the index verbatim, so the digests match exactly. If a registry ever re-wraps the + # index, the top-level digest changes while the actual per-platform images do not -- so rather + # than false-fail on a cosmetic difference, that case falls through to comparing the SET of + # child manifest digests, which is the real "did the bytes arrive" question. A mismatch there + # is a genuine failure and is reported as one. + - name: Assert every mirror really carries the pinned bytes + env: + IMAGES: ${{ steps.images.outputs.json }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + python3 - <<'PY' + import json, os, subprocess, sys + + def children(ref): + """The set of per-platform manifest digests behind a ref, via buildx's raw index.""" + raw = subprocess.run(["docker", "buildx", "imagetools", "inspect", "--raw", ref], + capture_output=True, text=True, check=True).stdout + doc = json.loads(raw) + return {m["digest"] for m in doc.get("manifests", [])} + + def index_digest(ref): + out = subprocess.run(["docker", "buildx", "imagetools", "inspect", ref], + capture_output=True, text=True, check=True).stdout + for line in out.splitlines(): + if line.lower().startswith("digest:"): + return line.split(":", 1)[1].strip() + return "" + + fail = 0 + for i in json.loads(os.environ["IMAGES"]): + src, dst, pinned = i["source"], i["mirror"], i["digest"] + got = index_digest(dst) + if got == pinned: + print("PASS: %s carries the pinned digest %s" % (dst, pinned)) + continue + # Index re-wrapped, or nothing landed. Compare what is actually behind each ref. + try: + src_children, dst_children = children(src), children(dst) + except subprocess.CalledProcessError as e: + print("::error::the mirror %s could not be inspected at all, so the copy did NOT " + "land. Nothing downstream may use it. stderr: %s" % (dst, e.stderr.strip())) + fail = 1 + continue + if src_children and src_children == dst_children: + print("PASS: %s index digest differs (%s vs pinned %s) but all %d per-platform " + "manifests match exactly -- the index was re-wrapped, the bytes are the same." + % (dst, got or "", pinned, len(src_children))) + else: + print("::error::MIRROR DID NOT LAND: %s should carry the bytes of %s but does not. " + "index=%s vs pinned=%s; %d source manifests vs %d mirrored. A mirror that " + "reports success while having copied nothing is the failure this assertion " + "exists to prevent. Do NOT repoint ci.yml at this image." + % (dst, src, got or "", pinned, len(src_children), len(dst_children))) + fail = 1 + sys.exit(fail) + PY + + # THE ASSERTION THE WHOLE DESIGN RESTS ON, AND IT BELONGS IN CI RATHER THAN IN SOMEBODY'S SHELL. + # + # The mirror is only useful if a fork PR -- which has NO secrets -- can pull it. That is a + # property of the package's VISIBILITY, a UI setting a human can change at any time, in either + # direction, with nothing in git recording it. Checking it once by hand proves it was true + # once. Without this step the first symptom of a private package is a broken CI run on a fork + # PR from an outside contributor: the worst possible place to discover it, on somebody's first + # interaction with the project. + # + # NOTICE OR FAILURE, DECIDED BY WHETHER ANYTHING CONSUMES THE MIRRORS. See the long note in + # this file's header for why a known-expected red is refused outright. In short: while + # `ci.yml` still pulls from Docker Hub a private package breaks nothing, so this reports the + # outstanding flip and stays GREEN; the moment `ci.yml` is repointed it becomes a hard + # failure. The condition is derived from the tree, so the guard arms itself as a consequence + # of the consumer moving rather than because someone remembered to flip a switch. + # + # NO CREDENTIALS ANYWHERE IN THIS STEP, DELIBERATELY. `docker/login-action` above wrote + # credentials into the runner's docker config, so `docker pull` here would succeed as the + # authenticated actor and prove nothing about what an anonymous client sees. This talks to the + # Distribution API directly with a token fetched anonymously, which is exactly what a fork + # PR's runner does. + - name: Anonymous pullability (notice until ci.yml consumes the mirrors, hard failure after) + env: + IMAGES: ${{ steps.images.outputs.json }} + run: | + set -uo pipefail + + state="$(python3 scripts/ci-images.py --consumer-state)" + echo "ci.yml service containers currently pull from: ${state}" + + # The grace clock runs from the commit that ADDED this workflow, so an unfinished mirror + # cannot sit behind a permanent notice. Derived from git rather than hardcoded. + added="$(git log --diff-filter=A --format=%aI -- .github/workflows/ci-images-mirror.yml | tail -1)" + if [ -n "${added:-}" ]; then + age_days=$(( ( $(date -u +%s) - $(date -u -d "$added" +%s 2>/dev/null || echo "$(date -u +%s)") ) / 86400 )) + else + age_days=0 + fi + echo "this workflow first landed ${added:-} (${age_days} day(s) ago); grace is ${GRACE_DAYS} day(s)" + + rc=0 + while read -r repo tag ref; do + tok="$(curl -fsS --max-time 30 "https://ghcr.io/token?service=ghcr.io&scope=repository:${repo}:pull" \ + | jq -r '.token // empty' 2>/dev/null || true)" + code=000 + if [ -n "${tok:-}" ]; then + code="$(curl -sS --max-time 30 -o /dev/null -w '%{http_code}' -I \ + -H "Authorization: Bearer ${tok}" \ + -H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json' \ + "https://ghcr.io/v2/${repo}/manifests/${tag}" || echo 000)" + fi + if [ "$code" = "200" ]; then + echo "PASS: ${ref} is pullable with no credentials (anonymous HEAD -> 200)" + continue + fi + why="anonymous HEAD returned HTTP ${code}" + [ -n "${tok:-}" ] || why="ghcr.io would not issue an anonymous pull token, so the package is PRIVATE" + fix="ONE-TIME FIX (UI only; there is no REST endpoint for container package visibility): https://github.com/orgs/GetBusbar/packages/container/${repo##*/}/settings -> Danger Zone -> Change visibility -> Public." + if [ "$state" = "mirrored" ]; then + echo "::error::ANONYMOUS PULL FAILED for ${ref}: ${why}. ci.yml PULLS THIS IMAGE, so every fork PR is currently unable to start its service container. ${fix}" + rc=1 + elif [ "$age_days" -gt "${GRACE_DAYS}" ]; then + echo "::error::${ref} has been un-flipped for ${age_days} days, past the ${GRACE_DAYS}-day grace window: ${why}. Nothing is broken yet because ci.yml still pulls from Docker Hub, but the mirror is unfinished work that has stopped being tracked, and an indefinite notice is just a red with better manners. Either complete it or delete the mirror. ${fix}" + rc=1 + else + echo "NOTICE: ${ref} is not yet anonymously pullable (${why})." + echo " Nothing is broken: ci.yml still pulls from Docker Hub, so no job consumes this image." + echo " ${fix}" + echo " STEP 3 (repointing ci.yml) IS BLOCKED until this reports PASS." + echo "::notice::${ref} awaits the one-time visibility flip. Not a failure: nothing consumes it yet. ${fix}" + fi + done < <(echo "$IMAGES" | python3 -c 'import json,sys;[print(i["mirror_repo"], i["tag"], i["mirror"]) for i in json.load(sys.stdin)]') + + if [ "$rc" != 0 ]; then + exit 1 + fi + if [ "$state" = "mirrored" ]; then + echo "All mirrors are anonymously pullable and ci.yml consumes them. The mirror is complete." + else + echo "Mirror created. ci.yml still pulls from Docker Hub, which is the correct state until the visibility flip." + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80b17f85..2bd409d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,51 @@ name: CI +# TWO TIERS, AND WHY. +# +# This workflow used to trigger on `push` to main/dev/qa ONLY, plus `pull_request`. That left every +# feature branch COMPLETELY UNGATED until someone opened a PR. During 1.5.4 development two work +# items were reported green from local runs and had never been CI-verified at all; the gap was only +# discovered when the PR was opened, days of work later. "It passed locally" is not a CI result, and +# a branch nobody has opened a PR for is exactly where that mistake is cheapest to make and most +# expensive to discover. +# +# So: `branches: ['**']`, and EVERY branch push now runs CI. But the full set is 12 jobs including a +# Windows runner, a release-mode timing gate, a loom model and a full-history migration corpus, and +# paying all of that on every intermediate commit of a long-lived feature branch is real money for +# very little marginal signal. So pushes to NON-PROMOTION branches run the FAST TIER: +# +# FAST TIER (every branch push) structure lint, fmt/clippy/build/test, config-stability, +# public-hygiene. This answers the only questions that matter +# mid-feature: does it build, does it lint, do the tests pass, is +# the layout/config-grammar/public-prose still clean. +# FULL TIER (dev, qa, main, and everything above PLUS openapi-schema, migration-corpus, +# EVERY pull request, and executable-config, no-default-features, no-plugins-gate, +# workflow_dispatch) txn-guards, timing, windows. +# +# THE TRADEOFF, STATED PLAINLY: a Windows-only or featureless-build-only regression is now caught at +# PR time rather than at push time. That is exactly where it was caught before this change, so the +# fast tier costs nothing anyone had; it is pure addition. The cost paid is that a branch with an +# open PR runs the fast tier (push) alongside the full tier (pull_request) for the same commit. The +# two events land in DIFFERENT concurrency groups (`github.ref` is `refs/heads/` for push and +# `refs/pull/N/merge` for pull_request) and they are deliberately NOT unified: a shared group would +# let the cheap push run cancel the full PR run that the required checks depend on. Four duplicated +# jobs is the accepted price of never again shipping an unverified branch. +# +# ESCAPE HATCH: `workflow_dispatch` forces the FULL tier on any ref, so a feature branch can buy the +# whole gate before opening a PR without waiting for one: +# gh workflow run ci.yml -R GetBusbar/busbar --ref feat/my-branch +# +# `branches:` never matches tag pushes, so the release tag (v1.5.3 et al) still does not run CI here. on: push: - branches: [main, dev, qa] + branches: ['**'] pull_request: + workflow_dispatch: # Without this, every push queues a fully independent run that competes for the same runners # instead of cancelling the one it superseded -- qa-gate.yml already has this; ci.yml never did, # so a burst of rapid pushes (e.g. iterating on a CI fix) pays for N full runs instead of 1. +# Doubly load-bearing now that every branch push triggers a run. concurrency: group: ci-${{ github.ref }} cancel-in-progress: true @@ -17,6 +55,54 @@ env: RUSTFLAGS: "-D warnings" jobs: + # WHICH TIER RAN, SAID OUT LOUD. A green check mark is read as "the gate passed", and after this + # change a green check on a feature branch means something WEAKER than a green check on a PR. + # An operator who cannot see the difference will assume the strong one, which is the precise + # mistake this workflow's two tiers otherwise re-introduce in a new place. So every run states its + # own tier in the job summary and names the jobs it did NOT run. Costs one echo; buys the run's + # verdict being self-describing instead of needing someone to remember the rule. + gate-tier: + name: gate tier (what this run actually proved) + runs-on: ubuntu-latest + steps: + - name: Declare the tier + # Ref and event come in as ENV, not as `${{ }}` spliced into the script text. A branch name + # is attacker-influenced data (anyone who can open a PR picks it) and GitHub allows plenty of + # shell metacharacters in one; splicing it into a `[ ... ]` test is a script-injection seam + # that happens to be quiet until someone names a branch to exploit it. + env: + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + REF_NAME: ${{ github.ref_name }} + run: | + set -euo pipefail + if [ "$EVENT" != "push" ] || \ + [ "$REF" = "refs/heads/main" ] || \ + [ "$REF" = "refs/heads/dev" ] || \ + [ "$REF" = "refs/heads/qa" ]; then + { + echo "### CI tier: FULL" + echo "" + echo "Every CI job ran. Event \`$EVENT\`, ref \`$REF\`." + } >> "$GITHUB_STEP_SUMMARY" + else + { + echo "### CI tier: FAST" + echo "" + echo "Push to a non-promotion branch (\`$REF\`), so this run proved:" + echo "structure lint, fmt/clippy/build/test, config-stability, public-hygiene." + echo "" + echo "It did NOT run: openapi-schema, migration-corpus, executable-config," + echo "no-default-features, no-plugins-gate, txn-guards, timing, windows." + echo "" + echo "Those run on every pull request, on dev/qa/main, and on demand via" + echo "\`gh workflow run ci.yml -R GetBusbar/busbar --ref $REF_NAME\`." + echo "" + echo "A green check here is NOT the full gate. Do not promote on it." + } >> "$GITHUB_STEP_SUMMARY" + fi + cat "$GITHUB_STEP_SUMMARY" + # LAYOUT GATE — its OWN job, deliberately. It used to be the first step of `check`, ahead of # clippy/build/test, which made it a MASK: any layout violation aborted the job before a single # test ran, and the `Test` step is the only place BUSBAR_TEST_POSTGRES_URL / VALKEY_URL are set @@ -47,6 +133,70 @@ jobs: run: scripts/release-script-lint.sh --selftest - name: Release-script lint run: scripts/release-script-lint.sh + # RELEASE-ORDER lint: nothing may be tagged until it has been verified from the consumer side, + # and nothing may be cut from a red commit. Both of those live in the SHAPE of the workflow + # graph (which job depends on which), which is edited by people in a hurry during an incident + # and has no other place to be asserted. Self-test runs FIRST, and it earns its place: it + # caught this lint's own `--draft` rule passing against a release that had had `--draft` + # deleted, because `--draft=false` elsewhere in the file contained the substring. + - name: Release-order lint self-test + run: python3 scripts/release-order-lint.py --selftest + - name: Release-order lint + run: python3 scripts/release-order-lint.py --root . + # AND WATCH IT FAIL. `--prove` fails each job of the release graph in turn and asserts that no + # name-minting job runs afterwards. It is the executable form of "a failure leaves nothing + # public", so that property is re-proven on every push rather than argued once in a comment. + - name: Release-order proof (a failure leaves nothing public) + run: python3 scripts/release-order-lint.py --prove + # WORKSPACE-DEPS lint: one version requirement per external dependency, written once in + # [workspace.dependencies]. The table makes that POSSIBLE; only this lint makes it TRUE — + # nothing in Cargo stops a member re-stating a version, and before the table `hex`, `sha2` and + # `tracing` were each declared two different ways and agreed purely by luck. It sits in the + # FAST tier deliberately: a manifest can drift on any branch push, and this costs one python + # process. Self-test runs FIRST and covers dev-dependencies and + # `[target.'cfg(...)'.dependencies]` explicitly, because a rule enforced only on + # `[dependencies]` is a rule scoped to where the bug was first seen — and the jemalloc pins, + # the ones that differ per shipped target, live in a target table. + - name: Workspace-deps lint self-test + run: python3 scripts/workspace-deps-lint.py --selftest + - name: Workspace-deps lint + run: python3 scripts/workspace-deps-lint.py --root . + # QA-GATE DISPATCHER DRIFT. `workflow_run` ALWAYS loads the workflow file from the DEFAULT + # branch, so the gate that actually fires after a push to `qa` is `main`'s copy, never the + # promoted commit's. That has already cost a silent green: measured on qa c736177 the + # auto-fired gate ran ONE job while the segmentation umbrella sat unused, and the run passed + # having done far less than anyone believed. Gate LOGIC now rides the commit (the dispatcher + # checks out the triggering SHA and runs scripts/qa-gate-run.sh from there), but everything + # GitHub reads BEFORE a checkout exists — `on:`, concurrency, permissions, the needs/if graph, + # the matrix expression — still comes from `main` and cannot. + # + # This compares the PARSED structure, not the bytes, so comments drift freely while the run + # graph may not. That split is what stops the check deadlocking: `main` only moves at a + # release, so a byte-identical rule would make every prose edit red until the very release it + # is meant to gate. Self-test first, as everywhere else here. + - name: qa-gate dispatcher drift self-test + run: python3 scripts/qa-gate-dispatch-lint.py --selftest + # `fetch-depth: 0` is not set on this job's checkout, so origin/main may be absent; fetch it + # explicitly rather than letting the lint fail closed for a reason that is not drift. + - name: Fetch the default branch (the dispatcher that will actually fire) + run: git fetch --no-tags --depth=1 origin main + - name: qa-gate dispatcher drift + run: python3 scripts/qa-gate-dispatch-lint.py + # ARTIFACT CONTRACT wholeness. The contract is data (.github/artifact-contract.json) and the + # verifier implements it, so the two can drift: a row declared with no implementation is a + # property everybody believes is checked and nobody checks, and a check outside the contract is + # invisible to anyone reading it. The guard asserts SET EQUALITY both ways plus a row floor, + # and this self-test proves that guard still discriminates rather than merely returning the + # rows unchanged. Its eight cases each construct one malformation and require it be refused. + - name: Artifact-contract wholeness self-test + run: python3 scripts/verify-artifact.py --selftest + # RELEASE-CHECK VERDICT ACCOUNTING. release-check.sh used to end with an unconditional + # "RELEASE GATE PASSED" banner even when phases had not run at all, so a green gate did not + # prove those phases executed. This selftest proves the two are now distinguishable: a + # coverage gap changes the banner, names the phase, and is fatal under --require-siblings, + # while a by-design segment skip stays a clean pass. Offline, no gate run, seconds. + - name: Release-check verdict self-test + run: scripts/release-check.sh --selftest # RESPONSE-HEADER lint: every busbar-INJECTED response header # (`Server-Timing: busbar;dur=`, `x-busbar-route-policy`/`-target`) must be emitted from its ONE # sanctioned, config-gated site — never a hand-rolled second emission that bypasses the @@ -114,8 +264,23 @@ jobs: # / VALKEY_URL (set on the Test step below) and, because `CI` is set in Actions, HARD-FAIL rather # than silently skip if a service is misconfigured - so this coverage cannot vanish unnoticed. services: + # PINNED BY DIGEST for the same reason as release.yml's gate: `postgres:16` and + # `valkey/valkey:8` are moving tags, and these containers are load-bearing (without them the + # store roundtrip tests hard-fail rather than skip). What they resolve to must be a fact about + # the commit, not about the day. Re-pin with `docker buildx imagetools inspect `; the + # monthly-refresh PR is the natural place. + # + # NO `credentials:` HERE, DELIBERATELY, AND IT LEAVES HALF THE HOLE OPEN. Authenticating would + # raise Docker Hub's anonymous per-IP rate limit, which is the half a digest pin cannot fix -- + # but this workflow runs on `pull_request`, where secrets are withheld from fork runs, so an + # unconditional `credentials:` block would hand every fork PR empty credentials. release.yml's + # gate (push to main only) IS authenticated. Closing this half properly means either + # restricting the credentials to non-fork events, which GitHub does not allow on a service + # container, or mirroring both images into GHCR and pulling from there. Written down rather + # than quietly left, because `branch-green` requires this workflow green: a throttled pull + # here stops a release. postgres: - image: postgres:16 + image: postgres:16@sha256:95206741a5b214807675e14165369d05b93a9cf692223b616d07cca227e74b0b env: POSTGRES_USER: busbar POSTGRES_PASSWORD: busbar @@ -129,7 +294,7 @@ jobs: --health-timeout 5s --health-retries 5 valkey: - image: valkey/valkey:8 + image: valkey/valkey:8@sha256:495e4fecdc98ee48a20b207726caa5ab6451e0fac3642a9be10d9e70b3068df6 ports: - 6379:6379 options: >- @@ -198,6 +363,9 @@ jobs: openapi-schema: name: openapi-schema clippy · drift · coverage runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable @@ -207,7 +375,21 @@ jobs: - name: Clippy (--features openapi-schema) run: cargo clippy -p busbar --all-targets --features openapi-schema --locked -- -D warnings - name: OpenAPI tests (drift guard + coverage lock) - run: cargo test -p busbar --features openapi-schema --locked openapi -- --nocapture + run: | + set -euo pipefail + # FLOOR ON THE MATCH COUNT. `openapi` is a SUBSTRING filter; a filter that matches nothing + # prints "running 0 tests / test result: ok" and EXITS 0. The floor is not 1 — this step + # claims to run the drift guard AND the coverage lock AND their neighbours, so a filter + # that quietly collapsed to a single surviving test would still be a coverage hole. + out="$(cargo test -p busbar --features openapi-schema --locked openapi -- --nocapture 2>&1 | tee /dev/stderr)" + n="$(echo "$out" | sed -nE 's/^test result: ok\. ([0-9]+) passed.*/\1/p' | awk '{s+=$1} END{print s+0}')" + if [ "$n" -lt 8 ]; then + echo "::error::the OpenAPI suite ran only ${n} tests, expected >= 8." + echo "::error::A zero/low-match 'cargo test' exits 0 — this is NOT a pass. Fix the filter" + echo "::error::(or lower this floor deliberately, in the same commit that removes tests)." + exit 1 + fi + echo "OpenAPI suite: ${n} tests ran (floor 8)" # CONFIG-STABILITY gate. 1.5.3 is the LAST config-breaking release; after # it the config grammar is FROZEN and every future feature may add only NEW OPTIONAL keys/sections/ @@ -230,6 +412,9 @@ jobs: migration-corpus: name: migration corpus (every shipped config still migrates) runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 with: @@ -321,6 +506,9 @@ jobs: executable-config-lint: name: executable-config gate (heredocs · test literals · shipped yaml) runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - name: Install Rust toolchain @@ -334,7 +522,9 @@ jobs: python3 -c "import yaml" 2>/dev/null || pip install --quiet pyyaml python3 scripts/executable-config-lint.py --busbar target/debug/busbar --selftest - name: Executable-config gate - run: python3 scripts/executable-config-lint.py --busbar target/debug/busbar --root . + # --min-docs: the gate is vacuously green over an empty extraction set. 50 documents are + # found today; the floor is 40 so genuine churn passes and a collapsed extractor cannot. + run: python3 scripts/executable-config-lint.py --busbar target/debug/busbar --root . --min-docs 40 # Compliance-by-compilation gate: busbar must build + lint clean with the built-in auth plugin # COMPILED OUT (`--no-default-features`), so a regulated deployment can ship a binary that provably @@ -350,6 +540,9 @@ jobs: no-default-features: name: no-default-features build · clippy · test runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable @@ -385,6 +578,9 @@ jobs: no-plugins-gate: name: no-plugins gate (compiled out · not installed) runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable @@ -402,6 +598,9 @@ jobs: txn-guards: name: txn compile fence · loom model runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable @@ -418,17 +617,34 @@ jobs: timing: name: timing gate (release) runs-on: ubuntu-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - name: Hot-path timing gate - run: cargo test --release --locked timing_gate -- --ignored + run: | + set -euo pipefail + # FLOOR ON THE MATCH COUNT. `timing_gate` is a SUBSTRING filter and these tests are + # `#[ignore]`d, so this job is their ONLY execution anywhere. A filter that matches nothing + # prints "running 0 tests / test result: ok" and EXITS 0 — rename the tests and the hot-path + # timing gate is green forever having measured nothing, with no other job to notice. + out="$(cargo test --release --locked timing_gate -- --ignored 2>&1 | tee /dev/stderr)" + echo "$out" | grep -qE 'test result: ok\. [1-9][0-9]* passed' || { + echo "::error::the hot-path timing gate ran ZERO tests (renamed, moved, or un-ignored?)." + echo "::error::A zero-match 'cargo test' exits 0 — this is NOT a pass." + exit 1 + } # Portability gate: busbar must build + pass tests on Windows too (no OS-specific code). windows: name: windows build · test runs-on: windows-latest + # FULL TIER ONLY. Skipped on pushes to non-promotion branches; runs on every pull + # request, on dev/qa/main, and on workflow_dispatch. See the tier note at the top. + if: github.event_name != 'push' || contains(fromJSON('["refs/heads/main", "refs/heads/dev", "refs/heads/qa"]'), github.ref) steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 46bd749c..cbf1b644 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,10 +3,39 @@ name: Docker # Build and publish the busbar container image (Docker Hub + GHCR) as a multi-arch # manifest (linux/amd64 + linux/arm64), FROM scratch over static musl binaries. # +# THIS WORKFLOW NO LONGER OWNS THE VERSION TAG, AND THAT IS THE WHOLE POINT. +# +# It used to fire on a `v*` tag push and emit `X.Y.Z` + `latest` in the same build-push that +# produced the bytes. So the user-facing name existed the instant the build succeeded, before one +# consumer check had run against it. Docker Hub has TAG IMMUTABILITY enabled on getbusbar/busbar: a +# published `X.Y.Z` CANNOT be overwritten, so a broken one is permanent and the only remedies are +# deleting the tag (which has needed owner scope that was not available) or burning a version +# number. Creating the name before proving the bytes is therefore the wrong order regardless of +# convenience, and 1.5.3 paid for it. +# +# The order is now: build -> push under a THROWAWAY name -> verify from the consumer side -> +# PROMOTE. This file provides both halves as separate, independently callable paths, and +# release.yml sequences them. Exactly ONE thing emits `X.Y.Z`: the `promote` job below, reached +# only from release.yml's promote step, only after staged verification went green. +# # Triggers: -# - v* tags: publish with the semver tag cascade (X.Y.Z, X.Y, X, latest) -# - workflow_dispatch: publish a `test` tag only — end-to-end pipeline check -# without cutting a release. +# - workflow_call (how release.yml drives it): +# * staging_tag: build the multi-arch image and push it under THAT name only. Really +# pullable, really executable, but no user is looking at `staging-`. Outputs the +# manifest digest so the promote path can retag those exact bytes. +# * promote_to (+ promote_from): manifest-only retag. Creates `X.Y.Z` from the already-pushed +# staged manifest and moves `latest` onto it. NO REBUILD, so the promoted image is +# bit-identical to the one verification actually pulled and ran, and the build-provenance +# attestation recorded against the digest carries over unchanged. +# - workflow_dispatch: the manual recovery door for the same two primitives, plus a bare dispatch +# that publishes only a `test` tag (pipeline smoke test, cuts nothing). +# +# THE INPUT THAT WAS REMOVED, AND WHY. There used to be a `version` dispatch input that decided +# whether the semver tag was emitted AT ALL: `type=semver` was gated `enable=`, so a +# bare dispatch published only `test`. That is how the headroom bundle rebuild "succeeded" and +# published nothing anybody pulls. Two mechanisms both believing they own tagging is how this +# breaks again, so `version` is gone and `label_version` (which sets the OCI version LABEL and the +# bundled plugin's version, and emits NO tag) took over the part of its job that was legitimate. # # Supply chain: both registries carry a GitHub build-provenance attestation # (stored by GitHub — no extra tags on the repo). Verify with: @@ -19,24 +48,63 @@ name: Docker # --certificate-oidc-issuer https://token.actions.githubusercontent.com on: - push: - tags: - - "v*" - - "!v*-*" # NEVER publish a pre-release tag (e.g. v1.5.2-rc.1) — qa staging markers only. + # NO `push: tags` TRIGGER. Its absence is a load-bearing part of the design, not an omission: a + # `v*` tag existing must not, by itself, cause anything to be published under a user-facing name. + # scripts/release-order-lint.py asserts this file has no tag trigger and fails the build if one + # is added back. + workflow_call: + inputs: + staging_tag: + description: "Build and push the image under THIS throwaway tag only (e.g. staging-abc1234)." + required: false + type: string + default: "" + label_version: + description: >- + The X.Y.Z this build IS, for the org.opencontainers.image.version label and the bundled + headroom plugin's version. Emits NO tag. verify-deploy's check (d) reads this label off + the STAGED image, so the staged bytes must already claim the right version. + required: false + type: string + default: "" + promote_to: + description: >- + Manifest-only promote: create this version tag (e.g. 1.5.4) and move `latest` onto it. + No rebuild. Skips the build jobs entirely. + required: false + type: string + default: "" + promote_from: + description: >- + Source tag the promote copies manifests from (normally the staging tag). Leave empty to + promote from `promote_to` itself, which is the old `retag_from` behaviour: move `latest` + onto an already-published, immutable version without re-pushing it. + required: false + type: string + default: "" + outputs: + digest: + description: "Manifest digest of the image this run pushed (staging path only)." + value: ${{ jobs.publish.outputs.digest }} workflow_dispatch: inputs: - version: + staging_tag: + description: "Build and push under this throwaway tag only (e.g. staging-abc1234). Cuts nothing." + required: false + default: "" + label_version: + description: "X.Y.Z for the OCI version label and the bundled plugin. Emits NO tag." + required: false + default: "" + promote_to: description: >- - Publish as this version (e.g. 1.1.0) with the full semver tag cascade + - latest. Only use when the dispatched ref is source-identical to that - release. Leave empty to publish only a `test` tag. + Manifest-only promote: create this version tag and move `latest` onto it. No rebuild. required: false default: "" - retag_from: + promote_from: description: >- - Point `latest` at an ALREADY-PUBLISHED version (e.g. 1.1.1) with a - manifest-only retag — no rebuild, and the immutable X.Y.Z tag is never - re-pushed. Skips the build. Leave empty for a normal build. + Source tag to promote FROM. Empty means promote from promote_to itself (move `latest` + onto an already-published immutable version). This is the old `retag_from` input. required: false default: "" @@ -57,11 +125,29 @@ permissions: attestations: write # build-provenance attestation jobs: - # Manifest-only retag: point `latest` at an already-published version, no rebuild. - # Used when version tags are immutable — the immutable X.Y.Z is never re-pushed. - retag-latest: - name: point latest at ${{ inputs.retag_from }} - if: ${{ inputs.retag_from != '' }} + # -- THE PROMOTE PRIMITIVE: manifest-only, no rebuild --------------------------------------------- + # + # This is the ONLY job in this repository that emits a user-facing `X.Y.Z` container tag. It takes + # bytes that are ALREADY pushed (normally under `staging-`, and already pulled, booted and + # executed by verification) and gives them their real names. Three properties make this the right + # promote primitive rather than a rebuild: + # + # * BIT-IDENTICAL. `imagetools create` copies manifests. The promoted `X.Y.Z` is the exact + # digest verification ran, not a second build that "should" be the same. A rebuild would make + # the verification a statement about a DIFFERENT artifact, which is no verification at all. + # * THE ATTESTATION CARRIES OVER. Build provenance is recorded against the DIGEST, so the same + # digest under a new tag is still attested. `gh attestation verify oci://...:X.Y.Z` passes + # without re-attesting. + # * CHEAP AND RETRYABLE. Seconds, not the ~40 minutes a two-arch PGO build costs, so a promote + # that fails half way can be re-run immediately. + # + # IMMUTABILITY IS WHY THE ORDER INSIDE THIS JOB IS WHAT IT IS. `X.Y.Z` on Docker Hub cannot be + # overwritten, so it is pushed FIRST, while nothing else has moved: if the registry rejects it + # (the tag already exists) the job dies before `latest` has been touched and before release.yml + # has pushed a git tag or published anything. `latest` is mutable and therefore safe to move last. + promote: + name: promote ${{ inputs.promote_from || inputs.promote_to }} -> ${{ inputs.promote_to }} + latest + if: ${{ inputs.promote_to != '' }} runs-on: ubuntu-latest steps: - uses: docker/setup-buildx-action@v4 @@ -76,26 +162,80 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Retag latest -> ${{ inputs.retag_from }} (multi-arch preserved) + - name: Promote (manifest-only, multi-arch preserved) env: - V: ${{ inputs.retag_from }} + V: ${{ inputs.promote_to }} + # Empty SRC means "promote from V itself", i.e. the old `retag_from` recovery: V is + # already published and immutable, and all that is wanted is `latest` pointed at it. + SRC: ${{ inputs.promote_from }} run: | set -euo pipefail - # Docker Hub: point latest at the (immutable) version. Idempotent. - docker buildx imagetools create -t "${DOCKERHUB_IMAGE}:latest" "${DOCKERHUB_IMAGE}:${V}" - # GHCR may be MISSING this version if a prior combined build-push aborted mid-way - # (e.g. Docker Hub tag-immutability rejected `latest` before GHCR finished). Mirror the - # version from Docker Hub into GHCR (cross-registry manifest copy, no rebuild), then move - # GHCR latest onto it. `imagetools create` is a no-op-safe overwrite if it already exists. + src="${SRC:-$V}" + echo "Promoting ${DOCKERHUB_IMAGE}:${src} -> :${V} and :latest on both registries." + + # 1. The IMMUTABLE tag first, on both registries, while nothing else has moved. + if [ "$src" != "$V" ]; then + docker buildx imagetools create -t "${DOCKERHUB_IMAGE}:${V}" "${DOCKERHUB_IMAGE}:${src}" + fi + # GHCR may be MISSING this version if a prior combined push aborted mid-way. Mirror it + # from Docker Hub (cross-registry manifest copy, no rebuild). `imagetools create` is a + # no-op-safe overwrite if it already exists, which is what makes a re-run idempotent. docker buildx imagetools create -t "${GHCR_IMAGE}:${V}" "${DOCKERHUB_IMAGE}:${V}" + + # 2. Only then the MUTABLE pointer. Moving `latest` before the pin exists would, for the + # window between the two, hand `docker pull getbusbar/busbar` an image no user can pin. + docker buildx imagetools create -t "${DOCKERHUB_IMAGE}:latest" "${DOCKERHUB_IMAGE}:${V}" docker buildx imagetools create -t "${GHCR_IMAGE}:latest" "${GHCR_IMAGE}:${V}" - echo "== Docker Hub latest =="; docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:latest" | grep -iE 'name|platform' - echo "== GHCR ${V} =="; docker buildx imagetools inspect "${GHCR_IMAGE}:${V}" | grep -iE 'name|platform' + + # PARTIAL PROMOTE IS DETECTABLE, NOT SILENT. `imagetools create` exiting 0 says the API call + # was accepted; it does not say the registry now serves that name, and "the pin published but + # `latest` never moved" is precisely the class of defect that shipped 1.5.3. So the promote is + # re-derived from OUTSIDE, over the Distribution API the way `docker pull` reads it, and all + # four names must agree on ONE digest. Any disagreement names which half landed. + - name: Assert the promote actually landed on all four names + env: + V: ${{ inputs.promote_to }} + run: | + set -euo pipefail + reg_digest() { # reg_digest + local auth_host="$1" reg_host="$2" repo="$3" tag="$4" token_url token + if [ "$auth_host" = "auth.docker.io" ]; then + token_url="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" + else + token_url="https://${auth_host}/token?service=${auth_host}&scope=repository:${repo}:pull" + fi + token="$(curl -fsS --max-time 30 "$token_url" | jq -r '.token // .access_token')" + [ -n "$token" ] && [ "$token" != "null" ] || return 1 + curl -fsS --max-time 30 -I \ + -H "Authorization: Bearer $token" \ + -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json' \ + "https://${reg_host}/v2/${repo}/manifests/${tag}" \ + | tr -d '\r' | grep -i '^docker-content-digest:' | awk '{print $2}' + } + hub_v="$(reg_digest auth.docker.io registry-1.docker.io getbusbar/busbar "$V" || true)" + hub_l="$(reg_digest auth.docker.io registry-1.docker.io getbusbar/busbar latest || true)" + ghcr_v="$(reg_digest ghcr.io ghcr.io getbusbar/busbar "$V" || true)" + ghcr_l="$(reg_digest ghcr.io ghcr.io getbusbar/busbar latest || true)" + echo "docker.io ${V} = ${hub_v:-}" + echo "docker.io latest = ${hub_l:-}" + echo "ghcr.io ${V} = ${ghcr_v:-}" + echo "ghcr.io latest = ${ghcr_l:-}" + fail=0 + [ -n "$hub_v" ] || { echo "::error::PARTIAL PROMOTE: docker.io/getbusbar/busbar:${V} does not resolve. The version pin was never created. Nothing downstream should treat ${V} as released."; fail=1; } + for pair in "docker.io latest:$hub_l" "ghcr.io ${V}:$ghcr_v" "ghcr.io latest:$ghcr_l"; do + name="${pair%%:*}"; got="${pair#*:}" + if [ "$got" != "$hub_v" ] || [ -z "$got" ]; then + echo "::error::PARTIAL PROMOTE: ${name} resolves to '${got:-}' but docker.io/${V} is '${hub_v}'. Exactly the 1.5.3 shape: one name moved and another did not, and nothing was red. Fix: re-run docker.yml with promote_to=${V} (the promote is idempotent), then re-run this assertion." + fail=1 + fi + done + [ "$fail" = 0 ] || exit 1 + echo "PASS: docker.io ${V}, docker.io latest, ghcr.io ${V} and ghcr.io latest all resolve to ${hub_v}." # Static musl binaries, one per architecture, built on native runners (no QEMU). build-binaries: name: musl ${{ matrix.arch }} - if: ${{ inputs.retag_from == '' }} + if: ${{ inputs.promote_to == '' }} runs-on: ${{ matrix.os }} strategy: fail-fast: true @@ -273,10 +413,12 @@ jobs: run: | set -euo pipefail # Same version resolution as the publish job's "Resolve version" step (duplicated here, - # not shared, since this runs in a different job): the workflow_dispatch `version` input, - # else the pushed v* tag, else a placeholder for a bare `test`-tag dispatch build. - ver="${{ inputs.version }}" - if [ -z "$ver" ] && [ "$GITHUB_REF_TYPE" = "tag" ]; then ver="${GITHUB_REF_NAME#v}"; fi + # not shared, since this runs in a different job): the `label_version` input, else a + # placeholder for a bare `test`-tag dispatch build. There is deliberately no fallback to + # a git tag: this workflow no longer runs on tag pushes, and inferring a release version + # from the ref is exactly the coupling the restructure removed. + ver="${{ inputs.label_version }}" + ver="${ver#v}" [ -n "$ver" ] || ver="0.0.0-test" outdir="plugin-dist"; mkdir -p "$outdir" unsigned_flag="" @@ -319,6 +461,8 @@ jobs: name: build & push image needs: build-binaries runs-on: ubuntu-latest + outputs: + digest: ${{ steps.push.outputs.digest }} steps: - uses: actions/checkout@v7 @@ -360,14 +504,15 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # The version being published: the pushed v* tag, or the workflow_dispatch - # `version` input. Empty (bare dispatch) → publish only a `test` tag. - - name: Resolve version + # The LABEL version. This does not, and must not, decide any tag. + - name: Resolve label version id: ver run: | - VER="${{ inputs.version }}" - if [ -z "$VER" ] && [ "$GITHUB_REF_TYPE" = "tag" ]; then VER="$GITHUB_REF_NAME"; fi - echo "ver=${VER#v}" >> "$GITHUB_OUTPUT" + VER="${{ inputs.label_version }}"; VER="${VER#v}" + # Matches the plugin-pack step's fallback, so a bare `test` dispatch labels the image + # honestly rather than with an empty string. + [ -n "$VER" ] || VER="0.0.0-test" + echo "ver=${VER}" >> "$GITHUB_OUTPUT" - name: Compute tags and labels id: meta @@ -376,12 +521,25 @@ jobs: images: | ${{ env.DOCKERHUB_IMAGE }} ${{ env.GHCR_IMAGE }} - # version resolved → X.Y.Z + latest (exact pins only, no floating - # major/minor aliases — one consistent tag shape); bare dispatch → `test` only. + # EXACTLY ONE TAG COMES OUT OF THIS JOB, AND IT IS NEVER A VERSION. + # staging_tag set -> that throwaway name and nothing else. + # staging_tag empty (bare dispatch) -> `test` and nothing else. + # `X.Y.Z` and `latest` are emitted by the `promote` job above, from already-verified + # bytes. There is no `type=semver` line here any more and there must not be one: the old + # `type=semver ... enable=` shape is what let a build decide, on its own, + # whether a release existed. scripts/release-order-lint.py fails the build if `semver`, + # or a `latest` raw tag, reappears in this block. tags: | - type=semver,pattern={{version}},value=v${{ steps.ver.outputs.ver }},enable=${{ steps.ver.outputs.ver != '' }} - type=raw,value=test,enable=${{ steps.ver.outputs.ver == '' }} + type=raw,value=${{ inputs.staging_tag }},enable=${{ inputs.staging_tag != '' }} + type=raw,value=test,enable=${{ inputs.staging_tag == '' }} + # org.opencontainers.image.version IS SET EXPLICITLY, and it has to be. metadata-action + # derives that label from the tag it computed, which under a throwaway name would make the + # staged image claim `staging-abc1234` as its version -- and verify-deploy's check (d) + # reads exactly this label to prove the tag and the bytes agree. Setting it from + # label_version means the staged image already claims the version it will be promoted to, + # so (d) is a real assertion at staging time and the promoted image needs no relabelling. labels: | + org.opencontainers.image.version=${{ steps.ver.outputs.ver }} org.opencontainers.image.title=busbar org.opencontainers.image.description=The reliability layer for LLM traffic — one endpoint, six wire protocols, fault-attributed circuit breaking, in-flight failover. org.opencontainers.image.url=https://getbusbar.com @@ -396,7 +554,18 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + # THE VERSION LABEL IS REPEATED HERE ON PURPOSE, AND IT IS NOT BELT-AND-BRACES FOR ITS OWN + # SAKE. metadata-action derives an `org.opencontainers.image.version` label from the tag it + # computed, which under a throwaway name would be `staging-`; the `labels:` input above + # overrides it, but that override depends on metadata-action's merge order, which is a + # property of a third-party action rather than of anything here. buildx applies `--label` + # last-one-wins, so restating it after the generated block makes the outcome deterministic + # regardless. This matters because verify-deploy's check (d) reads exactly this label: if it + # ever said `staging-`, EVERY release would fail the staged gate, and a check that + # blocks every release for a reason unrelated to the software is the fragile-gate defect. + labels: | + ${{ steps.meta.outputs.labels }} + org.opencontainers.image.version=${{ steps.ver.outputs.ver }} - name: Install cosign uses: sigstore/cosign-installer@v3 diff --git a/.github/workflows/mcp-conformance.yml b/.github/workflows/mcp-conformance.yml new file mode 100644 index 00000000..7896b0b2 --- /dev/null +++ b/.github/workflows/mcp-conformance.yml @@ -0,0 +1,467 @@ +name: MCP conformance + +# WHERE THIS RUNS, AND WHY IT RUNS HERE. +# +# A conformance battery is a statement ABOUT BUSBAR, so it belongs where busbar is built and where a +# red blocks the release it is about. These legs used to be attempted in the private design repo, on +# a Gitea instance with no registered runners for it: eight jobs never scheduled and stayed +# permanently red, and the only two that reported success had executed nothing. A standing red +# nobody can make green teaches everyone to ignore the repo's status, so the first REAL conformance +# failure would have looked exactly like the eight expected ones. +# +# busbar is a PUBLIC repo, so GitHub-hosted runners are free and already available. That is the +# org rule (public → `ubuntu-latest`, private → `busbar-selfhosted`) and it is why nothing here +# needs provisioning. +# +# THERE IS NO SECRET ANYWHERE IN THIS FILE, and that is load-bearing rather than incidental. The +# in-house battery used to be cloned from a private repository on an internal Gitea host +# with NO route from a GitHub-hosted runner: reaching it needed a secret AND a reachable private +# host, and the moment a control leg depends on either, "the control legs run ALWAYS" is +# aspirational. So the battery was moved into `testing/mcp-conformance/` in this repository +# (DECISION 2), exactly as the sibling A2A workflow had already done and for the same reason. A +# battery that contains no product knowledge by construction loses nothing by being readable. +# +# INDEPENDENCE IS A PROPERTY OF AUTHORSHIP, NOT OF LOCATION. The official suite is written and +# maintained by the people who write the MCP specification and shares none of busbar's code; running +# it from busbar's CI does not make it ours. +# +# THE TWO-LEG RULE, which is the whole shape of this file: +# +# CONTROL runs ALWAYS. A battery that cannot judge a known-good third-party peer cannot be trusted +# to judge ours. A red control leg is a finding about the harness, the pin or the runner — never +# about busbar — and it must be visible BEFORE any subject verdict is believed. +# +# SUBJECT IS ARMED OR RED. This is a REVERSAL of the previous policy and it is the point of this +# whole file. The subject legs used to SKIP until armed, and the argument was good: a job that is +# red today for a reason that is not a defect is how red stops meaning defect. It stopped being +# good the moment the release claim became "busbar implements MCP", because a disarmed subject leg +# proves the SUITE works and proves NOTHING about busbar — while rendering as the identical green +# tick a leg that judged busbar and passed would produce. `NOT ARMED, SO NOT RUN` is therefore a +# RED state, the transition is exercised by `mcp-conformance.sh --selftest` rather than asserted, +# and each subject job publishes its arm state as a JOB OUTPUT because an all-steps-skipped job +# reports `success` and the aggregator would otherwise be unable to tell the two apart. +# +# AND `verdict` IS NOT OPTIONAL. Ten green ticks mean nothing if one of them is green because it +# never ran. The last job asserts, per leg BY NAME, that the leg reached `success`; skipped or +# cancelled is RED there. Its `needs:` list is held to SET EQUALITY with the workflow's job set by +# `testing/verdict-covers-every-leg.py`, because a `needs:` list is exactly the kind of +# hand-maintained enumeration that stops covering whatever is added after it. +# +# This is a separate workflow rather than a job in ci.yml on purpose: the control leg builds a +# 53-package third-party TypeScript workspace, which has no business on the per-commit fast tier. +on: + push: + branches: ['**'] + pull_request: + workflow_dispatch: + +concurrency: + group: mcp-conformance-${{ github.ref }} + cancel-in-progress: true + +jobs: + gate-selftest: + name: gate self-test (the coverage assertion cannot be lied to) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + # FIRST, and separately from any verdict. The same discipline structure-lint and + # no-plugins-gate follow: never trust a gate's verdict before proving the gate still works. + # The self-test plants runs that MUST be refused (no results at all; a run missing a required + # scenario; an empty requirement set) and runs that must be accepted, so a green below cannot + # be produced by an assertion that refuses everything or accepts everything. It also drives + # the ARM-STATE TRANSITION in both directions, because a rule whose enforcement is only ever + # exercised by the real thing is a rule nobody has watched work. + - name: Prove the anti-vacuity and arm-state assertions bite + run: ./scripts/mcp-conformance.sh --selftest + + - name: Prove the verdict-coverage lint bites + run: | + set -euo pipefail + python3 -m pip install --quiet pyyaml + python3 testing/verdict-covers-every-leg.py --selftest + + # The aggregator's `needs:` list is itself a hand-maintained enumeration, and the last + # enumeration in this tree that stopped covering what came after it did so silently. + - name: The verdict must depend on, and judge, every leg + run: python3 testing/verdict-covers-every-leg.py + + - name: Prove the fixture-absence gate bites + run: ./scripts/mcp-fixture-absence-gate.sh --selftest + + official-control: + name: official suite · CONTROL (pinned reference SDK) + needs: gate-selftest + runs-on: ubuntu-latest + # The control peer is a full third-party workspace build. Generous, because a timeout here would + # be a red that is not a defect — the exact thing this workflow is arranged to avoid. + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + with: + node-version: '22' + # The control peer's repo pins pnpm through its own lockfile. Installed explicitly rather than + # via corepack so the version is visible in the log when a build breaks. + - name: Install pnpm + run: npm install -g pnpm@10 + # Keyed on the PIN, not on a lockfile hash: the checkout and its build are entirely a function + # of the commit the script pins, so the cache is correct by construction and a pin bump misses + # it on purpose. Without this the control leg builds a 53-package workspace on every push. + - name: Cache the pinned control peer + uses: actions/cache@v4 + with: + path: .mcp-conformance/sdk + key: mcp-control-peer-${{ hashFiles('scripts/mcp-conformance.sh') }} + - name: Judge the pinned reference SDK + run: ./scripts/mcp-conformance.sh --official-control + - name: Publish the control results + if: always() + uses: actions/upload-artifact@v7 + with: + name: mcp-official-control + path: .mcp-conformance/control + if-no-files-found: warn + + official-subject: + name: official suite · SUBJECT (busbar — ARMED OR RED) + needs: official-control + runs-on: ubuntu-latest + timeout-minutes: 45 + # PUBLISHED, not inferred. A job whose steps all skip still reports `success`, so the aggregator + # cannot tell "armed and passed" from "unarmed and did nothing" by reading the job result -- + # which is precisely the shape of false green this workflow exists to refuse. The arm state is + # therefore an explicit output the verdict reads, and the verdict treats `false` as RED. + outputs: + armed: ${{ steps.arm.outputs.armed }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + with: + node-version: '22' + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + # THE SUBJECT IS BUILT FROM THIS COMMIT, and that is the change that makes this leg mean + # something. It used to be armed from `vars.MCP_CONFORMANCE_SUBJECT_URL` -- an externally + # deployed busbar -- which made a RELEASE GATE depend on a deployment being up, reachable and + # correctly configured at the moment CI happened to run. Both verdicts were then unreadable: a + # green meant "that deployment was fine yesterday" and a red meant "somebody redeployed", and + # neither is a statement about the commit under test. The sibling A2A workflow settled the + # equivalent question the same way, by booting its peer in the job with no secret. + # + # `CARGO_INCREMENTAL=0`: incremental artifacts are worthless on a fresh runner and a partially + # populated incremental cache has produced spurious "unable to copy ... No such file or + # directory" build failures, which is a red that is not a defect. + - name: Build the subject from this commit + env: + CARGO_INCREMENTAL: '0' + run: cargo build --bin busbar + + # This step RECORDS the arm state and does not judge it; the `Judge busbar` step below fails + # on its own when unarmed, and the verdict fails independently on the output. Two mechanisms + # for one fact, deliberately: the step can be re-run in isolation and still be honest, and the + # verdict still catches a job that was cancelled before this step ever ran. + # + # NO SECRET AND NO REPOSITORY VARIABLE ARMS THIS ANY MORE. The arm is a FILE this job just + # built, so the leg cannot silently disarm because somebody deleted a variable or let a + # deployment lapse -- it can only disarm by the build failing, which is itself red. + - name: Record the arm state + id: arm + run: | + set -euo pipefail + if [ -x target/debug/busbar ]; then + echo "armed=true" >> "$GITHUB_OUTPUT" + else + echo "armed=false" >> "$GITHUB_OUTPUT" + { + echo "### MCP official-suite subject leg: NOT ARMED — this is RED" + echo "" + echo "The build produced no busbar binary, so the official suite could only have run" + echo "against its pinned third-party control. That proves the SUITE works. It proves" + echo "nothing about busbar." + } >> "$GITHUB_STEP_SUMMARY" + fi + + # NOT conditional on the arm state. Unarmed, this step FAILS, and that is the transition this + # goal is about. The script boots busbar on loopback, mints a REAL audience-bound credential + # for it with the signing key this job generated, and PROVES the plane boundary is still + # intact (no credential / no audience / wrong audience / flipped signature must all be 401, + # and the right token must be 200) before it lets the suite start -- because a leg that + # authenticated itself by weakening the thing under test would be green about a busbar nobody + # runs, which is worse than leaving it unarmed. The anti-vacuity assertion then holds the run + # to SET EQUALITY with the revision's required scenario set -- never a count, because a floor + # of 30 is satisfied by any 30 of 37. + - name: Judge busbar + env: + MCP_SUBJECT_BUSBAR_BIN: target/debug/busbar + run: ./scripts/mcp-conformance.sh --official-subject + + # THE OPTIONAL EXTRA LEG. If an operator also wants a real deployment judged, setting the + # variable adds a second, STRICT run against it -- it is never a substitute for the booted + # subject above, and it is never soft: a run that happens must pass. Absent, nothing here + # runs and nothing above depends on it. + - name: Also judge an external deployment, if one is configured + if: ${{ vars.MCP_CONFORMANCE_SUBJECT_URL != '' }} + env: + MCP_CONFORMANCE_SUBJECT_URL: ${{ vars.MCP_CONFORMANCE_SUBJECT_URL }} + MCP_SUBJECT_OUT: .mcp-conformance/subject-external + run: ./scripts/mcp-conformance.sh --official-subject + + # BOTH result sets, and the booted one is never overwritten by the optional one -- a verdict + # standing over another run's artifact is unreadable evidence. + - name: Publish the subject results + if: always() + uses: actions/upload-artifact@v7 + with: + name: mcp-official-subject + path: | + .mcp-conformance/subject + .mcp-conformance/subject-external + if-no-files-found: ignore + + battery-control: + name: in-house battery · CONTROL (pinned python reference peer) + needs: gate-selftest + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + with: + node-version: '22' + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + # No fetch, no clone, no secret. The battery is `testing/mcp-conformance/` in this checkout. + - name: Judge the pinned control peer + run: ./scripts/mcp-conformance.sh --battery-control + # THE SUBJECT LEG NEEDS THIS REPORT, and until now it never received it. `battery-subject` + # declared `needs: battery-control`, which orders the jobs but does NOT share a workspace, so + # `run-subject.sh` found no control report and exited 1 — under a message blaming busbar. + # A differential with nothing to differ against is not a comparison. + - uses: actions/upload-artifact@v7 + if: always() + with: + name: mcp-battery-control-report + path: testing/mcp-conformance/reports/control-*.json + if-no-files-found: error + + battery-negative-control: + name: in-house battery · NEGATIVE control (a broken peer MUST be red) + needs: gate-selftest + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + with: + node-version: '22' + # A battery that only ever runs against a good implementation says nothing about its own + # sensitivity. This drives deliberately broken fake peers and requires the battery to catch + # each one. Without it, "the control leg is green" is equally consistent with a battery that + # cannot fail. + - name: Deliberately broken peers must be caught + working-directory: testing/mcp-conformance + run: | + set -euo pipefail + out="$(./scripts/negative-control.sh)" + printf '%s\n' "$out" + # The `honest` row must have zero failures and every broken row must have at least one. + # Asserted on the OUTPUT, per row, rather than on an exit code the script does not set. + # Piped rather than heredoc'd: `python3 - <<'PY' <<<"$out"` has TWO stdin redirections and + # the last one wins, so the script would never be read at all and the step would pass + # having asserted nothing. + printf '%s\n' "$out" | python3 -c ' + import re, sys + rows = [l for l in sys.stdin.read().splitlines() if l.strip()][1:] + if len(rows) < 7: + sys.exit("negative control produced %d rows; it did not run." % len(rows)) + bad = [] + for row in rows: + mode = row.split()[0] + m = re.search(r"(\d+) fail", row) + if m is None: + bad.append("unreadable row (no fail count): %s" % row) + continue + caught = int(m.group(1)) > 0 + if mode == "honest" and caught: + bad.append("the HONEST peer was rejected: %s" % row) + if mode != "honest" and not caught: + bad.append("a BROKEN peer (%s) was blessed: %s" % (mode, row)) + if bad: + sys.exit("\n".join(bad)) + print("negative control behaved: %d rows, every broken peer caught." % len(rows)) + ' + + battery-subject: + name: in-house battery · SUBJECT (busbar — ARMED OR RED) + needs: battery-control + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + armed: ${{ steps.arm.outputs.armed }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + with: + node-version: '22' + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + # THE SUBJECT IS BUILT FROM THIS COMMIT, the same change that made the official subject leg + # mean something, applied to the leg that never ran at all. This one was armed out of + # `vars.MCP_SUBJECT_SERVER_CMD` -- a command line naming a binary on somebody's machine -- and + # CI supplied neither it nor its client twin for the whole of 1.5.5, so the leg reported `NOT + # ARMED, SO NOT RUN` from the day the rule was written. A variable nobody can set correctly + # from a GitHub-hosted runner is not an arm; a file this job just built is. + # + # `CARGO_INCREMENTAL=0` for the reason the official leg gives: a partially populated + # incremental cache has produced spurious build failures, which is a red that is not a defect. + - name: Build the subject from this commit + env: + CARGO_INCREMENTAL: '0' + run: cargo build --bin busbar + - name: Record the arm state + id: arm + env: + SERVER_CMD: ${{ vars.MCP_SUBJECT_SERVER_CMD }} + CLIENT_CMD: ${{ vars.MCP_SUBJECT_CLIENT_CMD }} + run: | + set -euo pipefail + if [ -x target/debug/busbar ] || [ -n "${SERVER_CMD:-}${CLIENT_CMD:-}" ]; then + echo "armed=true" >> "$GITHUB_OUTPUT" + else + echo "armed=false" >> "$GITHUB_OUTPUT" + { + echo "### MCP in-house battery subject leg: NOT ARMED — this is RED" + echo "" + echo "The build produced no busbar binary, so the battery could only have run against" + echo "its pinned third-party control and its broken fake peers. That proves the" + echo "BATTERY works. It proves nothing about busbar." + } >> "$GITHUB_STEP_SUMMARY" + fi + # MCP_NO_SKIPS=1 is set inside the script for this leg: a skipping test is not a passing test. + # It makes the six `pr`-tier SEAM tests RED while busbar has no MCP CLIENT direction able to + # mount the battery's fake server as an upstream. That is the correct answer and not a + # nuisance: the seam is the one property that is meaningless with only one direction built -- + # inbound audience validation defends nothing without an inbound surface, and outbound + # down-scoping means nothing without upstreams -- so a green there before the client direction + # lands would be a lie about exactly the property that matters most. + # + # The battery speaks stdio and busbar's MCP plane is HTTP, so the leg boots the built binary + # exactly as the official leg does and points the battery at it through the transport adapter + # in `testing/mcp-conformance/scripts/stdio-http-bridge.mjs`. The repository variables are + # still honoured and still take precedence, for an operator judging something that is not this + # build -- but nothing in CI depends on one being set any more. + # THE CONTROL REPORT THE DIFFERENTIAL COMPARES AGAINST. `needs:` orders the jobs; it does + # not share a workspace, so without this the subject leg reached its differential with + # nothing to differ against and exited non-zero — for years, on every commit. + - uses: actions/download-artifact@v7 + with: + name: mcp-battery-control-report + path: testing/mcp-conformance/reports/ + - name: Judge busbar + env: + MCP_SUBJECT_BUSBAR_BIN: target/debug/busbar + MCP_SUBJECT_SERVER_CMD: ${{ vars.MCP_SUBJECT_SERVER_CMD }} + MCP_SUBJECT_CLIENT_CMD: ${{ vars.MCP_SUBJECT_CLIENT_CMD }} + MCP_SUBJECT_UPSTREAM_CONFIG_CMD: ${{ vars.MCP_SUBJECT_UPSTREAM_CONFIG_CMD }} + run: ./scripts/mcp-conformance.sh --battery-subject + - uses: actions/upload-artifact@v7 + if: always() + with: + name: mcp-battery-subject + path: testing/mcp-conformance/reports/ + if-no-files-found: ignore + + fixture-absence: + name: the test fixtures are absent from a real build (both axes) + needs: gate-selftest + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + # Two axes, because a fixture can survive into a shipped build two independent ways and + # neither axis sees the other's. See the script's header. + - name: Both axes + run: ./scripts/mcp-fixture-absence-gate.sh --run + + verdict: + name: MCP conformance verdict + runs-on: ubuntu-latest + if: always() + needs: + - gate-selftest + - official-control + - official-subject + - battery-control + - battery-negative-control + - battery-subject + - fixture-absence + steps: + # A ROW THAT CANNOT RUN IS RED, NEVER SKIPPED. Every leg above is required to have reached + # `success`. `skipped` and `cancelled` are failures here, because the failure mode this whole + # workflow exists to prevent is a tick over a job that executed nothing. + # + # THE SUBJECT LEGS ARE NOT EXEMPT. In the sibling A2A workflow the subject leg is allowed to + # be unarmed, named explicitly so that arming it moves it into the strict set. Here that + # exemption is DELETED: GOAL-1.5.5 §A.3 requires that `NOT ARMED, SO NOT RUN` be a red state, + # so an `armed=false` output fails this job. The arm state is read from the job's own + # published OUTPUT and not from its result, because an all-steps-skipped job reports + # `success` and would otherwise be indistinguishable from a leg that judged busbar. + - name: Every leg must have EXECUTED, and every subject leg must have been ARMED + env: + SELFTEST: ${{ needs.gate-selftest.result }} + OFFICIAL_CONTROL: ${{ needs.official-control.result }} + OFFICIAL_SUBJECT: ${{ needs.official-subject.result }} + OFFICIAL_SUBJECT_ARMED: ${{ needs.official-subject.outputs.armed }} + BATTERY_CONTROL: ${{ needs.battery-control.result }} + BATTERY_NEGATIVE_CONTROL: ${{ needs.battery-negative-control.result }} + BATTERY_SUBJECT: ${{ needs.battery-subject.result }} + BATTERY_SUBJECT_ARMED: ${{ needs.battery-subject.outputs.armed }} + FIXTURE_ABSENCE: ${{ needs.fixture-absence.result }} + run: | + set -euo pipefail + fail=0 + strict () { + if [ "$2" != "success" ]; then + echo "::error::$1 did not succeed (result: $2). A leg that did not EXECUTE is red, not skipped." + fail=1 + else + echo " ok $1" + fi + } + # `armed `: a subject leg must have BOTH succeeded and + # published `armed=true`. An empty output means the job never reached its own arming + # step, so whether busbar was tested is unknown -- and unknown is red. + armed () { + case "${3:-}" in + true) strict "$1" "$2" ;; + false) + echo "::error::$1 published armed=false. NOT ARMED, SO NOT RUN is a RED state: the controls proved the instruments, not busbar. Set the repository variable that arms it." + fail=1 ;; + *) + echo "::error::$1 published no arm state (result: $2, armed: '${3:-}'). The leg did not reach its own arming check, so whether busbar was tested is unknown -- and unknown is red." + fail=1 ;; + esac + } + strict gate-selftest "$SELFTEST" + strict official-control "$OFFICIAL_CONTROL" + strict battery-control "$BATTERY_CONTROL" + strict battery-negative-control "$BATTERY_NEGATIVE_CONTROL" + strict fixture-absence "$FIXTURE_ABSENCE" + armed official-subject "$OFFICIAL_SUBJECT" "$OFFICIAL_SUBJECT_ARMED" + armed battery-subject "$BATTERY_SUBJECT" "$BATTERY_SUBJECT_ARMED" + + [ "$fail" -eq 0 ] || exit 1 + echo + echo "MCP conformance verdict: every leg executed, and both subject legs judged busbar." diff --git a/.github/workflows/plugin-ci.yml b/.github/workflows/plugin-ci.yml index 26f482a8..f17d0289 100644 --- a/.github/workflows/plugin-ci.yml +++ b/.github/workflows/plugin-ci.yml @@ -5,9 +5,13 @@ # file exists so every plugin repo's CI is the SAME file, not a fork of it. # # Standard stages every plugin gets, in this order: -# 1. Checkout the plugin + a sibling checkout of busbar (this repo) at BUSBAR_REF — the same -# interim local-path-dependency pattern every plugin's Cargo.toml already uses. Update -# BUSBAR_REF's default below once 1.5.0 ships and plugins can track `main` again. +# 0. RESOLVE WHICH BUSBAR. Every stage below runs ONCE PER LEG of a matrix with two legs: the +# PINNED busbar from the caller's own `.busbar-ref` (what that repo's release will actually +# build against) and the MOVING busbar from `busbar_ref` (the engine branch it tracks). Either +# leg red is red. The two collapse to one leg when they are the same commit. See the `refs` +# job for why running only one of them loses real coverage. +# 1. Checkout the plugin + a sibling checkout of busbar (this repo) at that leg's commit: the +# same interim local-path-dependency pattern every plugin's Cargo.toml already uses. # 2. Optionally boot a real backend service container (`postgres:16` / `valkey/valkey:8` / # `hashicorp/vault` dev-mode) — never a mock. `pg_isready`/the image's own Docker healthcheck # (or, for vault, a `wget` against its own `/v1/sys/health` endpoint) gates every later step so @@ -45,14 +49,6 @@ on: required: false type: string default: "none" - extra_sibling_test_command: - description: >- - Optional extra shell command run from the sibling busbarAI checkout after this repo's own - tests pass — e.g. `cargo test -p busbar-store-postgres` to also exercise the monorepo's - own live-service test against the same running container. - required: false - type: string - default: "" busbar_ref: description: "Which busbar branch to check out as the sibling dependency" required: false @@ -60,15 +56,103 @@ on: # main since v1.5.0 shipped (2026-08-02): plugins build against the released engine by # default, per the header's own "update once 1.5.0 ships" note. Pass busbar_ref: dev # explicitly to test a plugin against unreleased engine work. + # + # NOTE this is now only the MOVING leg of a two-leg matrix. See the `refs` job below: the + # PINNED leg is derived from the caller's own `.busbar-ref` and is not configurable, because + # it is not an opinion. It is whatever that repo's release will actually build against. default: "main" jobs: + # === WHICH BUSBAR DOES A PLUGIN'S CI ACTUALLY PROVE ANYTHING ABOUT === + # + # THE DEFECT THIS JOB EXISTS TO CLOSE. Until now a plugin's CI and a plugin's RELEASE built against + # two different busbars, and nothing anywhere compared them: + # - CI built against `inputs.busbar_ref`, which every caller sets to `github.ref_name` (or + # leaves at `main`). That is the MOVING engine: whatever busbar's branch holds right now. + # - RELEASE builds against field 1 of the plugin repo's own `.busbar-ref`, a PINNED SHA that + # `release-on-upstream.yml` rewrites only when it cuts. That is the engine the shipped + # artifact is actually compiled and linked against. + # So a green CI proved the plugin worked against a busbar its release would never build, and an + # API or config-grammar break between the two refs stayed INVISIBLE UNTIL RELEASE DAY. That is not + # hypothetical: it is why the 1.5.3 fixture breaks were latent across a long list of first-party + # repos whose CI was green the whole time they were writing configs 1.5.3 had already retired. + # + # WHY NOT SIMPLY UNIFY ON THE PIN. Because both refs are load-bearing and they answer different + # questions, and dropping either one loses real coverage: + # PINNED is the only ref whose green is load-bearing for SHIPPING. If this is red, the release + # is broken today. Nothing may be green without it. + # MOVING is the early warning. Testing only the pin makes a ratchet that never turns: a plugin + # stays green forever while drifting arbitrarily far from the engine, and the break + # finally lands inside `release-on-upstream.yml`, which re-pins `.busbar-ref` AND cuts + # the tag in one workflow. Discovering it there is discovering it too late. + # So: run BOTH, fail on EITHER. The cost is bounded, because when the pin and the moving ref + # resolve to the SAME COMMIT this job emits ONE leg, not two, and says so. On a plugin that is + # up to date with the engine (the steady state) this is free. + refs: + name: resolve the busbar refs this plugin must be proven against + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.build.outputs.matrix }} + steps: + - name: Checkout the plugin (for .busbar-ref only) + uses: actions/checkout@v7 + with: + path: plugin + + # The sibling busbar checkout is what OWNS the decision logic, in a script with a --selftest, + # rather than inline YAML nobody can run. `dev` deliberately, not the leg refs being resolved: + # this step is what decides those, so it cannot depend on them. + - name: Checkout busbar (for the resolver script) + uses: actions/checkout@v7 + with: + repository: GetBusbar/busbar + ref: dev + path: busbarAI + + - name: Ref resolver SELF-TEST (prove the decisions before trusting the verdict) + # A gate nobody watched fail is not a gate. This covers the dedupe, the hard failure on an + # unresolvable `.busbar-ref` pin, and the `/merge` pull_request fallback, offline. + run: bash busbarAI/scripts/plugin-ci-refs.sh --selftest + + - name: Build the ref matrix + id: build + run: | + bash busbarAI/scripts/plugin-ci-refs.sh \ + --plugin-root plugin \ + --moving-ref "${{ inputs.busbar_ref }}" >> "$GITHUB_OUTPUT" + + build-test-signoff: - name: build, test, clippy, fmt, signoff + name: build, test, clippy, fmt, signoff [busbar ${{ matrix.label }}] + needs: refs + strategy: + # fail-fast off ON PURPOSE. When the pinned leg and the moving leg disagree, the single most + # useful fact is WHICH ONE broke: pinned-red means the release is broken now, moving-red means + # the engine has moved out from under this plugin and the next re-pin will break it. Cancelling + # the sibling leg on the first failure destroys exactly that signal. + fail-fast: false + matrix: ${{ fromJSON(needs.refs.outputs.matrix) }} runs-on: ubuntu-latest services: + # SERVICE IMAGES ARE DIGEST-PINNED. A floating tag means a third party can change what this + # workflow tests against, overnight, with no commit here. That matters more in THIS file than + # anywhere else in the fleet: every first-party plugin repo calls it, so one upstream retag + # moves all of them at once, and the resulting breakage looks like the plugin's own tests + # failing. ci.yml pinned its own postgres and valkey containers; these are the same two images + # and reuse the SAME digests, so core and the plugin fleet cannot drift apart. + # + # `hashicorp/vault` was worse than unpinned: it carried NO tag at all, so it resolved to + # `latest`. It is pinned by digest alone, with no tag, and that digest is what `latest` + # resolved to at pin time, so this is behaviour-preserving rather than a version bump. Moving + # vault to a newer explicit version is a real change to what the secret-backend tests run + # against and belongs in its own commit, with a run to back it. + # + # wiremock and openldap already carried explicit VERSION tags rather than `latest`, which is + # most of the value, but a version tag can still be retagged upstream. They are digest-pinned + # too, at the digest their existing tag resolves to now, so no image in this file can move + # without a commit. postgres: - image: ${{ inputs.service == 'postgres' && 'postgres:16' || '' }} + image: ${{ inputs.service == 'postgres' && 'postgres:16@sha256:95206741a5b214807675e14165369d05b93a9cf692223b616d07cca227e74b0b' || '' }} env: POSTGRES_USER: busbar POSTGRES_PASSWORD: busbar @@ -78,7 +162,7 @@ jobs: options: >- ${{ inputs.service == 'postgres' && '--health-cmd "pg_isready -U busbar" --health-interval 10s --health-timeout 5s --health-retries 5' || '' }} valkey: - image: ${{ inputs.service == 'valkey' && 'valkey/valkey:8' || '' }} + image: ${{ inputs.service == 'valkey' && 'valkey/valkey:8@sha256:495e4fecdc98ee48a20b207726caa5ab6451e0fac3642a9be10d9e70b3068df6' || '' }} ports: - 6379:6379 # Runs INSIDE the container (it has its own valkey-cli — the runner does not); GitHub @@ -91,7 +175,7 @@ jobs: # once the container's healthy, rather than a boot flag — GitHub's `services:` block has no # supported way to pass mysqld command-line args, only `docker run` options. mysql: - image: ${{ inputs.service == 'mysql' && 'mysql:8' || '' }} + image: ${{ inputs.service == 'mysql' && 'mysql:8@sha256:b3b90af2a6552ae30c266fdb7d5dd55f3afb72404bb78d37fe8a23eb857fd3fb' || '' }} env: MYSQL_ROOT_PASSWORD: busbar MYSQL_USER: busbar @@ -105,7 +189,7 @@ jobs: # BUSBAR_TEST_VAULT_TOKEN, set on the Test steps below) — same hard-fail-under-CI-if-unset # discipline the plugin's own tests apply, mirroring busbarAI's own ci.yml vault service block. vault: - image: ${{ inputs.service == 'vault' && 'hashicorp/vault' || '' }} + image: ${{ inputs.service == 'vault' && 'hashicorp/vault@sha256:5be49781ecf78bfe775c5309c6a4d9f4e9e040b6c885c99eb2b12fb69855e1a2' || '' }} env: VAULT_DEV_ROOT_TOKEN_ID: ${{ inputs.service == 'vault' && 'root' || '' }} ports: @@ -118,7 +202,7 @@ jobs: # nothing here mounts a fixture file (service containers start before checkout). Exposed to the # test via BUSBAR_TEST_WIREMOCK_URL on the cargo test step below. wiremock: - image: ${{ inputs.service == 'wiremock' && 'wiremock/wiremock:3.9.2' || '' }} + image: ${{ inputs.service == 'wiremock' && 'wiremock/wiremock:3.9.2@sha256:d13997cd7b52583528a766019cfe7d4e91c4d224a67bdaa6f60efbb532f32176' || '' }} ports: - 8080:8080 # Ready-made OpenLDAP — the directory auth-ldap binds against for its live e2e. PINNED. The @@ -126,7 +210,7 @@ jobs: # tests/e2e.rs then seeds the test user/group over LDAP itself (ldap3) and retries the admin bind # until the container is ready. Exposed via BUSBAR_TEST_LDAP_URL on the cargo test step below. openldap: - image: ${{ inputs.service == 'openldap' && 'osixia/openldap:1.5.0' || '' }} + image: ${{ inputs.service == 'openldap' && 'osixia/openldap:1.5.0@sha256:18742e9c449c9c1afe129d3f2f3ee15fb34cc43e5f940a20f3399728f41d7c28' || '' }} env: LDAP_ORGANISATION: ${{ inputs.service == 'openldap' && 'Example Org' || '' }} LDAP_DOMAIN: ${{ inputs.service == 'openldap' && 'example.org' || '' }} @@ -139,17 +223,44 @@ jobs: with: path: plugin + # The ref was resolved to a COMMIT SHA by the `refs` job above (which also owns the + # `/merge` fallback and the pinned-vs-moving decision). Checking out a sha rather than + # a branch name also removes a race: the two matrix legs cannot silently pick up different + # commits of the same branch if busbar is pushed mid-run. + - name: Announce which busbar this leg proves the plugin against + run: echo "This leg builds against busbar ${{ matrix.ref }} -- ${{ matrix.label }}." + - name: Checkout busbar (sibling path dependency) uses: actions/checkout@v7 with: repository: GetBusbar/busbar - ref: ${{ inputs.busbar_ref }} + ref: ${{ matrix.ref }} path: busbarAI - uses: dtolnay/rust-toolchain@stable with: components: clippy, rustfmt - uses: Swatinem/rust-cache@v2 + with: + # `workspaces:` IS REQUIRED HERE and its absence was a silent, total cache miss. The action + # defaults to a single workspace at `$GITHUB_WORKSPACE`, and NOTHING is checked out there: + # both checkouts above use `path:`, so the two Cargo workspaces live at `plugin/` and + # `busbarAI/`. With no `workspaces:` the action looked for a Cargo.toml at the checkout + # root, did not find one, printed "could not find Cargo.toml", and carried on WITHOUT + # FAILING -- so every plugin repo rebuilt busbar and its whole dependency graph from + # scratch on every single run while the log said the cache step succeeded. + # + # Both entries are needed, not just the plugin: this workflow builds busbar-plugin-pack and + # a full `cargo build --release --bin busbar` out of busbarAI/, which is by far the larger + # of the two. + # + # The key must also separate the matrix legs. Two legs build DIFFERENT busbar commits into + # the same `busbarAI/target`; sharing one cache entry between them means each leg restores + # the other's artifacts and immediately invalidates them, which is worse than no cache. + workspaces: | + plugin + busbarAI + key: busbar-${{ matrix.ref }} - name: PUBLIC-HYGIENE GATE — nothing THIS repo publishes may describe how it was built # INHERITED BY EVERY PLUGIN REPO, deliberately, and that inheritance is the whole point: the @@ -208,16 +319,27 @@ jobs: BUSBAR_TEST_LDAP_URL: ${{ inputs.service == 'openldap' && 'ldap://127.0.0.1:389' || '' }} run: cargo test - - name: cargo test (sibling busbarAI checkout — this backend's own monorepo integration test) - if: inputs.extra_sibling_test_command != '' - working-directory: busbarAI - env: - BUSBAR_TEST_POSTGRES_URL: ${{ inputs.service == 'postgres' && 'postgres://busbar:busbar@localhost:5432/busbar_test' || '' }} - VALKEY_URL: ${{ inputs.service == 'valkey' && 'redis://localhost:6379/0' || '' }} - BUSBAR_TEST_VAULT_ADDR: ${{ inputs.service == 'vault' && 'http://127.0.0.1:8200' || '' }} - BUSBAR_TEST_VAULT_TOKEN: ${{ inputs.service == 'vault' && 'root' || '' }} - BUSBAR_TEST_MYSQL_URL: ${{ inputs.service == 'mysql' && 'mysql://busbar:busbar@127.0.0.1:3306/busbar_test' || '' }} - run: ${{ inputs.extra_sibling_test_command }} + # REMOVED, and deliberately not replaced by a skippable equivalent: a step named + # `cargo test (sibling busbarAI checkout — this backend's own monorepo integration test)` + # used to sit here behind `if: inputs.extra_sibling_test_command != ''`. NO CALLER IN THE + # FLEET EVER PASSED THAT INPUT, so on every run of every plugin repo the step reported + # `skipped` while the job reported `success` — a leg that read like the one place a plugin was + # exercised against the engine, and had never run once. That is the same "a skipped step reads + # as a passed step" shape the A2A conformance leg cost this release, and a gate nobody can + # tell apart from coverage is worse than no gate. + # + # It is gone rather than armed because there is nothing left for it to run: it existed to call + # `cargo test -p busbar-store-` against the monorepo's own copy of a store crate, and + # those crates no longer live in busbarAI — each backend's real logic is now in its own repo, + # covered by the `cargo test` step ABOVE (which cannot skip). The engine-side integration + # coverage this step pretended to give is what the FILE-DROP, INSTALL-AND-SERVE and + # EXECUTABLE-CONFIG steps below actually do — a real busbar boot with the real plugin — plus + # each store repo's own over-the-ABI e2e, whose `plugin_path()` PANICS under CI rather than + # skipping when the cdylib is missing. + # + # A repo that genuinely needs a second, service-backed job should declare it as its own job, + # the way store-valkey's `migrate-destructive-wipe-test` does: visible, named, and impossible + # to confuse with a skip. - name: Build busbar-plugin-pack (from the sibling checkout) working-directory: busbarAI diff --git a/.github/workflows/plugin-consumer-verify.yml b/.github/workflows/plugin-consumer-verify.yml new file mode 100644 index 00000000..6d6a5b7c --- /dev/null +++ b/.github/workflows/plugin-consumer-verify.yml @@ -0,0 +1,552 @@ +name: plugin-consumer-verify + +# plugin-consumer-verify - the ONE consumer-side check every first-party plugin repo calls, the way +# they all already call plugin-ci.yml. It answers exactly one question per repo: +# +# DOES THE THING WE PUBLISHED ACTUALLY WORK WHEN A USER GETS IT? +# +# WHY THE FLEET NEEDED THIS. Not one plugin repo verified what it published. Every repo's release +# workflow ends with an assertion about ITS OWN upload step, from inside the run that did the +# uploading, which proves the run believed it succeeded and nothing else. Two failures came straight +# out of that gap: +# +# * headroom-hook's SHIPPED docker/bundle/config.yaml carries config shapes busbar 1.5.3 retired +# (`auth.admin_auth:` with INLINE module entries, which moved under `identity-providers:`). The +# published bundle therefore CANNOT BOOT: busbar exits 1 with "this looks like a busbar 1.x +# config" before it ever binds a port. The image built fine, pushed fine, and every workflow +# involved was green, because "it built" and "it runs" are different claims and only the first +# one was ever checked. +# * webrequest-hook v1.0.4 published as a zero-asset phantom. The tag exists, the Release object +# exists, and there is nothing in it to download. +# +# Both are the same shape and neither needs a clever check to catch. It needs SOMEBODY TO ACTUALLY +# FETCH THE PUBLISHED THING AND USE IT, from outside, after publication. +# +# THIS IS DELIBERATELY SMALLER THAN CORE'S verify-deploy.yml. A plugin has no install.sh, no +# Homebrew tap, no helm chart, no marketing site. It has a Release with tarballs in it, and +# sometimes a container image. So the whole check is: the Release is real and public, every platform +# archive it owes is present and downloadable through the moving `latest` pointer, one of those +# archives unpacks into a plugin busbar would actually accept, and - where the repo ships a runnable +# bundle - the published image BOOTS AND SERVES rather than merely existing. +# +# HOW TO ADOPT IT, in the calling repo's .github/workflows/consumer-verify.yml: +# +# name: consumer-verify +# on: +# release: { types: [published] } +# schedule: [{ cron: "41 9 * * *" }] +# workflow_dispatch: { inputs: { version: { required: false, type: string } } } +# permissions: { contents: read, issues: write, actions: read } +# jobs: +# verify: +# uses: GetBusbar/busbar/.github/workflows/plugin-consumer-verify.yml@main +# with: +# asset_prefix: busbar-store-postgres +# plugin_name: busbar-store-postgres-plugin +# plugin_alias: postgres +# plugin_kind: store +# +# and, in the repo's OWN release.yml, as the final job so a broken publish turns the RELEASE red: +# +# consumer-verification: +# needs: [verify-assets] +# if: ${{ !cancelled() }} +# uses: GetBusbar/busbar/.github/workflows/plugin-consumer-verify.yml@main +# with: { version: ${{ github.ref_name }}, asset_prefix: ..., ... } +# permissions: { contents: read, issues: write, actions: read } +# secrets: inherit + +on: + workflow_call: + inputs: + asset_prefix: + description: >- + The release asset basename before the version, i.e. the pack --out prefix. The published + asset is --.tar.gz. NOTE it is not always the same as + plugin_name: the store repos drop the trailing -plugin from the filename and the auth + repos keep it, so this is an input rather than something derived. + required: true + type: string + plugin_name: + description: "The manifest `name` the tarball must declare (pack --name), e.g. busbar-store-postgres-plugin" + required: true + type: string + plugin_alias: + description: "The manifest `alias` (pack --alias), e.g. postgres" + required: true + type: string + plugin_kind: + description: "store | auth | hook | secret - the manifest `kind` (pack --kind)" + required: true + type: string + version: + description: >- + Version to verify, with or without a leading v. Leave empty and the newest published + release of the CALLING repo is used, which is what the daily schedule wants. + required: false + type: string + default: "" + targets: + description: >- + Space-separated platform triples the release owes an archive for. The default is the + 5-target matrix every plugin repo's release.yml builds. Windows ships .zip in core but + .tar.gz here, because plugin-pack always writes a tarball. + required: false + type: string + default: "x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu x86_64-apple-darwin aarch64-apple-darwin x86_64-pc-windows-msvc" + require_signed: + description: >- + Assert the published manifest carries a signature. Default true, and it should stay true: + every plugin release workflow falls back to --allow-unsigned when BUSBAR_SIGN_KEY is + unset, so an unsigned publish looks identical to a signed one from inside the release run + and is refused by busbar at load time on the user's machine. Set false ONLY with a comment + saying why, so the exemption is a visible decision rather than a silent gap. + required: false + type: boolean + default: true + bundle_image: + description: >- + Docker repository of a runnable bundle this repo publishes, e.g. getbusbar/busbar-headroom. + Empty means the repo ships no bundle and the boot check is declared not-applicable rather + than silently skipped. When set, the image is pulled FRESH, its :latest is checked against + the version pin by manifest digest, and the container must BOOT AND SERVE. + required: false + type: string + default: "" + bundle_env: + description: >- + Space-separated KEY=VALUE pairs passed to `docker run -e` for the bundle boot. The bundle's + shipped config decides what is required; headroom's needs ANTHROPIC_KEY and + BUSBAR_ADMIN_TOKEN. Dummy values are correct here: this asserts BOOT and health, and + deliberately spends no real provider call. + required: false + type: string + default: "" + bundle_health_path: + description: "Path the booted bundle must serve" + required: false + type: string + default: "/healthz" + bundle_health_body: + description: "Exact body the health path must return" + required: false + type: string + default: "ok" + +permissions: + contents: read + issues: write + actions: read + +jobs: + consumer: + name: consumer check (fetch what we published and use it) + runs-on: ubuntu-latest + timeout-minutes: 25 + outputs: + version: ${{ steps.check.outputs.version }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + ASSET_PREFIX: ${{ inputs.asset_prefix }} + PLUGIN_NAME: ${{ inputs.plugin_name }} + PLUGIN_ALIAS: ${{ inputs.plugin_alias }} + PLUGIN_KIND: ${{ inputs.plugin_kind }} + IN_VERSION: ${{ inputs.version }} + TARGETS: ${{ inputs.targets }} + REQUIRE_SIGNED: ${{ inputs.require_signed }} + BUNDLE_IMAGE: ${{ inputs.bundle_image }} + BUNDLE_ENV: ${{ inputs.bundle_env }} + HEALTH_PATH: ${{ inputs.bundle_health_path }} + HEALTH_BODY: ${{ inputs.bundle_health_body }} + steps: + # No actions/checkout, on purpose and for the same reason core's verify-deploy.yml has none: + # this job must not be able to read the repository. Every byte it judges is fetched from the + # published Release or pulled from the registry, because a fix that is committed but never + # published is still broken for every user, and a working copy is exactly how that gets hidden. + - name: Does what we published actually work? + id: check + run: | + # `set +e` FIRST, AND IT IS LOAD-BEARING. GitHub runs every `run:` block as + # `bash -e {0}`, so errexit is ALREADY ON before line 1 and `set -uo pipefail` does not + # turn it off. Without this the job aborts at the first non-zero command, and "the bundle + # does not boot" would hide "the bundle does not boot AND two archives are missing". + set +e + set -uo pipefail + fail=0 + : > /tmp/findings.md + + record() { # record + echo "FAIL: $1 | expected: $2 | observed: $3" + { + echo "- **$1**" + echo " - expected: \`$2\`" + echo " - observed: \`$3\`" + echo " - $4" + } >> /tmp/findings.md + echo "::error::CONSUMER CHECK FAILED: $1 - expected '$2', observed '$3'. $4" + fail=1 + } + declared() { echo "NOT APPLICABLE: $1 - $2"; } + + # -- Which version -------------------------------------------------------------------- + V="${IN_VERSION#v}" + if [ -z "$V" ]; then + # From the RELEASES list, not from /releases/latest, which is one of the pointers this + # job is about to test. Asking a pointer what the answer is and then checking the + # pointer against its own answer is a check that can never fail. + V="$(gh api --paginate "repos/${REPO}/releases" \ + --jq '.[] | select(.draft==false and .prerelease==false) | .tag_name' 2>/dev/null \ + | sed 's/^v//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1)" + fi + if [ -z "${V:-}" ]; then + echo "::error::${REPO} has no published, non-draft release to verify. Either nothing has shipped yet, or - the failure this exists to catch - the release was created as a DRAFT and never promoted, which means it is invisible to every user while looking perfectly fine on the Actions tab." + exit 1 + fi + TAG="v$V" + echo "version=$V" >> "$GITHUB_OUTPUT" + echo "Verifying what ${REPO} published as ${TAG}, from the outside." + + # -- C1: the Release is REAL and PUBLIC ----------------------------------------------- + # A draft Release is the failure mode two repos in the fleet are one bug away from: + # headroom-hook and webrequest-hook both create the Release with --draft and rely on their + # verify-assets job running `gh release edit --draft=false` afterwards. If that job is + # skipped (a `needs:` on a failed job skips its dependent by DEFAULT, which is exactly how + # busbar's own verify-assets got skipped precisely when the release was broken), the tag + # exists, the run may even be green, and users see nothing at all. + meta="$(gh api "repos/${REPO}/releases/tags/${TAG}" 2>/dev/null || true)" + if [ -z "$meta" ]; then + record "GitHub Release ${TAG}" "a published Release" "" \ + "The tag was pushed but no Release object is visible. Fix: check the release workflow's create-release job." + echo "$fail" >/dev/null + else + is_draft="$(printf '%s' "$meta" | jq -r '.draft')" + if [ "$is_draft" = "true" ]; then + record "GitHub Release ${TAG} draft flag" "false (published)" "true (still a DRAFT)" \ + "A draft Release is INVISIBLE to users: nothing downloads, and \`gh release download\` fails. The release workflow creates it with --draft and promotes it in verify-assets, so verify-assets was skipped or failed. Fix: \`gh release edit ${TAG} --repo ${REPO} --draft=false --latest\`, then fix the promote step." + else + echo "PASS: Release ${TAG} exists and is published (not a draft)" + fi + fi + + # -- C2: the moving `latest` pointer resolves to it ----------------------------------- + # Users and tooling that do not pin a version follow /releases/latest. It 404s if the + # newest Release is not flagged latest and silently serves the PREVIOUS release otherwise. + loc="$(curl -fsS --max-time 30 -o /dev/null -w '%{redirect_url}' \ + "https://github.com/${REPO}/releases/latest" || true)" + if [ "${loc##*/releases/tag/}" = "$TAG" ]; then + echo "PASS: github.com/${REPO}/releases/latest -> ${TAG}" + else + record "${REPO} /releases/latest" ".../releases/tag/${TAG}" "${loc:-}" \ + "Anyone fetching this plugin without pinning a version gets a different release than ${TAG}. Fix: flag ${TAG} as latest (\`gh release edit ${TAG} --latest\`)." + fi + + # -- C3: every platform archive is present, sized and DOWNLOADABLE -------------------- + # THE PHANTOM GUARD, and the reason it is not a count. webrequest-hook v1.0.4 published + # with zero assets; a `>= 1` assertion would have caught that one but passes happily for a + # release missing four of five platforms, which is the busbar-1.5.3 shape one level down. + # A count can never see a MISSING PLATFORM. Only a name can. And a name in the asset list + # is still not a usable artifact: GitHub creates the row when the upload STARTS, so a + # truncated upload lists identically to a good one. Assert name, size and real bytes. + listed="$(printf '%s' "$meta" | jq -r '.assets[]? | "\(.name)\t\(.size)"' 2>/dev/null || true)" + if [ -z "$listed" ]; then + record "Release ${TAG} assets" "one archive per platform in: ${TARGETS}" "ZERO assets" \ + "PHANTOM RELEASE: the tag and the Release object exist and there is nothing in them to download. This is webrequest-hook v1.0.4 verbatim. Fix: every leg of the release build matrix failed to upload - check the cdylib build and the sibling busbar checkout pin in .busbar-ref." + fi + first_archive="" + for t in $TARGETS; do + a="${ASSET_PREFIX}-${V}-${t}.tar.gz" + sz="$(printf '%s\n' "$listed" | awk -F'\t' -v n="$a" '$1==n{print $2; exit}')" + if [ -z "${sz:-}" ]; then + record "Release asset ${a}" "present on ${TAG}" "" \ + "Users on ${t} get a 404. Fix: that target's leg of the release build matrix did not upload - it runs with fail-fast:false, so the other platforms succeeding tells you nothing about this one." + continue + fi + # A packed plugin is a compressed cdylib: hundreds of KB at the very least. 50 KiB is far + # below any real one and far above a truncated or header-only upload. + if [ "$sz" -lt 51200 ]; then + record "Release asset ${a} size" ">= 51200 bytes" "${sz} bytes" \ + "A truncated upload: it is listed as present and is useless to anyone who downloads it. Fix: re-upload the asset." + continue + fi + # Through /releases/latest/download/, not the pinned tag URL: that is the version-agnostic + # path, so this proves the moving pointer AND the asset in one request. + u="https://github.com/${REPO}/releases/latest/download/${a}" + code="$(curl -sSL --max-time 60 --range 0-0 -o /dev/null -w '%{http_code}' "$u" || echo 000)" + case "$code" in + 200|206) + echo "PASS: ${a} present (${sz} bytes) and downloadable through /releases/latest/download/" + [ -z "$first_archive" ] && first_archive="$a" + ;; + *) record "/releases/latest/download/${a}" "HTTP 200/206" "HTTP ${code}" \ + "The asset is listed on the Release but downloading it fails, so a user following the documented URL gets nothing. Fix: re-upload it." ;; + esac + done + + # -- C4: the archive is a plugin busbar would actually accept ------------------------- + # An archive that downloads is not an archive that LOADS. busbar refuses a plugin whose + # manifest is absent, whose declared sha256 does not bind the cdylib beside it, or (under + # default trust) that carries no signature - and every one of those is invisible from + # inside the release run that produced it. + if [ -n "$first_archive" ]; then + work="$(mktemp -d)" + if curl -fsSL --max-time 120 \ + "https://github.com/${REPO}/releases/latest/download/${first_archive}" \ + -o "${work}/p.tar.gz" && tar xzf "${work}/p.tar.gz" -C "$work" 2>/dev/null; then + if [ ! -s "${work}/manifest.json" ]; then + record "${first_archive} contents" "a manifest.json at the archive root" "" \ + "busbar rejects the plugin at load time with no manifest. Fix: the pack step produced a malformed archive." + else + py_out="$(python3 - "$work" "$PLUGIN_NAME" "$PLUGIN_ALIAS" "$PLUGIN_KIND" "$V" "$REQUIRE_SIGNED" <<'PY' + import glob, hashlib, json, os, sys + work, want_name, want_alias, want_kind, want_ver, require_signed = sys.argv[1:7] + m = json.load(open(os.path.join(work, "manifest.json"))) + out = [] + def chk(field, got, want): + out.append(("ok" if got == want else "bad", "manifest %s" % field, str(want), str(got))) + chk("name", m.get("name"), want_name) + chk("alias", m.get("alias"), want_alias) + chk("kind", m.get("kind"), want_kind) + chk("version", m.get("version"), want_ver) + libs = [p for p in glob.glob(os.path.join(work, "*")) + if os.path.basename(p) not in ("manifest.json", "p.tar.gz")] + if not libs: + out.append(("bad", "cdylib in the archive", "one shared library beside manifest.json", "none")) + else: + b = open(libs[0], "rb").read() + chk("sha256 binding to %s" % os.path.basename(libs[0]), + hashlib.sha256(b).hexdigest(), m.get("sha256")) + # The magic byte check is cheap and catches the whole family of "we packed the wrong + # file": a build script that packed a .d, an empty stub, or a text error message. + magic = "ELF" if b[:4] == b"\x7fELF" else ("Mach-O" if b[:4] in (b"\xcf\xfa\xed\xfe", b"\xce\xfa\xed\xfe") else ("PE" if b[:2] == b"MZ" else "not a shared library")) + out.append(("ok" if magic != "not a shared library" else "bad", + "packed file is a real shared library", "ELF/Mach-O/PE", magic)) + if require_signed == "true": + out.append(("ok" if m.get("signature") else "bad", "manifest signature", "a non-empty signature", m.get("signature") or "")) + for verdict, what, want, got in out: + print("\t".join((verdict, what, want, got))) + PY + )" || py_out="" + if [ -z "$py_out" ]; then + record "${first_archive} manifest" "a readable manifest.json" "" \ + "The published archive's manifest could not be read at all. Fix: re-run the pack step." + fi + while IFS=$'\t' read -r verdict what want got; do + [ -n "${verdict:-}" ] || continue + if [ "$verdict" = "ok" ]; then + echo "PASS: ${what} == ${got}" + else + record "${first_archive}: ${what}" "$want" "$got" \ + "busbar refuses to load this plugin on the user's machine. A signature is empty when BUSBAR_SIGN_KEY was unset and the pack step silently fell back to --allow-unsigned; a sha256 mismatch means the archive was rebuilt after signing. Fix: provision BUSBAR_SIGN_KEY in this repo and re-cut." + fi + done <<< "$py_out" + fi + else + record "${first_archive} download+extract" "a valid .tar.gz" "" \ + "The published bytes are not a readable gzip archive. Fix: re-upload the asset." + fi + rm -rf "$work" + fi + + # -- C5: THE BUNDLE MUST BOOT AND SERVE ----------------------------------------------- + # "It built" and "it runs" are different claims, and only the first was ever checked + # anywhere in this fleet. headroom-hook's published bundle proves the gap is not theoretical: + # its shipped docker/bundle/config.yaml still uses `auth.admin_auth:` with inline module + # entries, retired in busbar 1.5.3, so the container exits 1 during config load with "this + # looks like a busbar 1.x config" and never binds a port. The image builds, pushes, and + # every workflow involved goes green. + if [ -z "$BUNDLE_IMAGE" ]; then + declared "runnable bundle boot" "this repo publishes no container bundle (bundle_image is empty), so there is no image to boot. If it grows one, set bundle_image and this check starts applying automatically." + else + # DELETE THE LOCAL COPIES FIRST. `docker pull` is a no-op against a tag the daemon + # already has and `docker run`/`inspect` then use the LOCAL image, which reports a stale + # bundle as fresh. Nothing below may be allowed to answer from cache. + docker rmi -f "${BUNDLE_IMAGE}:${V}" "${BUNDLE_IMAGE}:latest" >/dev/null 2>&1 || true + if ! docker pull -q "${BUNDLE_IMAGE}:${V}" >/dev/null 2>&1; then + record "${BUNDLE_IMAGE}:${V}" "a pullable image" "" \ + "The bundle for ${TAG} was never pushed, so \`docker run ${BUNDLE_IMAGE}\` cannot give anyone this version. Fix: re-run the bundle build workflow for ${TAG}." + else + # `:latest` must be the same image, by DIGEST. A bundle whose :latest never moved hands + # every unpinned user the previous release, silently and indefinitely. + d_ver="$(docker image inspect "${BUNDLE_IMAGE}:${V}" --format '{{index .RepoDigests 0}}' 2>/dev/null | sed 's/.*@//')" + d_latest="" + docker pull -q "${BUNDLE_IMAGE}:latest" >/dev/null 2>&1 \ + && d_latest="$(docker image inspect "${BUNDLE_IMAGE}:latest" --format '{{index .RepoDigests 0}}' 2>/dev/null | sed 's/.*@//')" + if [ -n "$d_ver" ] && [ "$d_ver" = "$d_latest" ]; then + echo "PASS: ${BUNDLE_IMAGE}:latest == :${V} (${d_ver})" + else + record "${BUNDLE_IMAGE}:latest" "${d_ver:-} (the :${V} digest)" "${d_latest:-}" \ + "\`docker run ${BUNDLE_IMAGE}\` with no tag - what the docs show - serves a different release than ${TAG}. Fix: the bundle build must tag and push \`latest\` as well as the version; docker/metadata-action does NOT imply latest from a semver pattern." + fi + + # THE PORT MUST BE PROVEN FREE BEFORE THE PROBE, and this is not paranoia: probing a + # port that something else already answers on returns a cheerful 200 from the wrong + # process while the container under test is dead in the water. That happened while this + # check was being written and briefly reported a bundle as healthy that had exited 1. + port=18723 + if curl -s -m 3 -o /dev/null "http://127.0.0.1:${port}${HEALTH_PATH}" 2>/dev/null; then + echo "::error::port ${port} on the runner is already answering before the container starts, so a health probe against it would prove nothing. Refusing to report a possibly-false PASS." + fail=1 + else + envargs="" + for kv in $BUNDLE_ENV; do envargs="${envargs} -e ${kv}"; done + docker rm -f busbar-bundle-verify >/dev/null 2>&1 || true + # shellcheck disable=SC2086 + docker run -d --name busbar-bundle-verify -p "${port}:8080" $envargs \ + "${BUNDLE_IMAGE}:${V}" >/dev/null 2>&1 || true + body="" + for _ in $(seq 1 30); do + # A container that EXITED will never answer, so stop waiting the moment it dies + # rather than burning the full 60s and then reporting a timeout, which reads like a + # slow boot instead of a refusal to boot. + st="$(docker inspect -f '{{.State.Status}}' busbar-bundle-verify 2>/dev/null || echo missing)" + [ "$st" = "exited" ] || [ "$st" = "missing" ] && break + body="$(curl -fsS -m 3 "http://127.0.0.1:${port}${HEALTH_PATH}" 2>/dev/null || true)" + [ "$body" = "$HEALTH_BODY" ] && break + sleep 2 + done + st="$(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}}' busbar-bundle-verify 2>/dev/null || echo 'missing exit=?')" + if [ "$body" = "$HEALTH_BODY" ]; then + echo "PASS: ${BUNDLE_IMAGE}:${V} boots and serves '${HEALTH_BODY}' on ${HEALTH_PATH}" + else + echo "--- container log ---" + docker logs busbar-bundle-verify 2>&1 | head -40 || true + echo "---------------------" + record "${BUNDLE_IMAGE}:${V} boot" "serves '${HEALTH_BODY}' on ${HEALTH_PATH}" "container ${st}, body '${body:-}'" \ + "THE PUBLISHED BUNDLE DOES NOT RUN. A user who does \`docker run ${BUNDLE_IMAGE}\` gets a container that dies. The log is above; the known cause in this fleet is the bundle's own shipped config.yaml using config shapes a newer busbar retired, in which case busbar exits 1 with 'this looks like a busbar 1.x config' before binding a port. Fix: run \`busbar --migrate-config\` over docker/bundle/config.yaml, and pin .busbar-ref to the engine the bundle actually embeds." + fi + docker rm -f busbar-bundle-verify >/dev/null 2>&1 || true + fi + fi + fi + + { + echo "### Consumer check for \`${TAG}\`" + echo + if [ "$fail" = 0 ]; then + echo "What ${REPO} published as \`${TAG}\` downloads, unpacks into a plugin busbar accepts, and (where a bundle is published) boots and serves." + else + echo "**What ${REPO} published as \`${TAG}\` does NOT work for a user:**" + echo + cat /tmp/findings.md + fi + } >> "$GITHUB_STEP_SUMMARY" + + [ "$fail" = 0 ] + + # Same alert shape as core's verify-deploy.yml, and for the same reason: a red run in a repo + # nobody is watching is not a signal. Because this is a reusable workflow, `github.repository` is + # the CALLING repo, so the issue lands where the broken release lives. Idempotent by label: one + # open issue at a time, retitled and rewritten on every later failure, so a daily schedule updates + # it instead of filing thirty. + alert: + name: alert (open or update the release-broken issue) + needs: consumer + if: ${{ always() && needs.consumer.result == 'failure' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + actions: read + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + VERSION: ${{ needs.consumer.outputs.version }} + LABEL: consumer-verification + steps: + - name: Open or update the single open release-broken issue + run: | + set +e # GitHub runs this as `bash -e {0}`; errexit is on before line 1. + set -uo pipefail + V="${VERSION:-unknown}" + TITLE="[release-broken] Published v${V} does not work for a consumer" + + # Read the findings out of the FAILED JOB'S OWN LOG rather than restating them. The job + # already printed `FAIL: | expected: | observed: ` and an `::error::` line + # naming the fix; re-deriving that here would be a second statement of the same fact, free + # to drift from the first, which is the exact defect shape this whole workflow exists to + # catch. Job logs are available over the API as soon as the JOB finishes, which it has. + : > /tmp/findings.md + jid="$(gh api "repos/${REPO}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \ + --jq '[.jobs[]? | select(.conclusion=="failure")][0].id' 2>/dev/null || true)" + if [ -n "${jid:-}" ] && gh api "repos/${REPO}/actions/jobs/${jid}/logs" > /tmp/job.log 2>/dev/null; then + sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' /tmp/job.log \ + | grep -E '^(FAIL:|::error::)' | sed -E 's/^::error::/DIAGNOSIS: /' \ + | head -40 | sed 's/^/ /' >> /tmp/findings.md + fi + [ -s /tmp/findings.md ] || echo " (the failed job's log was not retrievable over the API; open the run and read it there)" >> /tmp/findings.md + + { + echo "" + echo + echo "**What this repo published does not work when a user gets it.** Opened and" + echo "maintained automatically by \`GetBusbar/busbar/.github/workflows/plugin-consumer-verify.yml\`," + echo "which downloads the published artifact from the Release and uses it - it does not" + echo "read this repository, so a fix that is committed but not published still fails here." + echo + echo "This is **not a flaky test**. Every check reads what is published." + echo + echo "| | |" + echo "| --- | --- |" + echo "| version under test | \`v${V}\` |" + echo "| failing run | ${RUN_URL} |" + echo "| last checked | $(date -u '+%Y-%m-%d %H:%M UTC') |" + echo + echo "## What failed" + echo + cat /tmp/findings.md + echo + echo "## What to do" + echo + echo "Read the \`DIAGNOSIS:\` lines - each names the user-visible symptom and the fix." + echo "Fix it at the source (the release workflow, the bundle's shipped config), then" + echo "re-run the check. This issue is reused, not duplicated, and closes itself when a" + echo "run passes." + } > /tmp/issue-body.md + + gh label create "$LABEL" --repo "$REPO" --color B60205 \ + --description "A published release is broken for consumers (auto-filed)" >/dev/null 2>&1 || true + + existing="$(gh issue list --repo "$REPO" --label "$LABEL" --state open \ + --limit 1 --json number --jq '.[0].number // empty' 2>/dev/null || true)" + if [ -n "${existing:-}" ]; then + gh issue edit "$existing" --repo "$REPO" --title "$TITLE" --body-file /tmp/issue-body.md + gh issue comment "$existing" --repo "$REPO" \ + --body "Still failing as of $(date -u '+%Y-%m-%d %H:%M UTC') - ${RUN_URL}" + echo "::error::Consumer verification is FAILING for v${V}. Details in ${GITHUB_SERVER_URL}/${REPO}/issues/${existing}" + else + num="$(gh issue create --repo "$REPO" --title "$TITLE" --label "$LABEL" \ + --body-file /tmp/issue-body.md 2>/dev/null | tail -1)" + echo "::error::Consumer verification is FAILING for v${V}. Opened ${num}" + fi + + # Without this the issue is opened once and lives forever, and a stale open "release is broken" + # issue is worse than none: the next real breakage looks like the old one and gets ignored. + resolved: + name: close the release-broken issue when the check passes + needs: consumer + if: ${{ always() && needs.consumer.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + issues: write + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + LABEL: consumer-verification + steps: + - name: Close any open release-broken issue + run: | + set +e # GitHub runs this as `bash -e {0}`; errexit is on before line 1. + set -uo pipefail + existing="$(gh issue list --repo "$REPO" --label "$LABEL" --state open \ + --limit 1 --json number --jq '.[0].number // empty' 2>/dev/null || true)" + [ -n "${existing:-}" ] || { echo "No open ${LABEL} issue. Nothing to close."; exit 0; } + gh issue comment "$existing" --repo "$REPO" \ + --body "Consumer verification is GREEN again as of $(date -u '+%Y-%m-%d %H:%M UTC'): the published release downloads, unpacks into a plugin busbar accepts, and boots where a bundle is published. Closing. Run: ${RUN_URL}" + gh issue close "$existing" --repo "$REPO" --reason completed + echo "Closed #${existing}." diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index f5de71f8..cd1b0bbf 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -65,8 +65,20 @@ jobs: - name: Sanity — build + the drift/emit gates pass before we tag run: | + set -euo pipefail cargo build -p busbar --quiet - cargo test -p busbar --features openapi-schema openapi_json_matches_committed_file --quiet + # FLOOR ON THE MATCH COUNT. `cargo test ` is a SUBSTRING filter, and a filter that + # matches NOTHING prints "running 0 tests / test result: ok" and EXITS 0. Rename or move + # `openapi_json_matches_committed_file` and this step — the last gate before the release is + # tagged — goes green having run no test at all, and we tag a stale committed openapi.json + # that the runtime then serves. Asserting the test actually ran is the whole point. + out="$(cargo test -p busbar --features openapi-schema \ + openapi_json_matches_committed_file 2>&1 | tee /dev/stderr)" + echo "$out" | grep -qE 'test result: ok\. [1-9][0-9]* passed' || { + echo "::error::the OpenAPI drift test did not run (renamed, moved, or filtered to zero)." + echo "::error::A zero-match 'cargo test' exits 0 — this is NOT a pass. Fix the filter." + exit 1 + } - name: Commit + push to dev (NO tag — tag-on-main.yml cuts the release when this reaches main) run: | diff --git a/.github/workflows/qa-gate.yml b/.github/workflows/qa-gate.yml index 3f97f5e2..0fe208b9 100644 --- a/.github/workflows/qa-gate.yml +++ b/.github/workflows/qa-gate.yml @@ -10,7 +10,7 @@ # # BRANCH MODEL: `dev` gets only the cheap per-push CI (ci.yml) — push there often. Promoting # dev→`qa` is what spends this real-money full-plugin gate; it is the pre-release soak. A green qa -# is what earns a promotion qa→`main`, where tag-on-main.yml auto-cuts the release. This +# is what earns a promotion qa→`main`, where release.yml cuts the release. This # intentionally does NOT run on `dev`, `main`, or PRs — `scripts/release-check.sh` documents itself # as a pre-release gate, not a per-commit one. `qa` is where that cost belongs: proving the promoted # commit is release-ready before it ever reaches `main`. @@ -111,6 +111,19 @@ concurrency: env: BUILD_ARTIFACT: qa-gate-target-${{ github.run_id }} TARBALL: /tmp/busbar-target.tzst + # A PHASE THAT DID NOT RUN IS NOT A PHASE THAT PASSED. release-check.sh records `sibling-missing` + # when a sibling checkout is absent and used to end with an unconditional "RELEASE GATE PASSED" + # banner anyway, so a green qa-gate did not prove those phases executed. The clone step above + # warns and continues on failure, which is what makes this reachable rather than theoretical: one + # failed clone silently removed a repo from the gate. busbar-admin is the sharpest case, because + # its integration.sh is the ONLY cross-repo behavioural check on the widest mirror of busbar's + # wire shapes in the fleet. + # + # THE POLICY LIVES IN `scripts/qa-gate-run.sh`, NOT HERE, and deliberately so. `workflow_run` + # loads this file from the DEFAULT branch, so anything written here gates every commit except + # the one that introduced it — a qa-gate improvement could not gate the release that shipped it. + # `qa-gate-dispatch-lint.py` fails the build on exactly that shape. The script rides the commit, + # so `BUSBAR_RELEASE_CHECK_REQUIRE_SIBLINGS` is exported there with the reasoning intact. jobs: # ── fast: cheap tier + the matrix that drives the fan-out. Sibling of `build`, never in front of diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8e92ff7a..bf66b232 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,28 +1,379 @@ name: Release -# On a version tag (e.g. v0.9.0): create the GitHub Release, then cross-compile the busbar -# binary for major targets and attach tarballs to that Release. +# NOTHING IS TAGGED UNTIL A CONSUMER HAS PROVEN IT WORKS. +# +# THE OLD ORDER, AND WHAT IT COST. `tag-on-main.yml` read the version out of Cargo.toml the moment a +# commit landed on `main` and pushed `vX.Y.Z`. That tag push fired this workflow AND docker.yml in +# parallel, so the version name existed in public before one byte had been built. Everything after +# that was recovery, not prevention. v1.5.3 shipped exactly that way: the tag went up, the GitHub +# Release published with FIVE of seven assets because two build legs died, `install.sh` 404'd on +# Apple Silicon, and putting it right meant deleting the release AND the tag and re-cutting. +# Meanwhile docker.yml never produced `getbusbar/busbar:1.5.3` at all while the docs told users to +# pin it. +# +# AND THE CONSTRAINT THAT MAKES THIS NECESSARY RATHER THAN MERELY TIDIER. Docker Hub has TAG +# IMMUTABILITY enabled on getbusbar/busbar. A published tag CANNOT be overwritten. So a broken +# `1.5.3` is PERMANENT, and the remedies are deleting the tag -- which has previously required owner +# scope that was not available -- or burning a version number. Under the old order there was no +# point at which a bad release could be abandoned quietly, because the name was minted first. +# +# THE ORDER NOW: +# +# plan read the version from Cargo.toml. Refuse if that version is ALREADY released +# anywhere (git tag, non-draft release, or a Docker Hub tag we could never +# overwrite). Mint a throwaway name, `staging-`. +# gate the full suite, on the exact commit being released. +# draft create the GitHub Release as a DRAFT. Draft assets have real, downloadable URLs +# -- they can be fetched and EXECUTED -- but a draft does not resolve as +# `releases/latest`, is not on the releases page, and creates NO git tag. +# build+attach binaries, SBOM, OpenAPI, provenance, all onto the draft. +# stage-image docker.yml pushes the multi-arch image under `staging-` and NOTHING else. +# Really pullable, really runnable, and no user is looking at that name. +# verify-assets the draft owes every asset the platform manifest names. +# verify-staged THE GATE. verify-deploy.yml in `staging` mode against those real artifacts: +# `docker rmi` then pull FRESH, boot the container, download and EXECUTE the +# release binary, read `--version`, run the documented quickstart, prove the +# attestation verifies, prove every expected asset really downloads. +# promote-image ONLY ON GREEN. Manifest-only retag of the exact staged digest to `X.Y.Z`, then +# `latest`, on both registries. No rebuild, so the promoted image is the one that +# was verified rather than a second build that ought to match. +# promote-release push the git tag, flip the draft to published + latest, then RE-DERIVE all of it +# from outside and fail loud if any one name did not move. +# notify / +# consumer-verification the fan-out and the full public-facing sweep, unchanged. +# +# WHAT A FAILURE LEAVES BEHIND, WHICH IS THE WHOLE POINT. If anything up to and including +# `verify-staged` fails: no git tag exists, no release is listed (the draft is invisible and +# deletable), no `X.Y.Z` container tag was ever minted so nothing immutable was burned, no downstream +# repo was notified, and `latest` still points where it did. The next attempt is a CLEAN RETRY, not +# a recovery. That is the safe direction, and it is the direction the old order could not offer. +# +# WHAT IT COSTS, STATED HONESTLY. A release becomes visible later -- the staged verification adds +# roughly 15 to 25 minutes between "the artifacts exist" and "users can see them", on top of a +# pipeline already dominated by the two-arch PGO builds. And the promote is one more step that can +# fail. Both are priced in deliberately: a slow release is an inconvenience, an immutable broken tag +# is permanent. +# +# ONE MECHANISM OWNS TAGGING. `tag-on-main.yml` is GONE; its version read and its idempotency guard +# live in `plan` below. docker.yml has no tag trigger and no `type=semver` line. Every user-facing +# name -- the git tag, the GitHub Release, `X.Y.Z` and `latest` on both registries -- is created by +# the two promote jobs at the bottom of this file and nowhere else. on: + # LANDING ON `main` IS STILL THE RELEASE TRIGGER. What changed is that it no longer goes through a + # tag: this workflow runs on the branch push, and the tag is an OUTPUT of a successful run rather + # than its input. push: - tags: - - "v*" - - "!v*-*" # NEVER publish a pre-release tag (e.g. v1.5.2-rc.1) — those are qa staging markers, - # queryable across repos, that must not cut a real release. Only final vX.Y.Z publishes. + branches: [main] + # Manual re-run of the same commit after fixing whatever went red. Safe because `plan` refuses a + # version that is already released and every step below is idempotent against a re-run. + workflow_dispatch: + +# Serialize. A burst of pushes to main must not put two runs into the promote phase for the same +# version. (This replaces tag-on-main.yml's `concurrency: tag-on-main`.) +concurrency: + group: release + cancel-in-progress: false permissions: - contents: write # create the Release + upload assets + contents: write # draft the Release, upload assets, push the version tag on promote + packages: write # docker.yml (called below) pushes to GHCR id-token: write # OIDC identity for keyless Sigstore signing (provenance) attestations: write # record the build-provenance attestation + actions: read + issues: write jobs: + # -- PLAN: what version is this, and is it safe to mint that name at all? ------------------------- + # + # The version comes from crates/busbar/Cargo.toml, exactly as tag-on-main.yml read it, so the + # release-cutting ritual is unchanged for a human: bump on dev via prepare-release.yml, promote + # dev -> qa -> main, and landing on main cuts the release. + # + # IDEMPOTENT BY DESIGN, same as before: if the version is already released this is a NO-OP, not a + # failure. Pushing docs to main without bumping the version must never re-release, and must never + # go red either, or the red stops meaning anything. + # + # THE PRE-FLIGHT THAT IS NEW, AND IT IS THE IMMUTABILITY GUARD. It also asks Docker Hub whether + # `getbusbar/busbar:X.Y.Z` already exists. That tag cannot be overwritten, so discovering it at + # PROMOTE time -- after a full build and a full verification -- would mean a run that has done + # everything right and still cannot finish, with a half-promoted state to unpick. Asking here + # costs one unauthenticated HEAD request and turns that into a clean refusal before anything is + # built. + plan: + name: plan (version, staging name, is this name still free) + runs-on: ubuntu-latest + outputs: + version: ${{ steps.p.outputs.version }} + tag: ${{ steps.p.outputs.tag }} + staging_tag: ${{ steps.p.outputs.staging_tag }} + sha: ${{ steps.p.outputs.sha }} + release: ${{ steps.p.outputs.release }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 # so `git rev-parse vX.Y.Z` can see every existing tag + - id: p + shell: bash + run: | + set -euo pipefail + V="$(python3 -c "import tomllib; print(tomllib.load(open('crates/busbar/Cargo.toml','rb'))['package']['version'])")" + echo "$V" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' \ + || { echo "::error::crates/busbar/Cargo.toml version '$V' is not X.Y.Z"; exit 1; } + SHA="${GITHUB_SHA}" + { + echo "version=$V" + echo "tag=v$V" + echo "sha=$SHA" + # The throwaway name. Derived from the commit, so a re-run of the SAME commit reuses the + # SAME staging tag and overwrites it rather than littering the registry, while two + # different commits can never collide. `staging-` prefixed so it is unmistakable on the + # tag list and unmistakably not something to pin. + echo "staging_tag=staging-${SHA:0:12}" + } >> "$GITHUB_OUTPUT" + + if git rev-parse "v$V" >/dev/null 2>&1; then + echo "::notice::tag v$V already exists - nothing to cut (safe no-op). Bump the version on dev via prepare-release.yml to cut a new release." + echo "release=0" >> "$GITHUB_OUTPUT"; exit 0 + fi + # A PUBLISHED (non-draft) release for this version is the same "already cut" signal even + # if the git tag was deleted by hand during a recovery. + state="$(gh release view "v$V" --repo "$GITHUB_REPOSITORY" --json isDraft --jq '.isDraft' 2>/dev/null || echo none)" + if [ "$state" = "false" ]; then + echo "::notice::Release v$V is already published - nothing to cut (safe no-op)." + echo "release=0" >> "$GITHUB_OUTPUT"; exit 0 + fi + [ "$state" = "true" ] && echo "::notice::A DRAFT release for v$V already exists; this run will reuse and re-fill it." + + # IMMUTABILITY PRE-FLIGHT. Anonymous pull token -> HEAD the manifest. A 200 means the tag + # is taken, permanently, and this version can never be published correctly. + tok="$(curl -fsS --max-time 30 'https://auth.docker.io/token?service=registry.docker.io&scope=repository:getbusbar/busbar:pull' | jq -r '.token // .access_token' || true)" + if [ -n "${tok:-}" ] && [ "$tok" != "null" ]; then + code="$(curl -sS --max-time 30 -o /dev/null -w '%{http_code}' -I \ + -H "Authorization: Bearer $tok" \ + -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.index.v1+json' \ + "https://registry-1.docker.io/v2/getbusbar/busbar/manifests/$V" || echo 000)" + if [ "$code" = "200" ]; then + echo "::error::getbusbar/busbar:${V} ALREADY EXISTS on Docker Hub, and Docker Hub tag immutability means it can never be overwritten. Refusing to start a release that could not finish. Fix: bump to the next patch version on dev (prepare-release.yml) and promote that, or have an owner delete the tag first." + exit 1 + fi + echo "Docker Hub tag ${V} is free (HTTP ${code} on HEAD manifest)." + else + echo "::warning::could not obtain an anonymous Docker Hub pull token; skipping the tag-immutability pre-flight. The promote step will still refuse to overwrite an existing tag, just later." + fi + echo "release=1" >> "$GITHUB_OUTPUT" + echo "::notice::Cutting v$V from ${SHA:0:7}. Staging name: staging-${SHA:0:12}. Nothing public until verification is green." + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # -- GATE ZERO: NOTHING IS RELEASED FROM A RED BRANCH ------------------------------------------- + # + # Owner, 2026-08-08: "nothing should ever be released red", and "or ignored". Both halves are + # rules, and the second one is the harder one. + # + # WHAT MAKES THIS NECESSARY. GetBusbar/headroom-hook cut and published v2.0.5 with its CI RED. The + # red was `public-hygiene-lint`, unrelated to the change, and it was sitting on a list under the + # heading "KNOWN, TRACKED, NOT BLOCKING". That heading is institutionalised ignoring: a red that + # sits on a list is still ignored, only with paperwork attached. Every red found across the fleet + # that night had been permitted for weeks by exactly that reasoning. + # + # RED IS RED. Not "red for an unrelated reason". Not "red before my change". Not "red on a check + # that does not matter". If someone believes a particular red does not matter, that is a decision + # about the CHECK, made in daylight by changing or deleting the check. It is not a decision to be + # taken silently at release time by whoever happens to be cutting. + # + # THERE IS NO BYPASS. No override input, no force flag, no waiver, no exception list, no + # allowed-to-fail set. That absence is the feature: a waiver IS the permission-to-ignore + # mechanism, and permission-to-ignore is what shipped a red release. If a future edit to this file + # starts to look like an escape hatch "just in case", the escape hatch is the thing being banned. + # + # UNKNOWN IS RED. If the status lookup errors, times out, is rate-limited, or simply cannot find a + # CI run for this commit, this job REFUSES. It never degrades to a pass. A check that cannot + # determine its answer and reports success is the `sibling-missing` defect, which has already + # cost this project a silently-unverified mirror. + # + # THE ACCEPTED COST, STATED PLAINLY. With no waiver, a red we do not control stops the release: a + # third-party outage, a rate limit, a flaky check. That is accepted. The correct response to a + # check that blocks releases for reasons unrelated to the software is to HARDEN IT OR DELETE IT, + # never to mute it. Under this rule a fragile check is a release-stopper, which finally makes its + # fragility cost something to the people who can fix it. + # + # WHY IT WAITS RATHER THAN SAMPLES. This workflow and ci.yml both fire on the same push to main, + # so at the instant this job starts, CI is IN PROGRESS. Sampling "is anything red yet" would pass + # every time, on every commit, forever - a gate that is green because it looked too early is worse + # than no gate, because it reads as evidence. So it waits for every workflow run on this commit to + # reach a conclusion, and a wait that runs out is a REFUSAL. + branch-green: + name: gate 0 (refuse to release from a red commit) + needs: plan + if: ${{ needs.plan.outputs.release == '1' }} + runs-on: ubuntu-latest + # Bounded well above a normal CI run (the `check` job with its Postgres and Valkey service + # containers is the long pole) and well below anything that would sit here all day. Running out + # is a refusal, not a pass, so a generous bound costs waiting and never correctness. + timeout-minutes: 90 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SHA: ${{ needs.plan.outputs.sha }} + # Workflows that must have RUN and CONCLUDED on this commit. An empty check set is the + # vacuous-pass trap: no runs found reads as "nothing is red". These names are asserted + # PRESENT, so a commit that CI never saw is refused rather than waved through. + REQUIRED_WORKFLOWS: "CI" + # WORKFLOWS THAT ARE NOT CHECKS OF THIS COMMIT, AND THIS IS NOT AN EXCEPTION LIST. + # + # THE ONLY ARGUMENT FOR EITHER NAME BELOW IS THAT IT DOES NOT ANSWER THIS GATE'S QUESTION. + # `branch-green` asks ONE thing: is this COMMIT's code green. Neither of these reports on the + # commit, so including them makes the gate incoherent rather than stricter. Read this list as + # covering exactly that and nothing more. + # + # Release is THIS run. A job cannot wait for the run it is part of, and a re-run must + # not be failed by an earlier attempt's conclusion. + # + # Verify deploy verifies the LIVE, PUBLISHED world: Docker Hub's `latest`, the Homebrew tap, + # the Helm chart, getbusbar.com, the site's counters. Every way it can carry + # this commit's SHA is incoherent as a pre-publication gate, and there are two: + # + # * `release: published` and `workflow_run` fire only AFTER this release + # publishes. On a first run no such run exists, so requiring it is either + # vacuous or a deadlock; on a re-run it is a report on the state of + # production at the previous attempt. + # * the daily and 3-hourly `schedule` runs carry the DEFAULT BRANCH's head + # SHA, which on release day is this very commit -- while the version they + # resolve and verify comes from the live `/releases/latest` redirect, i.e. + # the PREVIOUS release. Requiring that green would gate release N on the + # channel health of release N-1, attributed to release N's commit. It is + # not a weaker check of this commit; it is a check of something else + # wearing this commit's SHA. + # + # NOTHING ABOUT A CURRENT PRODUCTION RED JUSTIFIES THIS ENTRY, AND THAT IS SAID EXPLICITLY + # BECAUSE AN EARLIER DRAFT OF THIS CHANGE LEANED ON ONE. It cited verify-deploy's check (n), + # the live-counters assertion, as permanently red until the marketing counts Worker was + # deployed. That was true when written and is not true now: GetBusbar/marketing#2 fixed the + # Worker's hourly refresh (a Docker Hub login failure thrown outside any try block aborted the + # whole handler), put both Workers on deploy-from-main, and added an hourly live guard so + # staleness has its own red. Check (n) was re-run verbatim against production on 2026-08-08 + # and PASSES: pulls 27187 hub vs 27174 served (delta 13, tolerance 271) and stars 106 vs 106 + # (delta 0, tolerance 2). So that half of the reasoning is struck, not merely outdated, and + # this entry stands on the structural argument above ALONE. "A check is currently red" must + # never be a reason to stop asking it -- that is the permission-to-ignore mechanism this whole + # gate exists to remove. + # + # ITS VERDICT IS NOT IGNORED, IT IS TAKEN AT THE POINT WHERE IT MEANS SOMETHING. verify-deploy + # runs inside this release graph twice, both times against the artifact actually being cut: + # as `verify-staged`, which GATES the promote, and as `consumer-verification`, which turns + # this run red and opens a labelled issue. + NOT_COMMIT_CHECKS: "Release,Verify deploy" + steps: + - name: Every check on this commit must have concluded, and every conclusion must be green + shell: bash + run: | + set -uo pipefail # deliberately NOT -e: an API hiccup must be RETRIED, then refused. + + # THE API IS THE ONE THING HERE THAT CAN FAIL FOR REASONS UNRELATED TO THE SOFTWARE, so it + # is retried properly rather than trusted once. Bounded retries, then refusal: a lookup + # that never answered has not told us the branch is green. + api() { # api -> body on stdout, non-zero if it could not be read + local path="$1" out rc + for attempt in 1 2 3 4 5; do + out="$(gh api --paginate "$path" 2>/tmp/api.err)"; rc=$? + if [ "$rc" = 0 ] && [ -n "$out" ]; then printf '%s' "$out"; return 0; fi + echo " api read of ${path} failed (attempt ${attempt}/5): $(tr '\n' ' ' < /tmp/api.err)" >&2 + sleep $(( attempt * 10 )) + done + return 1 + } + + deadline=$(( SECONDS + 75 * 60 )) + while :; do + runs="$(api "repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${SHA}&per_page=100")" + if [ $? -ne 0 ]; then + echo "::error::REFUSING TO RELEASE: the GitHub Actions API could not be read after 5 attempts, so the CI status of ${SHA} is UNKNOWN. Unknown is not green. Nothing has been built, no tag exists and nothing is public. Re-run this workflow when the API is answering." >&2 + exit 1 + fi + # Exclude THIS run (it is in progress by definition) and every other run of THIS + # workflow (a re-run of the release must not deadlock waiting on itself or judge itself + # by an older attempt's conclusion). + echo "$runs" | jq -s --arg skip "$NOT_COMMIT_CHECKS" ' + ($skip | split(",")) as $not + | [.[] | .workflow_runs[]?] + | map(select(.id != (env.GITHUB_RUN_ID | tonumber) and (.name | IN($not[]) | not))) + | unique_by(.name) + | map({name, status, conclusion, url: .html_url})' > /tmp/runs.json + total="$(jq 'length' /tmp/runs.json)" + pending="$(jq '[.[] | select(.status != "completed")] | length' /tmp/runs.json)" + echo "checks on ${SHA:0:7}: ${total} workflow(s), ${pending} still running" + jq -r '.[] | " \(.name): \(.status)/\(.conclusion // "-")"' /tmp/runs.json + [ "$pending" = 0 ] && break + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::REFUSING TO RELEASE: ${pending} check(s) on ${SHA} were still running after 75 minutes, so their result is UNKNOWN. Unknown is not green. Nothing has been built, no tag exists and nothing is public. Fix whatever is hanging and re-run." >&2 + jq -r '.[] | select(.status != "completed") | " STILL RUNNING: \(.name) \(.url)"' /tmp/runs.json >&2 + exit 1 + fi + sleep 30 + done + + fail=0 + + # EVERY REQUIRED WORKFLOW MUST BE PRESENT. Without this the whole gate is vacuous on a + # commit no workflow ever ran against: zero runs, zero red runs, green. + for w in $REQUIRED_WORKFLOWS; do + if [ "$(jq --arg w "$w" '[.[] | select(.name == $w)] | length' /tmp/runs.json)" = 0 ]; then + echo "::error::REFUSING TO RELEASE: no '${w}' run exists for ${SHA}, so this commit's test status is UNKNOWN. A commit CI never saw is not a green commit. Nothing has been built, no tag exists and nothing is public." + fail=1 + fi + done + + # `skipped` and `neutral` are green: a workflow that legitimately did not apply to this + # commit (path filters, an `if:` that was false) has not failed. `cancelled`, `failure`, + # `timed_out`, `action_required` and `stale` are all RED, and `stale` especially so: it + # means GitHub never ran the check, which is the unknown case wearing a conclusion. + red="$(jq -r '.[] | select(.conclusion != "success" and .conclusion != "skipped" and .conclusion != "neutral") | " RED: \(.name) -> \(.conclusion // "no conclusion") \(.url)"' /tmp/runs.json)" + if [ -n "$red" ]; then + echo "$red" + names="$(jq -r '[.[] | select(.conclusion != "success" and .conclusion != "skipped" and .conclusion != "neutral") | .name] | join(", ")' /tmp/runs.json)" + echo "::error::REFUSING TO RELEASE FROM A RED COMMIT. ${SHA:0:7} is red on: ${names}. There is no override, no waiver and no exception list, and that is deliberate: 'unrelated to my change' and 'known, tracked, not blocking' are how a red release shipped. If one of these checks should not block a release, change or delete the CHECK - do not bypass it here. Nothing has been built, no tag exists, no release is listed and nothing was fanned out." + fail=1 + fi + + # External commit statuses (anything reporting through the statuses API rather than as a + # workflow run) get the same treatment, including the same unknown-is-red rule. + st="$(api "repos/${GITHUB_REPOSITORY}/commits/${SHA}/status")" + if [ $? -ne 0 ]; then + echo "::error::REFUSING TO RELEASE: the commit-status API could not be read after 5 attempts, so external check state on ${SHA} is UNKNOWN. Unknown is not green." + fail=1 + else + state="$(echo "$st" | jq -r -s '.[0].state // "unknown"')" + count="$(echo "$st" | jq -r -s '.[0].statuses | length')" + echo "external commit statuses: ${count} (combined state: ${state})" + if [ "$count" -gt 0 ] && [ "$state" != "success" ]; then + echo "$st" | jq -r -s '.[0].statuses[] | select(.state != "success") | " RED: \(.context) -> \(.state) \(.target_url)"' + echo "::error::REFUSING TO RELEASE FROM A RED COMMIT: external commit status on ${SHA} is '${state}'. Nothing has been built and nothing is public." + fail=1 + fi + fi + + { + echo "### Gate 0: is ${SHA:0:7} green?" + echo + echo "| check | status | conclusion |" + echo "| --- | --- | --- |" + jq -r '.[] | "| \(.name) | \(.status) | \(.conclusion // "-") |"' /tmp/runs.json + } >> "$GITHUB_STEP_SUMMARY" + + [ "$fail" = 0 ] || exit 1 + echo "::notice::${SHA:0:7} is green across every check. Proceeding to build." + # ── TEST GATE ──────────────────────────────────────────────────────────────────────────────── - # Nothing is built, signed, attested or published until the suite passes ON THE TAGGED COMMIT. + # Nothing is built, signed, attested or published until the suite passes ON THE COMMIT BEING + # RELEASED. # - # `ci.yml` is armed on pushes to main/dev/qa and on PRs — NOT on tags. So a `v*` tag ran this - # workflow with no test job anywhere in the graph, and the build-provenance attestation would - # faithfully certify the provenance of an artifact whose suite never ran in that run. Test gating - # depended entirely on someone having pushed the same commit to a branch first and on that run - # having been green, which is a convention, not a gate. + # `ci.yml` is armed on pushes to main/dev/qa and on PRs. This job existed because the release used + # to run off a TAG push, which ci.yml does not see, so test gating depended entirely on someone + # having pushed the same commit to a branch first and on that run having been green -- a + # convention, not a gate. The release now runs on the main push itself, so ci.yml does see this + # commit; the gate stays anyway, and deliberately. A separate run being green is still a fact + # about a DIFFERENT run that nothing in this graph depends on, and the build-provenance + # attestation below would otherwise certify an artifact whose suite never ran in the run that + # attested it. # # Mirrors `ci.yml`'s `check` job, including the live Postgres/Valkey service containers: without # them the store roundtrip tests skip, and `CI` being set makes them hard-fail rather than skip @@ -30,10 +381,39 @@ jobs: # would still be wrong to omit them, since that coverage is exactly what a release should prove. gate: name: gate (fmt · clippy · build · test) + # `branch-green` FIRST. Nothing is built for a release from a red commit, so the expensive part + # of the pipeline never starts on one either. + needs: [plan, branch-green] runs-on: ubuntu-latest services: + # PINNED BY DIGEST, NOT BY TAG. `postgres:16` and `valkey/valkey:8` are MOVING tags: Docker Hub + # re-points them at every patch, so the same commit gets different bytes on different days and + # a store-roundtrip failure could be caused by a database nobody in this repository changed. + # These service containers are load-bearing (without them the Postgres/Valkey roundtrip tests + # hard-fail rather than skip, which is deliberate), so what they resolve to has to be a fact + # about this commit, not about the calendar. + # + # WHAT A DIGEST PIN DOES NOT FIX, SAID PLAINLY SO NOBODY THINKS THE HOLE IS CLOSED: it does + # nothing about Docker Hub RATE LIMITS. The pull still goes to Docker Hub and an anonymous + # runner still gets the anonymous quota. The pin buys reproducibility; authentication is what + # buys quota. See the `credentials:` block below (release.yml) and the note in ci.yml. + # + # RE-PINNING: `docker buildx imagetools inspect postgres:16` prints the current digest. These + # are ordinary dependencies and should move on the same cadence as the rest -- if they go + # stale enough to matter, re-pin in the monthly-refresh PR. postgres: - image: postgres:16 + image: postgres:16@sha256:95206741a5b214807675e14165369d05b93a9cf692223b616d07cca227e74b0b + # AUTHENTICATED PULL, WHICH IS THE PART THAT ACTUALLY ADDRESSES RATE LIMITS. Docker Hub's + # anonymous quota is per source IP and GitHub's runner IPs are shared, so an anonymous pull + # can be throttled for reasons that have nothing to do with busbar -- and under the + # no-waiver rule that would be a red nobody can clear by fixing the software. These are the + # same credentials docker.yml already uses. Safe to reference unconditionally HERE because + # this workflow only triggers on a push to `main` and on workflow_dispatch, never on + # `pull_request`, so the secrets are always present; ci.yml cannot do the same without + # breaking fork PRs, where secrets are withheld. + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} env: POSTGRES_USER: busbar POSTGRES_PASSWORD: busbar @@ -46,7 +426,18 @@ jobs: --health-timeout 5s --health-retries 5 valkey: - image: valkey/valkey:8 + image: valkey/valkey:8@sha256:495e4fecdc98ee48a20b207726caa5ab6451e0fac3642a9be10d9e70b3068df6 + # AUTHENTICATED PULL, WHICH IS THE PART THAT ACTUALLY ADDRESSES RATE LIMITS. Docker Hub's + # anonymous quota is per source IP and GitHub's runner IPs are shared, so an anonymous pull + # can be throttled for reasons that have nothing to do with busbar -- and under the + # no-waiver rule that would be a red nobody can clear by fixing the software. These are the + # same credentials docker.yml already uses. Safe to reference unconditionally HERE because + # this workflow only triggers on a push to `main` and on workflow_dispatch, never on + # `pull_request`, so the secrets are always present; ci.yml cannot do the same without + # breaking fork PRs, where secrets are withheld. + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} ports: - 6379:6379 options: >- @@ -74,31 +465,101 @@ jobs: VALKEY_URL: redis://localhost:6379 run: cargo test --workspace --locked --verbose - # Create the Release first so the parallel upload jobs have something to attach to - # (uploading from a matrix without a pre-existing release races → "release not found"). - create-release: - needs: gate + # -- THE DRAFT ------------------------------------------------------------------------------------ + # + # Created first so the parallel upload jobs have something to attach to (uploading from a matrix + # without a pre-existing release races -> "release not found"). + # + # `--draft` IS THE LOAD-BEARING FLAG, AND IT WAS VERIFIED END TO END ON A THROWAWAY REPO BEFORE + # THE EIGHT PLUGIN REPOS TOOK IT: a draft 404s from `releases/tags/`, does NOT resolve as + # `releases/latest`, and is absent from the public releases list -- yet `gh release upload` and + # `gh release view` still find it BY TAG, its assets have real downloadable URLs through the API, + # and `gh release edit --draft=false --latest` promotes it in place. So the artifacts are real + # enough to fetch and execute while the version is not yet a thing anyone can find. Core is the + # last repo to adopt this, not the pioneer. + # + # `--target` REPLACES `--verify-tag`. There IS no tag yet -- that is the point -- so the draft is + # anchored to the commit instead, and GitHub materialises `vX.Y.Z` at that commit when the draft + # is published. `promote-release` pushes the tag explicitly first anyway, so the two agree. + draft: + name: draft the release (invisible, no tag, real assets) + needs: [plan, gate] runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - name: Create GitHub Release + # THE BODY NAMES WHAT CHANGED. `--generate-notes` alone produced a one-line compare link, so + # v1.5.3's published body did not mention that `--validate` now RESOLVES `env:`/`file:` secret + # references and exits 1 when one cannot resolve (boot behaviour unchanged). An operator + # deciding whether to take a config-affecting release could not learn that from the release. + # The CHANGELOG section for the version being cut is already written, already reviewed, and + # already the canonical answer, so the release body is now that section followed by the + # auto-generated commit/contributor notes rather than the notes alone. + # + # FAIL-SOFT, DELIBERATELY: a missing or unparseable CHANGELOG section must never block a + # release that is otherwise good. It degrades to the previous behaviour and says so in the log. + - name: Extract this version's CHANGELOG section for the release body + id: notes + env: + V: ${{ needs.plan.outputs.version }} + run: | + set -euo pipefail + python3 - "$V" > /tmp/relnotes.md <<'PY' + import re, sys + version = sys.argv[1] + try: + text = open("CHANGELOG.md", encoding="utf-8").read() + except OSError: + sys.exit(0) + # Headings look like `## [1.5.3], 2026-08-08`. Capture through to the next `## `. + m = re.search( + r"^##\s*\[%s\][^\n]*\n(.*?)(?=^##\s|\Z)" % re.escape(version), + text, re.M | re.S) + if m: + sys.stdout.write(m.group(1).strip() + "\n") + PY + if [ -s /tmp/relnotes.md ]; then + echo "found=1" >> "$GITHUB_OUTPUT" + echo "CHANGELOG section for ${V} ($(wc -l < /tmp/relnotes.md) lines) will lead the release body." + else + echo "found=0" >> "$GITHUB_OUTPUT" + echo "::warning::No CHANGELOG section found for ${V}; falling back to generated notes only." \ + "The release will still publish, but its body will not say what changed." + fi + - name: Create the DRAFT GitHub Release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ needs.plan.outputs.tag }} + SHA: ${{ needs.plan.outputs.sha }} run: | - gh release create "${GITHUB_REF_NAME}" \ - --repo "${GITHUB_REPOSITORY}" \ - --title "busbar ${GITHUB_REF_NAME}" \ - --verify-tag --generate-notes \ - || gh release view "${GITHUB_REF_NAME}" --repo "${GITHUB_REPOSITORY}" + set -euo pipefail + # Idempotent: a re-run of the same commit reuses the existing draft rather than failing or + # creating a second one. A NON-draft release here means the version is already out, which + # `plan` should have caught; refuse rather than silently re-upload over a live release. + state="$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isDraft --jq '.isDraft' 2>/dev/null || echo none)" + case "$state" in + true) echo "Reusing the existing DRAFT ${TAG}."; exit 0 ;; + false) echo "::error::${TAG} is already PUBLISHED. Refusing to re-upload assets onto a live release."; exit 1 ;; + esac + notes_args=(--generate-notes) + [ "${{ steps.notes.outputs.found }}" = "1" ] && notes_args+=(--notes-file /tmp/relnotes.md) + gh release create "$TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --title "busbar ${TAG}" \ + --target "$SHA" \ + --draft \ + "${notes_args[@]}" + echo "::notice::${TAG} drafted at ${SHA:0:7}. It is NOT listed, does NOT resolve as releases/latest, and NO git tag exists." # Generate a CycloneDX Software Bill of Materials (every dependency + version + - # license) and attach it to the Release. Lets downstream users answer "is the + # license) and attach it to the draft. Lets downstream users answer "is the # crate in advisory X inside busbar v1.0.1?" without decompiling, and satisfies # enterprise/government (EO 14028) procurement that increasingly requires an SBOM. sbom: - needs: create-release + needs: [plan, draft] name: sbom (cyclonedx) runs-on: ubuntu-latest + env: + TAG: ${{ needs.plan.outputs.tag }} steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable @@ -106,16 +567,15 @@ jobs: - name: Install cargo-cyclonedx run: cargo install cargo-cyclonedx --locked - name: Generate SBOM - run: cargo cyclonedx --format json --override-filename "busbar-${GITHUB_REF_NAME}.cdx" - - name: Attach SBOM to Release + run: cargo cyclonedx --format json --override-filename "busbar-${TAG}.cdx" + - name: Attach SBOM to the draft env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | # cargo-cyclonedx writes the SBOM next to the package's Cargo.toml. - sbom="$(find . -name "busbar-${GITHUB_REF_NAME}.cdx.json" -print -quit)" + sbom="$(find . -name "busbar-${TAG}.cdx.json" -print -quit)" test -n "$sbom" || { echo "SBOM not found"; exit 1; } - gh release upload "${GITHUB_REF_NAME}" "$sbom" \ - --repo "${GITHUB_REPOSITORY}" --clobber + gh release upload "$TAG" "$sbom" --repo "$GITHUB_REPOSITORY" --clobber # Publish the admin API's OpenAPI 3.1 document as a release asset. The gateway generates this doc # from its typed route contract (and serves it live at GET /api/v1/admin/openapi.json), but that @@ -124,9 +584,11 @@ jobs: # release-over-release WITHOUT running busbar. The document is emitted by the `emit_openapi_artifact` # test — the same `openapi_doc()` the gateway serves — so publishing needs no user-facing CLI flag. openapi: - needs: create-release + needs: [plan, draft] name: openapi (3.1 document) runs-on: ubuntu-latest + env: + TAG: ${{ needs.plan.outputs.tag }} steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable @@ -137,141 +599,124 @@ jobs: # typed one; it equals the committed `openapi.json` the runtime serves (drift-locked by the # `openapi_json_matches_committed_file` test in ci.yml). env: - BUSBAR_EMIT_OPENAPI: ${{ github.workspace }}/busbar-openapi-${{ github.ref_name }}.json + BUSBAR_EMIT_OPENAPI: ${{ github.workspace }}/busbar-openapi-${{ needs.plan.outputs.tag }}.json run: cargo test -p busbar --bin busbar --features openapi-schema emit_openapi_artifact -- --nocapture - name: Validate it is a well-formed OpenAPI 3.1 document run: | - doc="${GITHUB_WORKSPACE}/busbar-openapi-${GITHUB_REF_NAME}.json" + doc="${GITHUB_WORKSPACE}/busbar-openapi-${TAG}.json" test -s "$doc" || { echo "OpenAPI doc is empty"; exit 1; } jq -e '.openapi | startswith("3.1")' "$doc" > /dev/null \ || { echo "not an OpenAPI 3.1 document"; exit 1; } - - name: Attach OpenAPI document to Release + - name: Attach OpenAPI document to the draft env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh release upload "${GITHUB_REF_NAME}" "${GITHUB_WORKSPACE}/busbar-openapi-${GITHUB_REF_NAME}.json" \ - --repo "${GITHUB_REPOSITORY}" --clobber + gh release upload "$TAG" "${GITHUB_WORKSPACE}/busbar-openapi-${TAG}.json" \ + --repo "$GITHUB_REPOSITORY" --clobber - upload-assets: - needs: create-release - name: ${{ matrix.target }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - # `pgo: true` routes a HOST-NATIVE target through scripts/pgo-build.sh (PGO is MANDATORY for - # release: the same fail-closed build docker.yml ships). PGO needs the instrumented binary - # to run ON THE RUNNER to train, so only the targets whose arch matches the runner's are - # PGO'd: x86_64-linux on the x86_64 ubuntu runner, and aarch64-darwin on the arm64 macos - # runner. The cross targets (aarch64-linux built on an x86_64 host, x86_64-darwin on an arm64 - # host) and windows cannot self-train, so they keep the plain --release build. - include: - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - pgo: true - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - - target: x86_64-apple-darwin - os: macos-latest - - target: aarch64-apple-darwin - os: macos-latest - pgo: true - - target: x86_64-pc-windows-msvc - os: windows-latest + # THE PLATFORM LIST, IN EXACTLY ONE PLACE. This job emits the build matrix AND the exact set of + # asset filenames that matrix is contractually obliged to produce, from one literal. `upload-assets` + # consumes the first; `verify-assets` consumes the second. + # + # WHY IT IS A JOB AND NOT A LITERAL MATRIX. v1.5.3 published with FIVE assets where seven were + # expected: `busbar-aarch64-apple-darwin.tar.gz` and `busbar-x86_64-unknown-linux-gnu.tar.gz` were + # both missing, which is Apple Silicon Mac and x86_64 Linux, the two most common platforms there + # are. `curl -fsSL https://getbusbar.com/install.sh | sh` returned a 404 on Apple Silicon (the + # documented one-line install, dead on the most common developer machine) and two of five download + # links on the live /download/ page 404'd. The guard below asserted `assets != 0`, which a + # five-asset release passes comfortably. A count can never see a MISSING platform; only a name can. + # + # Deriving both lists here means adding a platform is one edit and its verification comes along + # automatically. A hardcoded expected-names list in the verifier would be a second place to forget, + # which is the same defect one level up. + targets: + name: release target matrix (single source of truth) + needs: [plan, branch-green] + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.emit.outputs.matrix }} + assets: ${{ steps.emit.outputs.assets }} + # The bare list of target triples, for the set-equality job. It is emitted HERE, from the same + # parse as the build matrix, so "the set that was supposed to be produced" and "the set that + # was supposed to be verified" are literally the same computation and cannot drift apart. + targets: ${{ steps.emit.outputs.targets }} steps: - uses: actions/checkout@v7 - - # ── PGO path (host-native targets): the release binaries users download ARE PGO-optimized ── - # scripts/pgo-build.sh runs the three-phase PGO build (instrumented -> on-host training -> - # optimized) exactly as docker.yml does, FAIL-CLOSED: if any PGO phase fails the script exits - # non-zero and the job FAILS (it never falls back to a plain build). This closes the - # "PGO mandatory for release" drift where the GitHub-Release binaries were plain --release. - - name: Build busbar (PGO, required/fail-closed) - if: ${{ matrix.pgo }} + - name: Emit the target matrix and the asset names it must produce + id: emit env: - BUSBAR_RELEASE_PUBKEY: ${{ vars.BUSBAR_RELEASE_PUBKEY }} - PGO_TARGET: ${{ matrix.target }} - run: | - set -euo pipefail - scripts/pgo-build.sh - file "target/pgo/${{ matrix.target }}/release/busbar" - - # POSITIVE PGO GATE: pgo-build.sh writes this marker ONLY after a non-empty merged profile fed - # a successful -Cprofile-use build. Asserting it here (not just the script exit code) means a - # release cannot ship a non-PGO host binary and still pass. (Mirrors docker.yml's gate.) - - name: Verify PGO was applied (marker gate) - if: ${{ matrix.pgo }} + TAG: ${{ needs.plan.outputs.tag }} run: | set -euo pipefail - marker="target/pgo/${{ matrix.target }}/release/busbar.pgo-verified" - if [ ! -s "$marker" ]; then - echo "::error::PGO proof marker missing or empty at $marker - refusing to ship a non-PGO binary" >&2 - exit 1 - fi - echo "--- PGO proof marker ---" - cat "$marker" - grep -q '^pgo-verified=1$' "$marker" || { echo "::error::marker not marked verified" >&2; exit 1; } - bytes="$(grep '^profile_bytes=' "$marker" | cut -d= -f2)" - raw="$(grep '^profraw_count=' "$marker" | cut -d= -f2)" - if [ -z "$bytes" ] || [ "$bytes" -le 0 ] 2>/dev/null; then - echo "::error::merged profile was empty (profile_bytes=$bytes) - build was not PGO-optimized" >&2 - exit 1 - fi - if [ -z "$raw" ] || [ "$raw" -le 0 ] 2>/dev/null; then - echo "::error::no .profraw files fed the profile (profraw_count=$raw) - build was not PGO-optimized" >&2 - exit 1 - fi - echo "PGO verified: ${bytes} bytes of merged profile from ${raw} .profraw file(s)." + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import json, os + spec = json.load(open(".github/release-targets.json")) + tag = os.environ["TAG"] + # EVERY per-target difference travels in the matrix as a PARAMETER. The build workflow has + # no `if:` and no second path, so anything a target needs has to arrive this way or not at + # all -- that is what makes "no way for 1 to be different" a property of the graph rather + # than a promise in a comment. + fields = ("target", "runner", "pgo", "archive", "plugin_asset") + inc = [{k: t[k] for k in fields} for t in spec["targets"]] + assets = ["busbar-%s.%s" % (t["target"], t["archive"]) for t in spec["targets"]] + assets += [a.replace("{tag}", tag) for a in spec["metadata_assets"]] + # A FLOOR, BECAUSE A LOOP OVER A DISCOVERED SET WITH NO FLOOR PASSES WHEN THE SET IS EMPTY. + # Both the build matrix and the verify matrix are enumerated from this output; a truncated + # or mis-parsed manifest would otherwise build nothing, verify nothing, and report green + # all the way to a published release with no assets on it. + if len(inc) < 5: + raise SystemExit( + "release-targets.json declares %d targets; busbar ships 5. Refusing to run a " + "build matrix and a verify matrix over a set this small: an empty expectation " + "list passes for a release that published nothing." % len(inc)) + print("matrix=" + json.dumps({"include": inc})) + print("targets=" + json.dumps(sorted(t["target"] for t in spec["targets"]))) + print("assets=" + json.dumps(assets)) + PY + cat "$GITHUB_OUTPUT" - # Archive the PGO binary into the SAME `busbar-.tar.gz` name the plain path produces and - # upload it to the Release, so the attestation glob below covers it identically. - - name: Package and upload busbar (PGO) - if: ${{ matrix.pgo }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - bin="target/pgo/${{ matrix.target }}/release/busbar" - archive="busbar-${{ matrix.target }}.tar.gz" - tar -czf "$archive" -C "$(dirname "$bin")" busbar - gh release upload "${GITHUB_REF_NAME}" "$archive" \ - --repo "${GITHUB_REPOSITORY}" --clobber - - # ── Plain path (cross / windows targets that cannot self-train for PGO) ── - # Builds --release for the target (installing the toolchain + cross-linker as - # needed), tarballs the `busbar` binary, and uploads it to the Release for this tag. - # - # BUSBAR_RELEASE_PUBKEY (repository VARIABLE, not a secret — it is the PUBLIC half) is - # embedded into the binary at build time (plugin-sign's option_env!): busbar-signed - # plugins then verify as first-party with zero configuration. The matching PRIVATE half - # is the BUSBAR_SIGN_KEY secret used by the hook-plugins job below (and, independently, by - # each standalone plugin repo's own release workflow — see the "Store/auth plugin releases - # moved out" note above hook-plugins for why store/auth plugins are no longer packed here). - # TODO(release-keys): the release orchestrator generates the real keypair separately - # (`busbar-plugin-pack keygen`) and provisions BOTH: the variable (public) + the secret - # (private). Until provisioned, release binaries embed no key and first-party plugin - # verification is unavailable (plugins still load via the third-party/unsigned paths). - - name: Build and upload busbar - if: ${{ !matrix.pgo }} - uses: taiki-e/upload-rust-binary-action@v1 - env: - BUSBAR_RELEASE_PUBKEY: ${{ vars.BUSBAR_RELEASE_PUBKEY }} - with: - bin: busbar - target: ${{ matrix.target }} - archive: busbar-$target - token: ${{ secrets.GITHUB_TOKEN }} - - # Generate a keyless (Sigstore/OIDC) build-provenance attestation binding THIS - # archive's digest to this workflow run + commit. A user verifies the download with - # gh attestation verify --repo ${{ github.repository }} - # so a swapped/backdoored artifact on the Release page (the LiteLLM-PyPI scenario) - # fails verification. The glob matches whichever extension was produced (.tar.gz on - # unix, .zip on windows); `archive: busbar-$target` leaves it in the workspace. - - name: Attest build provenance - uses: actions/attest-build-provenance@v4 - with: - subject-path: "busbar-${{ matrix.target }}.*" + # -- THE BUILD: ONE PIPELINE, ONE STEP, FIVE TARGETS --------------------------------------------- + # + # This job used to be the build. It had TWO build steps -- scripts/pgo-build.sh behind + # `if: matrix.pgo` for the host-native targets, upload-rust-binary-action behind `if: !matrix.pgo` + # for the cross targets -- two packaging steps and two upload steps, each pair gated on the same + # flag. Both build steps carried `BUSBAR_RELEASE_PUBKEY` in their own `env:` block. Both were + # green. And busbar-aarch64-unknown-linux-gnu shipped with no embedded release key in 1.5.1, 1.5.2 + # and 1.5.3, so on ARM Linux every correctly-signed first-party plugin was refused. + # + # The key was never the point: the org variable was set the whole time. TWO BUILD PATHS PRODUCING + # ONE RELEASE is the point. A property established on one path is unproven on the other, silently + # and permanently, and no amount of care on either path fixes that. + # + # Owner, 2026-08-08: "should just be 1 build pipeline that takes what its building: arm, windows, + # mac, but it does the same thing for each no way for 1 to be different". + # + # So the build is now .github/workflows/build-artifact.yml: one reusable workflow that builds ONE + # artifact, with no `if:` on any step. Everything a target needs is a matrix VALUE passed in as an + # input -- runner label, whether PGO applies, archive extension -- read from + # .github/release-targets.json by the `targets` job above, which is the same single source of + # truth that computes the asset-name list `verify-assets` enforces. + # + # `fail-fast: false` is deliberate and is why `verify-assets` below runs on `!cancelled()`: one + # target failing must not cancel the other four, because the verifier's job is to NAME the + # platforms that are missing, and it cannot name what was never attempted. + upload-assets: + name: build + needs: [plan, draft, targets] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.targets.outputs.matrix) }} + uses: ./.github/workflows/build-artifact.yml + with: + target: ${{ matrix.target }} + runner: ${{ matrix.runner }} + archive: ${{ matrix.archive }} + tag: ${{ needs.plan.outputs.tag }} + permissions: + contents: write + id-token: write + attestations: write + secrets: inherit # ── Every plugin releases from its own repo — no plugin upload jobs live here at all ───────────── # Through 1.5.0 this file built and published every first-party plugin itself: `store-plugins` @@ -294,62 +739,528 @@ jobs: # hazard with no offsetting benefit: which one is canonical, which one do docs point at, which # one gets the CVE fix first. One release pipeline per plugin removes the ambiguity. # - # Every first-party store/auth/secret plugin crate has now been EXTRACTED out of this monorepo - # entirely: `crates/auth-oidc` + `crates/auth-oidc-plugin`, `crates/secret-vault` + - # `crates/secret-vault-plugin`, the Valkey store crate + its plugin crate, - # `crates/store-postgres` + `crates/store-postgres-plugin`, and `crates/store-sqlite` + - # `crates/store-sqlite-plugin` all now live in their own repos (GetBusbar/auth-oidc, - # GetBusbar/hashicorp-vault, GetBusbar/store-valkey, GetBusbar/store-postgres, - # GetBusbar/store-sqlite) — same-repo 2-crate Cargo workspaces bringing the real logic crate - # in-repo, the reference restructure GetBusbar/auth-oidc's "Restructure as a same-repo 2-crate - # workspace" commit established. None of their coverage comes from an in-tree crate any more; - # scripts/release-check.sh's sibling-checkout phases run each repo's own test suite (real dlopen - # ABI + a real backing service container, where applicable) instead — see that script's Phase 1 - # and its registry-driven Phase 2 suite loop over plugins.yaml. "Every plugin, including its logic crate, lives in - # its own repo" is no longer an aspiration — it's the current state. # See docs/plugins.md's "Signing and packaging" section for the corrected, current release-source # story per plugin kind. + # -- THE IMAGE, UNDER A NAME NOBODY PINS --------------------------------------------------------- + # + # docker.yml builds the multi-arch image and pushes it as `staging-` and NOTHING else. It is + # a real push to the real registry: `docker pull getbusbar/busbar:staging-` works, the image + # boots, the binary inside runs, and the OCI version label already reads X.Y.Z. What it is not is + # a name any user, any doc, or any Helm chart refers to. + # + # It runs in PARALLEL with the binary matrix, not after it: they are independent artifacts and the + # two-arch PGO container build is the long pole. `verify-staged` waits for both. + stage-image: + name: stage the image (staging tag only, nothing user-facing) + needs: [plan, gate] + uses: ./.github/workflows/docker.yml + with: + staging_tag: ${{ needs.plan.outputs.staging_tag }} + label_version: ${{ needs.plan.outputs.version }} + permissions: + contents: read + packages: write + id-token: write + attestations: write + secrets: inherit - # PHANTOM-RELEASE GUARD: assert the published Release actually carries assets before we treat this - # as a real release. The per-target build/upload jobs run with fail-fast:false, and `create-release` - # always makes the (initially empty) Release up front — so a build/pack failure on EVERY target - # (e.g. a stale Cargo.lock tripping `--locked`) leaves a tag + Release with ZERO assets: a "phantom" - # that silently breaks busbar's plugin-registry-gate. This job fails the whole release run loud if - # assets == 0, so a phantom can never ship (or notify downstream) unnoticed. It depends on the build - # matrix but does NOT inherit its fail-fast:false — one green target is enough to have assets, but - # zero across the board must hard-fail here. + # PHANTOM-RELEASE GUARD: assert the draft actually carries every asset before anything is + # promoted. The per-target build/upload jobs run with fail-fast:false, and `draft` always makes the + # (initially empty) release up front, so a build/pack failure on EVERY target (e.g. a stale + # Cargo.lock tripping `--locked`) leaves a release with ZERO assets: a "phantom" that silently + # breaks busbar's plugin-registry-gate. This job fails the run loud if assets are missing. + # + # IT NOW GUARDS SOMETHING IT COULD NOT GUARD BEFORE. When this ran against a PUBLISHED release it + # could only report the damage; the tag and the release page already existed and recovery meant + # deleting both. Running it against the DRAFT means a red verdict here stops the release before a + # single user-facing name is minted. verify-assets: - needs: [upload-assets] + name: the draft owes every asset the manifest names + needs: [plan, targets, upload-assets, sbom, openapi] runs-on: ubuntu-latest + # `!cancelled()` IS THE POINT OF THIS LINE, and it is the second half of the 1.5.3 defect. + # `upload-assets` runs `fail-fast: false`, so when two of its five legs failed the JOB failed -- + # and a `needs:` on a failed job SKIPS the dependent by default. So the one guard that existed to + # notice a broken release was skipped precisely because the release was broken. A verifier that + # only runs when everything already worked is not a verifier. This one runs whenever the run was + # not cancelled, so a partial matrix produces a RED verify-assets naming the platforms that are + # missing, instead of a grey one naming nothing. + if: ${{ !cancelled() && needs.plan.outputs.release == '1' }} + steps: + - name: Assert the draft carries every asset the matrix owes it + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EXPECTED: ${{ needs.targets.outputs.assets }} + TAG: ${{ needs.plan.outputs.tag }} + run: | + set -euo pipefail + gh release view "$TAG" --repo "$GITHUB_REPOSITORY" \ + --json assets --jq '.assets[] | "\(.name)\t\(.size)"' > /tmp/got.tsv || : > /tmp/got.tsv + echo "Draft $TAG carries these assets:" + cat /tmp/got.tsv + python3 - <<'PY' + import json, os, sys + expected = json.loads(os.environ["EXPECTED"]) + got = {} + for line in open("/tmp/got.tsv"): + line = line.rstrip("\n") + if not line: + continue + name, _, size = line.partition("\t") + got[name] = int(size or 0) + + missing = [a for a in expected if a not in got] + # A NAME IN THE ASSET LIST IS NOT A USABLE ARTIFACT: GitHub creates the row as soon as the + # upload starts, so a 0-byte or truncated upload lists identically to a good one. 1 KiB is + # far below any real busbar tarball and far above an empty or header-only file. + empty = [a for a in expected if a in got and got[a] < 1024] + + summary = os.environ.get("GITHUB_STEP_SUMMARY") + lines = ["### Draft asset verification", "", "| asset | bytes | verdict |", "| --- | --- | --- |"] + for a in expected: + if a not in got: + lines.append("| `%s` | - | MISSING |" % a) + elif got[a] < 1024: + lines.append("| `%s` | %d | TOO SMALL |" % (a, got[a])) + else: + lines.append("| `%s` | %d | ok |" % (a, got[a])) + extra = sorted(set(got) - set(expected)) + if extra: + lines += ["", "Also present (not required): " + ", ".join("`%s`" % e for e in extra)] + if summary: + open(summary, "a").write("\n".join(lines) + "\n") + print("\n".join(lines)) + + tag = os.environ["TAG"] + if not got: + print("::error::PHANTOM RELEASE: the %s draft has 0 assets. Every build/pack target " + "failed to upload. Nothing is public and no tag exists, so this is a clean " + "retry: fix the build (check Cargo.lock freshness vs --locked and the plugin " + "cdylib build step) and re-run this workflow." % tag, file=sys.stderr) + sys.exit(1) + if missing or empty: + if missing: + print("::error::INCOMPLETE RELEASE: the %s draft is missing %d of %d required " + "asset(s): %s. Each missing name is a platform whose users would get a 404 " + "from install.sh and from the /download/ page. Nothing was promoted, so fix " + "that target's leg in `upload-assets` and re-run: no tag to delete, no " + "release to unpublish." % + (tag, len(missing), len(expected), ", ".join(missing)), file=sys.stderr) + if empty: + print("::error::TRUNCATED RELEASE: the %s draft has these assets at under 1 KiB, " + "which means the upload was cut short: %s" % (tag, ", ".join(empty)), + file=sys.stderr) + sys.exit(1) + print("All %d required assets present and plausibly sized." % len(expected)) + PY + + # -- THE PER-ARTIFACT CONTRACT: 100% OR 0%, FOR EVERY ARTIFACT ----------------------------------- + # + # `verify-assets` above answers "is every name present, and plausibly sized". That is a question + # about the RELEASE. This job answers "is this artifact everything a busbar binary must be", which + # is a question about the ARTIFACT, and it is the question 1.5.3 never asked: five assets were + # present and correctly sized, and one of them refused every signed plugin on its platform. + # + # ONE VERIFIER, ONE ARTIFACT, EVERY ROW. scripts/verify-artifact.py takes (artifact, target) and + # checks EVERY row of .github/artifact-contract.json that applies to that target. The rows are + # data, so adding a property is one entry in that file plus its check function -- and it then + # applies to all five targets automatically, with no per-target list for a platform to be + # forgotten from. The verifier refuses to run at all unless the declared row ids and the + # implemented checks are the SAME SET in both directions, so a row nobody implemented cannot look + # like a row that passed. + # + # IT RUNS ON THE TARGET'S OWN NATIVE RUNNER, and that is what makes the contract uniform. Because + # every target now builds natively, every artifact can be EXECUTED where it is verified: the + # aarch64-linux asset runs on ubuntu-24.04-arm, the Intel-mac asset on macos-15-intel, the Windows + # asset on windows-latest. So the executable rows -- the real signed first-party plugin, the + # anchored --version, the documented quickstart -- run for EVERY platform. Nothing is statically + # approximated because a runner could not execute it, and there is no per-platform subset. + # + # THE ARTIFACT IS DOWNLOADED BACK FROM THE DRAFT, NOT REUSED FROM THE BUILD. Every property is + # asserted on the bytes a user would receive. Asserting them on the build machine's copy is what + # 1.5.3 did with an env var, and the env var was set. + verify-artifact: + name: contract ${{ matrix.target }} + needs: [plan, targets, upload-assets] + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.targets.outputs.matrix) }} + # THE SAME `!cancelled()` REASONING AS `verify-assets`, FOR THE SAME REASON. `upload-assets` + # runs fail-fast:false, so one failed leg fails the matrix job, and a `needs:` on a failed job + # SKIPS the dependent by default -- the verifier would be switched off precisely when the + # release is broken. A skipped leg leaves no receipt, so `verify-set-equality` below turns it + # red anyway; this line is what lets the other four legs still report what they found. + if: ${{ !cancelled() && needs.plan.outputs.release == '1' }} steps: - - name: Assert the Release has at least one asset + - uses: actions/checkout@v7 + + - name: Download this artifact from the draft, and its build evidence + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + archive="busbar-${{ matrix.target }}.${{ matrix.archive }}" + gh release download "${{ needs.plan.outputs.tag }}" --repo "$GITHUB_REPOSITORY" \ + --pattern "$archive" --dir . --clobber + # The REAL signed first-party plugin for THIS platform, from its own repo's own release, + # signed with the real private half of the release key. Nothing here is packed locally: a + # locally-signed fixture would prove the signature code compiles and would have gone green + # on the broken 1.5.3 aarch64 artifact. + gh release download "$(jq -r .plugin_probe.tag .github/release-targets.json)" \ + --repo "$(jq -r .plugin_probe.repo .github/release-targets.json)" \ + --pattern "${{ matrix.plugin_asset }}" --dir plugin-probe --clobber + ls -l "$archive" plugin-probe + + - uses: actions/download-artifact@v4 + with: + name: build-evidence-${{ matrix.target }} + path: build-evidence + + - name: The artifact contract + shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail - count="$(gh release view "${GITHUB_REF_NAME}" \ - --repo "${GITHUB_REPOSITORY}" \ - --json assets --jq '.assets | length')" - echo "Release ${GITHUB_REF_NAME} has ${count} asset(s)." - if [ "${count}" -eq 0 ]; then - echo "::error::PHANTOM RELEASE: ${GITHUB_REF_NAME} was published with 0 assets." \ - "Every build/pack target failed to upload a tarball. Failing the release run so this" \ - "tag is not mistaken for a real release by busbar's plugin-registry-gate. Fix the" \ - "build (check Cargo.lock freshness vs --locked and the plugin cdylib build step)," \ - "delete this tag+release, and re-cut." >&2 + python3 scripts/verify-artifact.py \ + --archive "busbar-${{ matrix.target }}.${{ matrix.archive }}" \ + --target "${{ matrix.target }}" \ + --version "${{ needs.plan.outputs.version }}" \ + --pubkey "${{ vars.BUSBAR_RELEASE_PUBKEY }}" \ + --plugin "plugin-probe/${{ matrix.plugin_asset }}" \ + --evidence build-evidence \ + --repo "$GITHUB_REPOSITORY" \ + --repo-root . + + # THE RECEIPT. Written only after the contract exits 0, so its existence means every applicable + # row passed for this target -- not that a job was scheduled. `verify-set-equality` reads these. + - name: Write the verified-receipt + shell: bash + run: | + set -euo pipefail + mkdir -p verified + echo "${{ matrix.target }}" > "verified/${{ matrix.target }}" + - uses: actions/upload-artifact@v4 + with: + name: verified-${{ matrix.target }} + path: verified + if-no-files-found: error + retention-days: 7 + + # -- SET EQUALITY, NOT SUBSET -------------------------------------------------------------------- + # + # THIS IS THE CLAUSE THAT MAKES A FORGOTTEN LEG IMPOSSIBLE RATHER THAN UNLIKELY. + # + # The third structural fault behind the 1.5.3 aarch64 defect was that the PRODUCED set and the + # VERIFIED set were never compared. The `targets` job was already correct and already the single + # source of truth; verification simply did not enumerate from it, so an artifact could be produced + # and never checked and nothing anywhere noticed. verify-deploy.yml still hardcodes + # `busbar-x86_64-unknown-linux-gnu.tar.gz` for its quickstart check -- one platform, chosen once, + # standing in for five. + # + # So this job asserts three sets are IDENTICAL: + # + # DECLARED the targets `targets` emitted, from .github/release-targets.json. + # BUILT one `build-evidence-` receipt per artifact the build actually produced. + # VERIFIED one `verified-` receipt per artifact the contract actually passed. + # + # A target that is built but not verified is an ERROR. A target that is verified but not built is + # an ERROR. A subset check in either direction would have passed the whole 1.5.3 release. + # + # IT READS RECEIPTS, NOT JOB CONCLUSIONS, and the difference matters: a matrix leg that is skipped + # (the default `needs:` behaviour when an upstream job fails) leaves a grey square, no receipt, and + # a set that has silently shrunk to fit. Receipts can only be created by a step that ran and + # succeeded, so the sets can only shrink in a direction this job can see. + # + # `!cancelled()` for the same reason as its two upstream verifiers: this check must be loudest + # exactly when something upstream went wrong. + verify-set-equality: + name: the set built and the set verified must be identical + needs: [plan, targets, upload-assets, verify-artifact] + runs-on: ubuntu-latest + if: ${{ !cancelled() && needs.plan.outputs.release == '1' }} + steps: + - uses: actions/download-artifact@v4 + with: + path: receipts + pattern: "*-*" + - name: Compare the declared, built and verified sets + env: + DECLARED: ${{ needs.targets.outputs.targets }} + run: | + set -euo pipefail + ls -la receipts || true + python3 - <<'PY' + import json, os, sys + + declared = set(json.loads(os.environ["DECLARED"])) + names = set(os.listdir("receipts")) if os.path.isdir("receipts") else set() + built = {n[len("build-evidence-"):] for n in names if n.startswith("build-evidence-")} + verified = {n[len("verified-"):] for n in names if n.startswith("verified-")} + + lines = ["### produced vs verified", "", "| target | built | contract |", "| --- | --- | --- |"] + for t in sorted(declared | built | verified): + lines.append("| `%s` | %s | %s |" % ( + t, "yes" if t in built else "**NO**", "PASS" if t in verified else "**NOT VERIFIED**")) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + open(summary, "a").write("\n".join(lines) + "\n") + print("\n".join(lines)) + + fail = False + # A FLOOR FIRST. Every comparison below is between sets, and all three being empty is the + # state in which every comparison is trivially satisfied -- the false green this whole + # design exists to make impossible. + if len(declared) < 5: + print("::error::the declared target set has %d entries; busbar ships 5. Set equality " + "between three empty sets is not a check." % len(declared), file=sys.stderr) + fail = True + for label, got in (("built", built), ("verified", verified)): + missing = sorted(declared - got) + extra = sorted(got - declared) + if missing: + print("::error::NOT %s: %s. Every one of those targets is an artifact this release " + "owes that was never %s. A target that is built but not verified is an error, " + "and so is a target that is verified but not built -- this is set equality, " + "not a subset check, because a subset check passes the 1.5.3 release in which " + "busbar-aarch64-unknown-linux-gnu shipped with no release key." + % (label.upper(), ", ".join(missing), label), file=sys.stderr) + fail = True + if extra: + print("::error::%s but NOT DECLARED: %s. An artifact nothing declares is an artifact " + "nothing owes and nothing tracks; add it to .github/release-targets.json or " + "stop producing it." % (label.upper(), ", ".join(extra)), file=sys.stderr) + fail = True + if fail: + sys.exit(1) + print("SET EQUALITY HOLDS: %d targets declared, built and contract-verified: %s" + % (len(declared), ", ".join(sorted(declared)))) + PY + + # -- THE GATE: CONSUMER VERIFICATION, BEFORE THE NAME EXISTS ------------------------------------- + # + # This is the job the whole restructure exists to make possible. It runs verify-deploy.yml -- the + # same verifier, the same checks, reused rather than reimplemented -- in `staging` mode against + # the artifacts that were just produced: + # + # * `docker rmi` FIRST, then pull the staged image fresh. A cached local image lies, and it has + # lied to a human on this project inside the last week. Then boot it and read the OCI version + # label off the running image. + # * download the real linux tarball FROM THE DRAFT, unpack it, EXECUTE the binary, and assert + # `busbar --version` says X.Y.Z. Not "the archive exists" -- the bytes run and identify + # themselves. + # * run the DOCUMENTED quickstart from docs/getting-started.md, both halves: the binary with the + # minimal config, and the docker one-liner, each of which must answer `ok` on /healthz. + # * prove `gh attestation verify` -- the exact command the docs tell users to run -- passes on + # the real downloaded bytes. + # * prove every asset the platform manifest names is not merely listed but actually downloads. + # + # WHAT IT DELIBERATELY DOES NOT CHECK YET: the Homebrew tap, the Helm chart, getbusbar.com, + # /releases/latest, the `latest` container tag. Those are downstream channels that only move once + # the release is public, so asserting them here would be asserting a falsehood. They are checked + # by `consumer-verification` at the bottom, after the promote, exactly as before. The split is by + # what CAN be true before publication, not by what is convenient. + # + # NO `!cancelled()` HERE, AND THAT IS DELIBERATE. Every other guard in this file uses it so a red + # upstream job cannot switch the guard off. This one is a GATE, not a report: if the artifacts did + # not build, there is nothing to verify and nothing must be promoted. Default `needs` semantics -- + # skip on upstream failure, and therefore skip the promote too -- is exactly the behaviour wanted. + verify-staged: + name: consumer verification (STAGED, this is the gate) + needs: [plan, verify-assets, stage-image] + uses: ./.github/workflows/verify-deploy.yml + with: + version: ${{ needs.plan.outputs.version }} + stage: staging + image_ref: getbusbar/busbar:${{ needs.plan.outputs.staging_tag }} + permissions: + contents: read + issues: write + actions: read + secrets: inherit + + # -- PROMOTE, PART 1: THE IRREVERSIBLE HALF, FIRST ----------------------------------------------- + # + # Manifest-only retag of the exact digest verification just pulled and ran, onto `X.Y.Z` and then + # `latest`, on Docker Hub and GHCR. No rebuild: the promoted image IS the verified image. + # + # IT GOES FIRST BECAUSE IT IS THE ONLY IRREVERSIBLE STEP IN THE RELEASE. A Docker Hub `X.Y.Z` can + # never be overwritten. A git tag and a GitHub release can both be deleted. So the ordering rule + # is: attempt the thing that cannot be undone while everything else is still undone. If this job + # fails, no git tag was pushed, no release was published, and the draft is still invisible -- a + # clean retry. Had it run last, a failure here would leave a published release pointing at an + # image that does not exist, which is the 1.5.3 shape upside down. + # + # docker.yml's `promote` job re-derives all four names from the registry afterwards and fails loud + # if only some of them moved, so a partial promote is red rather than silent. + promote-image: + name: promote the image (X.Y.Z + latest, no rebuild) + needs: [plan, verify-staged] + uses: ./.github/workflows/docker.yml + with: + promote_to: ${{ needs.plan.outputs.version }} + promote_from: ${{ needs.plan.outputs.staging_tag }} + permissions: + contents: read + packages: write + id-token: write + attestations: write + secrets: inherit + + # -- PROMOTE, PART 2: MINT THE NAME --------------------------------------------------------------- + # + # Push the git tag, flip the draft to published-and-latest, then RE-DERIVE the whole promote from + # outside and fail if any single name did not move. + # + # WHY THE VERIFICATION IS A SEPARATE STEP AND NOT AN ASSUMPTION. `gh release edit --draft=false` + # exiting 0 means the API accepted the request. It does not mean `/releases/latest` now resolves + # to this tag, and "the version published but the pointer did not move" is precisely the class of + # bug that cost this project a night: `latest` on Docker Hub sat frozen at 1.5.2 through the whole + # of the 1.5.3 release with nothing red anywhere. Every promote step here is therefore followed by + # an independent read of the thing it was supposed to change. + # + # IDEMPOTENT. The tag push is skipped if the tag already exists at the right commit and REFUSED if + # it exists at a different one; `--draft=false` on an already-published release is a no-op. So a + # re-run after a half-completed promote completes it rather than creating a second half-state. + # + # GITHUB_TOKEN IS ENOUGH NOW, AND THAT IS A SIMPLIFICATION WORTH NAMING. tag-on-main.yml needed an + # org PAT for one reason only: a tag pushed with GITHUB_TOKEN does not trigger other workflows, + # and the release DEPENDED on the tag triggering release.yml and docker.yml. Nothing is triggered + # by a tag any more, so that requirement is gone with it. + promote-release: + name: promote the release (tag + publish + prove it landed) + needs: [plan, promote-image] + runs-on: ubuntu-latest + outputs: + version: ${{ needs.plan.outputs.version }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + # GATE 0, RE-ASSERTED IMMEDIATELY BEFORE THE TAG IS MINTED. `branch-green` is already upstream + # of everything here, so this cannot be the first time the question is asked - but an hour of + # building and verifying passes between the two, and a check that was green then can be red + # now: someone re-runs a flaky job and it fails the second time, or a scheduled workflow lands + # on this commit. The tag is the irreversible-ish moment, so the question is asked again at + # exactly that moment. Same rule as before: no bypass, and unknown is red. + - name: Refuse to tag a commit that is red RIGHT NOW + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SHA: ${{ needs.plan.outputs.sha }} + shell: bash + run: | + set -uo pipefail + for attempt in 1 2 3 4 5; do + runs="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${SHA}&per_page=100" 2>/dev/null)" && [ -n "$runs" ] && break + echo " api read failed (attempt ${attempt}/5)"; sleep $(( attempt * 10 )); runs="" + done + if [ -z "${runs:-}" ]; then + echo "::error::REFUSING TO TAG: the Actions API could not be read, so the CI status of ${SHA} is UNKNOWN right now. Unknown is not green. No tag was pushed and the release is still a draft; re-run this workflow." + exit 1 + fi + red="$(echo "$runs" | jq -r -s --arg skip "Release,Verify deploy" ' + ($skip | split(",")) as $not + | [.[] | .workflow_runs[]?] + | map(select(.id != (env.GITHUB_RUN_ID | tonumber) and (.name | IN($not[]) | not))) + | unique_by(.name) + | .[] | select(.conclusion != "success" and .conclusion != "skipped" and .conclusion != "neutral") + | "\(.name) -> \(.conclusion // "still running") \(.html_url)"')" + if [ -n "$red" ]; then + echo "$red" + echo "::error::REFUSING TO TAG A RED COMMIT. ${SHA:0:7} went red between gate 0 and the promote. No tag has been pushed, the release is still an invisible draft, no fan-out has fired. There is no override. Fix the red and re-run." exit 1 fi + echo "PASS: ${SHA:0:7} is still green at promote time." + - name: Push the version tag + env: + TAG: ${{ needs.plan.outputs.tag }} + SHA: ${{ needs.plan.outputs.sha }} + run: | + set -euo pipefail + if git ls-remote --exit-code --tags origin "refs/tags/$TAG" >/dev/null 2>&1; then + have="$(git ls-remote --tags origin "refs/tags/$TAG^{}" | awk '{print $1}')" + [ -n "$have" ] || have="$(git ls-remote --tags origin "refs/tags/$TAG" | awk '{print $1}')" + if [ "$have" = "$SHA" ]; then + echo "::notice::${TAG} already exists at ${SHA:0:7} (re-run). Nothing to push." + else + echo "::error::${TAG} already exists on origin but points at ${have:0:7}, not the commit being released (${SHA:0:7}). Refusing to move a version tag. Fix: delete the stale tag or bump the version." >&2 + exit 1 + fi + else + git config user.name "Matthew Jackson" + git config user.email "matthew@pq.io" + git tag -a "$TAG" -m "$TAG" "$SHA" + git push origin "$TAG" + echo "::notice::Pushed ${TAG} at ${SHA:0:7}." + fi + - name: Publish the draft + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ needs.plan.outputs.tag }} + run: | + set -euo pipefail + gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --draft=false --latest + echo "::notice::${TAG} promoted from draft to published." + # PARTIAL PROMOTE MUST BE DETECTABLE, NOT SILENT. Four independent reads, none of them the + # command that was just run: the remote git tag, the release's own draft flag, the release's + # `isLatest` flag, and the plain unauthenticated /releases/latest redirect a user's shell + # follows. Applying the version tag but failing to move `latest` is exactly the shape this + # exists to catch, and it names WHICH half did not land. + - name: Assert the promote actually landed + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ needs.plan.outputs.tag }} + SHA: ${{ needs.plan.outputs.sha }} + run: | + set -euo pipefail + fail=0 + if git ls-remote --exit-code --tags origin "refs/tags/$TAG" >/dev/null 2>&1; then + echo "PASS: git tag ${TAG} exists on origin" + else + echo "::error::PARTIAL PROMOTE: the container tags for ${TAG#v} were published but the git tag ${TAG} does not exist on origin. Fix: re-run this workflow; the promote is idempotent and will complete the missing half." + fail=1 + fi + state="$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isDraft,isLatest 2>/dev/null || echo '{}')" + draft="$(echo "$state" | jq -r '.isDraft // "unknown"')" + latest="$(echo "$state" | jq -r '.isLatest // "unknown"')" + if [ "$draft" = "false" ]; then + echo "PASS: Release ${TAG} is published (not a draft)" + else + echo "::error::PARTIAL PROMOTE: Release ${TAG} still reports isDraft=${draft}. The image was promoted but the release is still invisible. Fix: re-run this workflow." + fail=1 + fi + if [ "$latest" = "true" ]; then + echo "PASS: Release ${TAG} is marked latest" + else + echo "::error::PARTIAL PROMOTE: Release ${TAG} reports isLatest=${latest}. The version published but the pointer did not move -- the exact defect that left \`latest\` frozen through the 1.5.3 release. Fix: re-run this workflow, or \`gh release edit ${TAG} --latest\`." + fail=1 + fi + loc="$(curl -fsS --max-time 30 -o /dev/null -w '%{redirect_url}' "https://github.com/${GITHUB_REPOSITORY}/releases/latest" || true)" + case "$loc" in + */releases/tag/"$TAG") echo "PASS: /releases/latest redirects to ${TAG}" ;; + *) + echo "::error::PARTIAL PROMOTE: https://github.com/${GITHUB_REPOSITORY}/releases/latest redirects to '${loc:-}', not ${TAG}. install.sh and the download page follow that redirect, so users would still be served the previous release. Fix: re-run this workflow." + fail=1 + ;; + esac + [ "$fail" = 0 ] || exit 1 + echo "::notice::${TAG} is fully public: git tag, published release, latest pointer, and both container registries." # ── Fan out an instant "a real release just happened" signal to every downstream repo that # self-heals off busbar's releases (Homebrew tap, Helm chart, and whatever else gets added to # .github/release-notify-targets.txt). Each target already has its OWN daily-poll workflow as # the durable fallback (so a missed/failed dispatch here is never a silent miss, just a same-day # catch-up) — this job only makes the common case instant instead of up-to-24h-later. Adding a - # new downstream consumer to the fan-out is a one-line addition to that text file, no workflow - # changes needed here or in this job. + # new downstream consumer to the fan-out is a one-line addition to that text file. + # + # IT NOW FIRES ONLY AFTER A PROVEN-GOOD PROMOTE. It used to hang off the build jobs, so a fan-out + # could tell nineteen repos to go and consume a release that did not work. There is nothing to + # fan out about until the release is real, so `promote-release` is its only dependency. notify-downstream: - needs: [sbom, openapi, upload-assets, verify-assets] + needs: [plan, promote-release] runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -361,10 +1272,11 @@ jobs: # this secret (org-level, so every downstream repo's own workflow can also use it if # needed) before this job can do anything; until then it fails loudly rather than # silently no-op'ing, so a missing token can't masquerade as "nothing to notify." - # GH_TOKEN is what `gh api` itself reads; DISPATCH_TOKEN is the same value under a - # separate name purely so the emptiness check below reads clearly. GH_TOKEN: ${{ secrets.RELEASE_DISPATCH_TOKEN }} DISPATCH_TOKEN: ${{ secrets.RELEASE_DISPATCH_TOKEN }} + TAG: ${{ needs.plan.outputs.tag }} + VER: ${{ needs.plan.outputs.version }} + SHA: ${{ needs.plan.outputs.sha }} shell: bash run: | set -euo pipefail @@ -374,24 +1286,61 @@ jobs: "up within 24h (self-healing fallback intact), but the instant path is unavailable." >&2 exit 1 fi - ver="${GITHUB_REF_NAME#v}" - sha="${GITHUB_SHA}" fail=0 while IFS= read -r repo; do # Skip blank lines and comments. case "$repo" in ''|'#'*) continue ;; esac - echo "Dispatching upstream-release to ${repo} (tag ${GITHUB_REF_NAME}, sha ${sha:0:7})..." + echo "Dispatching upstream-release to ${repo} (tag ${TAG}, sha ${SHA:0:7})..." # A downstream repo that has no release-on-upstream workflow yet simply ignores the # event — a harmless no-op — so a full target list is safe before every repo is wired. # A real API/auth failure must still surface, so collect failures and report at the end. if ! gh api "repos/${repo}/dispatches" \ -f event_type=upstream-release \ - -f "client_payload[tag]=${GITHUB_REF_NAME}" \ - -f "client_payload[version]=${ver}" \ - -f "client_payload[sha]=${sha}" \ + -f "client_payload[tag]=${TAG}" \ + -f "client_payload[version]=${VER}" \ + -f "client_payload[sha]=${SHA}" \ -f "client_payload[repo]=${GITHUB_REPOSITORY}"; then echo "::warning::dispatch to ${repo} failed" fail=1 fi done < .github/release-notify-targets.txt [ "$fail" = 0 ] || { echo "::error::one or more downstream dispatches failed (see warnings)"; exit 1; } + + # -- THE LAST STEP: DOES THE PUBLISHED THING WORK FOR A USER? ------------------------------------ + # + # `verify-staged` above already proved the ARTIFACTS work, before they had names. This proves the + # CHANNELS moved: `docker pull getbusbar/busbar` (untagged), `curl -fsSL + # https://getbusbar.com/install.sh | sh`, `brew install getbusbar/busbar/busbar`, the Helm chart's + # appVersion, /releases/latest, the download page. Those are different systems, in different + # repos, on different clocks, and not one of them can be checked before publication -- which is + # why this half stays post-promote and why the split between the two is by what is knowable, not + # by preference. + # + # It cannot gate publication, and does not pretend to. What it does is make the verdict impossible + # to miss: it turns THIS run red, on the release's own status page, and verify-deploy's `alert` + # job opens or updates a labelled issue naming the failing check, its expected and observed + # values, and the run URL. + # + # `!cancelled()` FOR THE SAME REASON `verify-assets` HAS IT. A `needs:` on a failed job skips the + # dependent by default, so without this a failed `notify-downstream` (which fails outright while + # RELEASE_DISPATCH_TOKEN is unprovisioned) would SKIP consumer verification on every release. + consumer-verification: + name: consumer verification (PUBLIC, last step) + needs: [plan, promote-release, notify-downstream] + # Bracket form, not `needs.promote-release.result`: `-` is the subtraction operator in a GitHub + # expression, and a hyphenated job id reached by dot access is at best relying on the lexer to + # guess. The index form is unambiguous. The clause itself is what stops the public sweep running + # against a release that was never promoted, which would fail on every channel and say nothing. + if: ${{ !cancelled() && needs['promote-release'].result == 'success' }} + uses: ./.github/workflows/verify-deploy.yml + with: + version: ${{ needs.plan.outputs.version }} + # Job-level permissions REPLACE the workflow-level block for a called workflow, and a called + # workflow can never exceed what it is granted here. `issues: write` is what lets the alert job + # file the issue; `actions: read` is what lets it read the failed job's log to quote the exact + # expected/observed values. + permissions: + contents: read + issues: write + actions: read + secrets: inherit diff --git a/.github/workflows/tag-on-main.yml b/.github/workflows/tag-on-main.yml deleted file mode 100644 index 57e52a10..00000000 --- a/.github/workflows/tag-on-main.yml +++ /dev/null @@ -1,74 +0,0 @@ -# tag-on-main — the release trigger. Pushing/merging to `main` cuts the release. -# -# BRANCH MODEL: prepare-release.yml bumps the version on `dev` → promote dev→`qa` -# (qa-gate.yml runs the ~2h full-plugin gate) → promote qa→`main`. Landing on main runs THIS -# workflow, which reads crates/busbar/Cargo.toml's version and, if the matching `vX.Y.Z` tag does -# not already exist, creates and pushes it. That tag push is what triggers release.yml (binaries, -# SBOM, OpenAPI asset, provenance, and the 19-repo downstream `upstream-release` dispatch) and -# docker.yml (Docker Hub + GHCR + cosign). One push to main = the whole BOOM. -# -# IDEMPOTENT BY DESIGN: if the tag for the current Cargo.toml version already exists, this is a -# safe no-op — so re-pushing main, or landing docs/hotfixes to main WITHOUT bumping the version, -# never re-releases. To cut a release you bump the version (via prepare-release.yml on dev) and let -# that bumped commit reach main. -# -# WHY A PAT (RELEASE_DISPATCH_TOKEN), NOT GITHUB_TOKEN: a tag pushed with the default GITHUB_TOKEN -# does NOT trigger other workflows (GitHub blocks recursive workflow triggering), so release.yml -# and docker.yml would never fire. The org fine-grained PAT already provisioned for the downstream -# fan-out (release.yml's notify-downstream) pushes the tag as a real actor, so the tag-triggered -# workflows run. That PAT must have Contents:write on THIS repo (to push the tag) in addition to -# the downstream Contents:read + Actions:write it already carries. -name: tag-on-main - -on: - push: - branches: [main] - -# Serialize: a burst of main pushes must not race two taggers onto the same version. -concurrency: - group: tag-on-main - cancel-in-progress: false - -permissions: - contents: write - -jobs: - tag: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - with: - # fetch-depth: 0 so `git rev-parse v$V` can see every existing tag (the idempotency - # guard). token: the PAT so the tag push below triggers release.yml + docker.yml. - fetch-depth: 0 - token: ${{ secrets.RELEASE_DISPATCH_TOKEN }} - - - name: Tag the Cargo.toml version if it isn't tagged yet (→ cuts the release) - env: - # Fail loud if the PAT is missing rather than pushing a tag with the checkout's default - # token (which would silently NOT trigger release.yml/docker.yml — a release that looks - # cut but ships nothing). - RELEASE_DISPATCH_TOKEN: ${{ secrets.RELEASE_DISPATCH_TOKEN }} - shell: bash - run: | - set -euo pipefail - if [ -z "${RELEASE_DISPATCH_TOKEN:-}" ]; then - echo "::error::RELEASE_DISPATCH_TOKEN is not provisioned — cannot cut the release from" \ - "main. A tag pushed with the default GITHUB_TOKEN would not trigger release.yml/" \ - "docker.yml, so refusing to push a dud tag. Provision the org PAT (Contents:write" \ - "on this repo) and re-run." >&2 - exit 1 - fi - # Robust version read from the package table (tomllib; ubuntu-latest ships Python ≥3.11). - V="$(python3 -c "import tomllib; print(tomllib.load(open('crates/busbar/Cargo.toml','rb'))['package']['version'])")" - echo "$V" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' || { echo "::error::crates/busbar/Cargo.toml version '$V' is not X.Y.Z"; exit 1; } - if git rev-parse "v$V" >/dev/null 2>&1; then - echo "::notice::tag v$V already exists — nothing to cut (safe no-op). Bump the version on" \ - "dev via prepare-release.yml to cut a new release." - exit 0 - fi - git config user.name "Matthew Jackson" - git config user.email "matthew@pq.io" - git tag -a "v$V" -m "v$V" - git push origin "v$V" - echo "::notice::Tagged v$V on main — release.yml + docker.yml + the downstream cascade are now firing." diff --git a/.github/workflows/verify-deploy.yml b/.github/workflows/verify-deploy.yml index 61f67ca9..03b88d80 100644 --- a/.github/workflows/verify-deploy.yml +++ b/.github/workflows/verify-deploy.yml @@ -53,12 +53,32 @@ name: Verify deploy # these back the live numbers on getbusbar.com and are currently unmonitored. # (l) the DOCUMENTED quickstart actually runs: both the binary quickstart and the docker # one-liner from docs/getting-started.md boot and answer on /healthz. +# (m) `gh attestation verify --repo GetBusbar/busbar` -- the exact command the docs tell +# users to run before trusting a download -- passes on real downloaded bytes. Previously "the +# release is attested" meant a workflow step exited 0, never that a user could verify it. +# (n) numbers getbusbar.com presents as LIVE actually track their authority (Docker pull count vs +# Docker Hub, star count vs GitHub). The download page served a pull count baked into static +# HTML at build time, beside a working Worker nothing called, so it aged until a redeploy. +# +# THE PATTERN THESE LAST TWO EXIST FOR. Three defects in the 1.5.3 release were one shape: something +# DECLARED a behaviour and nothing CHECKED it. docker.yml's comment claimed a `latest` tag its tag +# list never emitted, so `docker pull getbusbar/busbar` served 1.5.2 for the whole release. The site +# claimed a live pull count that was a build-time snapshot. A "zero broken links" claim turned out +# to be unreproducible, and there is still no link checker anywhere. Check (a) already guards the +# first of those; (n) guards the second. A claim with no assertion behind it decays invisibly, which +# is worse than not making it. # # Triggers: -# - workflow_run: fires when release.yml ("Release") or docker.yml ("Docker") completes on a tag -# push. The two run in parallel off the same tag push (tag-on-main.yml), so there is no fixed -# ordering — each check below retries with backoff for a few minutes to ride out the race -# instead of false-failing because the OTHER workflow hasn't finished publishing yet. +# - workflow_run: LARGELY VESTIGIAL NOW, AND SAID SO RATHER THAN QUIETLY LEFT. It used to fire +# when release.yml or docker.yml completed on a tag push, because the two ran in parallel off +# one tag and neither could wait for the other. Neither runs on a tag any more: release.yml +# runs on the main push and CALLS this file twice (in `staging` mode as the pre-promote gate, +# then in public mode as the last step), and docker.yml only runs as a called workflow, which +# emits no workflow_run event of its own. The trigger is kept because it costs nothing and +# still catches a manual dispatch of either workflow, and its `head_branch` starts-with-'v' +# guard means the main-push release runs do not double-fire it. The retry-with-backoff loops +# below are kept for the same reason they always existed: registries and CDNs settle at their +# own pace. # - workflow_dispatch: manual re-check of an already-shipped version. # - schedule (daily): install.sh worked on release day and broke LATER — GitHub's unauthenticated # rate limiting is environmental, not a property of our artifacts, so nothing about the release @@ -73,10 +93,131 @@ name: Verify deploy # instead of just killing the job, and the job carries an overall `timeout-minutes: 60`. This # matters more on the daily schedule than on release: a 6-hour hung run would still be "in # progress" when the next day's run fires. +# +# WHEN THIS RUNS, AND THE HOLE THAT WAS IN IT. The `workflow_run` trigger below is gated on the +# triggering run having SUCCEEDED. During the 1.5.3 release the Release workflow FAILED (two of five +# build legs died), so this verifier never ran at all -- on precisely the release that most needed +# verifying. Every defect it exists to catch was then found BY HAND: install.sh 404ing on Apple +# Silicon, two dead links on the live /download/ page, `docker pull getbusbar/busbar` serving the +# previous version. A verifier that only runs when the build already went well is not a verifier. +# +# THE `workflow_run` DEFAULT-BRANCH TRAP, AND WHY THIS FILE DOES NOT NEED A DISPATCHER. +# `workflow_run` ALWAYS loads the workflow YAML from the repository's DEFAULT BRANCH, whatever +# branch the triggering run was on. That bit qa-gate.yml hard: `main` was over a hundred commits +# behind, so every auto-fired gate ran stale logic, and the repair was to rewrite it as a thin +# DISPATCHER that checks out the triggering SHA and runs the gate scripts from THAT checkout. +# +# This workflow is structurally IMMUNE to the same trap, and the reason is worth stating so nobody +# has to re-derive it or "fix" it by adding a checkout: +# * It has ZERO `uses:` steps, therefore NO `actions/checkout` at all, therefore no working copy +# that could be stale. That is deliberate, and check (h) depends on it: reading ./install.sh +# from a checkout would hide a fix that merged but never deployed. +# * It reads NO repository file. Every path it touches is either created on the runner (the +# digest helper, the quickstart config, both written inline here) or fetched over HTTPS. The one +# repository file it consults, .github/release-targets.json, is fetched from raw.githubusercontent +# AT THE TAG UNDER TEST, not read from a checkout, precisely so it tracks the release rather +# than the default branch. +# * The VERSION under test comes from the event payload (`release.tag_name` / +# `workflow_run.head_branch`) or, on the schedule, from the live /releases/latest redirect. It +# never comes from crates/busbar/Cargo.toml, so a stale default branch cannot make it verify the +# wrong version. +# So the only thing loaded from `main` is this file's own text, and this file's own text has no +# repo-state dependency to go stale against. A dispatcher would add a checkout whose entire purpose +# is to be read, in a workflow whose entire purpose is to read nothing local. +# +# So `release: [published]` is now a trigger in its own right, and under the draft-then-promote order +# it means something sharper than it used to. The release object is created as a DRAFT, which fires +# NO event; the `published` event fires at the moment the draft is promoted. So this trigger now +# means exactly "a release just became visible to users", whoever made it visible and whether or not +# the workflow that made it finished happy. A release promoted by hand gets verified the same as one +# promoted by the pipeline. +# +# -- THIS IS THE LAST STEP OF THE RELEASE, NOT A BYSTANDER --------------------------------------- +# Until now every trigger above fired this workflow BESIDE the release. release.yml's own graph +# ended at `notify-downstream`, nothing in it waited on this, and nothing reported this verdict as +# part of the release's status. So a release could be green, fanned out to the Homebrew tap and the +# Helm chart, and simultaneously unusable, with the only evidence a separate red run in a repo full +# of runs. That is what happened to 1.5.3: it published five of seven assets, `install.sh` 404'd on +# Apple Silicon, and a human found it by hand. +# +# `workflow_call` fixes that. release.yml's FINAL job now calls this file directly, so a failing +# consumer check turns the RELEASE RUN red, on the release's own status, with the failure named in +# the release run's own job list. Two properties make this the right mechanism rather than "have +# release.yml poll for the separate run's conclusion": +# * `uses: ./.github/workflows/verify-deploy.yml` from a tag-triggered caller loads this file AT +# THE TAG, so the release is verified by the verifier that shipped with it. (The other triggers +# load it from the default branch - see the workflow_run note below for why that is harmless +# here: this file reads no repository state.) +# * A called workflow's failure is the caller job's failure. No polling, no timeout heuristics, +# no second source of truth about whether verification passed. +# +# THIS FILE USED TO SAY CONSUMER VERIFICATION WAS POST-PUBLICATION BY NATURE. THAT WAS WRONG, AND +# THE CORRECTION IS THE `stage` INPUT ABOVE. +# +# The argument was: you cannot `docker pull` an image that was never pushed, or `brew install` a +# formula the tap has not bumped, therefore verification cannot gate publication. That conflated two +# different things - the artifact EXISTING, and the artifact being PUBLISHED UNDER THE NAME USERS +# CONSUME - and they are separable. An image pushed as `staging-` exists and pulls. A DRAFT +# release's assets have real, downloadable URLs and can be fetched and executed. Neither is a name +# any user, doc or chart refers to. +# +# So the file now runs in two modes, and the split is by what CAN be true before publication rather +# than by convenience: +# * `stage: staging` - the ARTIFACT checks, run by release.yml BEFORE anything is tagged. These +# DO gate publication: a failure means no version tag is ever created. +# * default (public) - the CHANNEL checks, which genuinely cannot run earlier because the tap, the +# chart, the site and `/releases/latest` only move once the release is public. For these the old +# paragraph still holds: they cannot gate, so instead they +# 1. make the verdict part of the release's own status (RED on the release run), and +# 2. open or update a GitHub issue naming the failing check (see the `alert` job below), +# because a release that is published and unusable is a fact somebody has to be TOLD, and red +# alone is a signal only for whoever happens to be looking at that run. on: + release: + types: [published] workflow_run: workflows: ["Release", "Docker"] types: [completed] + workflow_call: + inputs: + version: + description: "Version to verify (e.g. 1.5.2, no leading v)" + required: true + type: string + # -- STAGING MODE: the same checks, run BEFORE the version has a public name ----------------- + # + # `stage: staging` is how release.yml gates a release on consumer verification instead of + # merely reporting it afterwards. The release object is still a DRAFT and the image is still + # under a throwaway `staging-` tag, so the subset of checks below that can be true at + # that point runs, and the subset that structurally cannot is skipped VISIBLY. + # + # WHAT RUNS IN STAGING MODE: (c) every expected asset is present, plausibly sized and really + # downloadable; (c2) the real linux binary is unpacked and EXECUTED and `--version` agrees; + # (d) the image is pulled FRESH and its OCI version label agrees; (l) the documented quickstart + # boots, both the binary half and the docker one-liner; (m) `gh attestation verify` passes on + # the real bytes. That is the artifact-side half of this file, unchanged and reused rather + # than reimplemented -- a second copy of these checks is exactly the defect this file's own + # comments keep warning about. + # + # WHAT IS SKIPPED IN STAGING MODE, AND WHY IT IS NOT A GAP: (a)(b) `latest` and the version pin + # agreeing, (e) the Terraform registry, (f) the Helm chart appVersion, (g) getbusbar.com, (h)(i) + # install.sh, (j) Homebrew, (k) the site's APIs, (n) the live counters. Every one of those + # reads a DOWNSTREAM channel that only moves once the release is published. Asserting them + # against an unpublished version would not be a stricter gate, it would be an assertion that + # is false by construction. They all still run, unchanged, from release.yml's + # `consumer-verification` job after the promote. + stage: + description: "'public' (default, every check) or 'staging' (artifact-side checks only, pre-promote)" + required: false + type: string + default: public + image_ref: + description: >- + Fully-qualified image to verify instead of getbusbar/busbar:, e.g. + getbusbar/busbar:staging-abc123456789. Staging mode only. + required: false + type: string + default: "" workflow_dispatch: inputs: version: @@ -86,14 +227,444 @@ on: # 13:17 UTC daily. Off the top of the hour on purpose: :00 cron slots are the most contended on # GitHub's shared scheduler and get delayed the most, and mid-day UTC lands in working hours for # both EU and US-East so a red rot alert is seen the day it fires, not the next morning. + # THE FULL SWEEP runs on this one only. - cron: "17 13 * * *" + # 3-HOURLY POINTER SWEEP (the `pointers` job only; `verify` is gated off it below). + # + # WHY A SECOND, FASTER CRON. "if docker or anything isn't latest we need to know right away." + # Daily means up to 24h of `docker pull getbusbar/busbar` handing users the previous release, + # which is exactly what happened across at least two releases when docker.yml's `tags:` block + # emitted `type=semver,pattern={{version}}` and nothing else - docker/metadata-action does NOT + # imply `latest` from a semver pattern, so `latest` stayed frozen wherever a human last put it + # while the comment above the block said "X.Y.Z + latest" the whole time. + # + # WHY 3 HOURS AND NOT HOURLY. The cost side is real but small: the pointer sweep is HEAD requests + # and small JSON reads (no image pull, no build, no brew, no boot), ~1-2 minutes of runner time. + # 3-hourly is 8 runs/day, under 20 minutes of runner time a day, and caps the window in which a + # stale default pointer can go unnoticed at 3h instead of 24h. Hourly would be 3x the runs for a + # 2h improvement on a number already inside the "someone notices this shift" range, and GitHub + # deprioritises high-frequency crons on shared runners, so the nominal interval would not be the + # real one anyway. The release-publication trigger is what makes the common case immediate; this + # cron exists for the case where a pointer rots WITHOUT a release, which is how the Docker `latest` + # freeze survived: nothing about our artifacts changed on the day it broke. + - cron: "23 */3 * * *" +# `issues: write` is for the `alert` job, which is the second half of "make a failure impossible to +# miss": red on the release run, AND an issue that comes and finds a human. `actions: read` lets +# that job read the FAILED job's own log through the API so the issue can quote the exact FAIL / +# ::error:: lines (the failing check, expected, observed) rather than saying "something went wrong, +# go read a log". permissions: contents: read + issues: write + actions: read jobs: + # -- MOVING POINTERS ----------------------------------------------------------------------------- + # A separate job from `verify`, on purpose, and it is the cheap one. + # + # THE DEFECT CLASS. Every channel below has a DEFAULT pointer: the thing a user gets when they do + # not name a version. `docker pull getbusbar/busbar`. `/releases/latest/download/...`. `brew + # install`. `pip install busbar-admin`. `uses: GetBusbar/validate-action@v1`. Each of those + # pointers is written by a DIFFERENT publish step in a DIFFERENT repo, and each one can silently + # fail to move while the version-pinned artifact beside it publishes perfectly. When that happens + # nothing is red anywhere: the pinned thing exists, the release is green, and users quietly get + # old code. `docker pull getbusbar/busbar` served the previous release to tens of thousands of + # pulls that way, across at least two releases. + # + # THE RULE THIS JOB ENFORCES: for every channel, the pointer a user gets by DEFAULT must resolve to + # the newest thing that channel's repo actually published. Two families, because busbar's + # distribution is deliberately mixed-model: + # * TRACKS BUSBAR'S VERSION - docker.io/ghcr.io `:latest`, github `/releases/latest`, the Homebrew + # tap, the helm chart's `appVersion`, the download page. These must equal the release under test. + # * INDEPENDENT SEMVER - the three SDKs (PyPI/npm/Go), the Terraform provider, validate-action's + # `@v1`. These do NOT mirror busbar's version and asserting they do is simply wrong (it was, and + # it red-failed check (e)). For these the invariant is registry-latest == that repo's own newest + # published tag, which catches the real defect ("we tagged it and the publish job never ran") + # without false-failing on the legitimate no-op-release case. + # + # DIGESTS, NOT TAG NAMES, AND NEVER A LOCAL IMAGE. The registry assertions compare MANIFEST DIGESTS + # over the Distribution API. They deliberately do not `docker run ... --version`: a local image + # cache will happily answer with the OLD image for the SAME tag and report a stale `latest` as + # fresh (or a fresh one as stale). This job pulls nothing at all, so it cannot be fooled that way; + # check (d) in the `verify` job, which does need a real image, deletes its local copy first. + pointers: + name: moving pointers (every default a user gets is the newest release) + runs-on: ubuntu-latest + # NOT IN STAGING MODE. Every assertion in this job is "the default pointer resolves to the + # NEWEST PUBLISHED release". Pre-promote the release under test is not published, so the honest + # answer to every one of these is the PREVIOUS version and the job would fail on a release that + # is doing exactly the right thing. release.yml runs it, unchanged, after the promote. + if: ${{ inputs.stage != 'staging' }} + # HEAD requests and small JSON reads only. If this has not finished in 20 minutes something is + # hanging, and on a 3-hourly cron a hung run must die well before the next one fires. + timeout-minutes: 20 + outputs: + version: ${{ steps.sweep.outputs.version }} + env: + DOCKERHUB_IMAGE: getbusbar/busbar + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Every default pointer must resolve to the newest published release + id: sweep + # THE ONLY `${{ }}` IN THIS STEP LIVES HERE, AND IT HAS TO. A `run:` block containing any + # `${{ }}` is compiled as ONE expression, and GitHub caps an expression at 21000 characters + # - which this script comfortably exceeds, so the whole WORKFLOW fails to parse with + # "Exceeded max expression length" and cannot even be dispatched. Hoisting the interpolation + # into `env:` leaves the script as a plain literal with no length limit. It is also the safer + # shape regardless: an event-supplied value reaches the shell as an environment variable + # rather than being pasted into the script text. + env: + SUPPLIED_VERSION: ${{ inputs.version || github.event.release.tag_name || '' }} + run: | + # `set +e` FIRST, AND IT IS LOAD-BEARING. GitHub runs every `run:` block as + # `bash -e {0}`, so errexit is ALREADY ON before the first line of the script: writing + # `set -uo pipefail` does not turn it off, it just leaves it on. Without the explicit + # `set +e` this sweep would abort at the first non-zero command and report ONE stale + # channel while hiding the rest, and "docker is stale" and "docker AND homebrew AND helm + # are stale" are different incidents. It cost the alert job a red run to find this. + set +e + set -uo pipefail + fail=0 + : > /tmp/pointer-failures.md + + record() { # record + echo "FAIL: $1 | expected: $2 | observed: $3" + { + echo "- **$1**" + echo " - expected: \`$2\`" + echo " - observed: \`$3\`" + echo " - $4" + } >> /tmp/pointer-failures.md + echo "::error::STALE MOVING POINTER: $1 - expected '$2', observed '$3'. $4" + fail=1 + } + declared() { # declared -- a channel with NO meaningful "latest" + echo "NOT APPLICABLE: $1 -- $2" + } + + # Same anonymous pull-token -> HEAD manifest -> Docker-Content-Digest flow the `verify` job + # uses. It is duplicated rather than shared because jobs cannot share a file without an + # artifact round-trip, and this is a pure function of its arguments with no repository + # state in it - the duplication that is dangerous is a duplicated FACT (a platform list), + # not a duplicated pure function. + reg_digest() { # reg_digest + local auth_host="$1" reg_host="$2" repo="$3" tag="$4" token_url token + if [ "$auth_host" = "auth.docker.io" ]; then + token_url="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" + else + token_url="https://${auth_host}/token?service=${auth_host}&scope=repository:${repo}:pull" + fi + token="$(curl -fsS --max-time 30 "$token_url" | jq -r '.token // .access_token' 2>/dev/null)" + [ -n "${token:-}" ] && [ "$token" != "null" ] || return 1 + curl -fsS --max-time 30 -I \ + -H "Authorization: Bearer $token" \ + -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json' \ + "https://${reg_host}/v2/${repo}/manifests/${tag}" \ + | tr -d '\r' | grep -i '^docker-content-digest:' | awk '{print $2}' + } + + # Newest published tag of a repo, from the RELEASES list - deliberately NOT from + # /releases/latest, which is itself one of the pointers under test. Asking the pointer what + # the newest release is and then checking the pointer against that answer is a check that + # can never fail. + newest_tag() { # newest_tag + local out + out="$(gh api --paginate "repos/$1/releases" \ + --jq '.[] | select(.draft==false and .prerelease==false) | .tag_name' 2>/dev/null \ + | sed 's/^v//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1)" + [ -n "$out" ] && { printf '%s\n' "$out"; return 0; } + # FALL BACK TO GIT TAGS, and this is not a nicety. The three SDK repos (busbar-python, + # busbar-js, busbar-go) publish to PyPI/npm/the Go proxy off a pushed TAG and create no + # GitHub Release at all. Reading releases only, this function returned empty for all + # three, and the caller then declared them "no releases yet, nothing to assert" -- three + # live, shipping, user-facing channels silently exempted from the sweep while PyPI, npm + # and proxy.golang.org were all serving 0.4.0. A pointer check that quietly excuses the + # channels it cannot read is worse than one that is absent, because it looks covered. + gh api --paginate "repos/$1/tags" --jq '.[].name' 2>/dev/null \ + | sed 's/^v//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 + } + + # -- Which busbar release is under test ------------------------------------------------ + SUPPLIED="${SUPPLIED_VERSION#v}" + NEWEST="$(newest_tag GetBusbar/busbar)" + if [ -z "${NEWEST:-}" ]; then + echo "::error::could not enumerate GetBusbar/busbar's published releases; every pointer assertion below would be vacuous, so this fails rather than passes." + exit 1 + fi + if [ -n "$SUPPLIED" ] && [ "$SUPPLIED" != "$NEWEST" ]; then + # Not a failure by itself (a re-verify of an older version is legitimate), but the + # pointers are only ever asserted against the newest release, so say which one won. + echo "note: supplied version ${SUPPLIED} is not the newest published release (${NEWEST}); pointers are asserted against ${NEWEST}." + fi + V="$NEWEST" + TAG="v$V" + # Written BEFORE any assertion runs, on purpose: the `alert` job needs the version to + # title the issue, and it only ever reads this output when this step has FAILED. + echo "version=$V" >> "$GITHUB_OUTPUT" + echo "Newest published busbar release: ${TAG}. Every default pointer below must resolve to it." + + # -- P1/P2: container registries. THE 1.5.3 BUG, EXACTLY. ------------------------------ + # Retried because on release day docker.yml and this can race; a stale pointer is + # permanent and survives the retries, a race resolves inside them. + dh_ver="" dh_latest="" ghcr_ver="" ghcr_latest="" + for i in $(seq 1 10); do + dh_ver="$(reg_digest auth.docker.io registry-1.docker.io "$DOCKERHUB_IMAGE" "$V" || true)" + dh_latest="$(reg_digest auth.docker.io registry-1.docker.io "$DOCKERHUB_IMAGE" latest || true)" + [ -n "$dh_ver" ] && [ "$dh_ver" = "$dh_latest" ] && break + echo " ...docker.io :latest not yet == :${V} (attempt $i/10), retrying in 15s" + sleep 15 + done + if [ -z "$dh_ver" ]; then + record "docker.io ${DOCKERHUB_IMAGE}:${V}" "a resolvable manifest digest" "" \ + "The version-pinned image was never pushed. Fix: re-run docker.yml for ${TAG}." + elif [ "$dh_latest" = "$dh_ver" ]; then + echo "PASS: docker.io ${DOCKERHUB_IMAGE}:latest == :${V} (${dh_latest})" + else + record "docker.io ${DOCKERHUB_IMAGE}:latest" "$dh_ver (the :${V} digest)" "${dh_latest:-}" \ + "\`docker pull ${DOCKERHUB_IMAGE}\` -- the command in the README, the docs and on the site - is serving a DIFFERENT image than ${TAG}. This is the exact 1.5.3 defect: docker/metadata-action does not imply \`latest\` from \`type=semver,pattern={{version}}\`, so \`latest\` froze wherever a human last set it. Fix: confirm docker.yml's \`tags:\` block still emits an explicit \`type=raw,value=latest\` (gated on a real release), then re-run it for ${TAG}." + fi + for i in $(seq 1 10); do + ghcr_ver="$(reg_digest ghcr.io ghcr.io getbusbar/busbar "$V" || true)" + ghcr_latest="$(reg_digest ghcr.io ghcr.io getbusbar/busbar latest || true)" + [ -n "$ghcr_ver" ] && [ "$ghcr_ver" = "$ghcr_latest" ] && break + echo " ...ghcr.io :latest not yet == :${V} (attempt $i/10), retrying in 15s" + sleep 15 + done + if [ -z "$ghcr_ver" ]; then + record "ghcr.io/getbusbar/busbar:${V}" "a resolvable manifest digest" "" \ + "Fix: re-run docker.yml's ghcr push for ${TAG}." + elif [ "$ghcr_latest" != "$ghcr_ver" ]; then + record "ghcr.io/getbusbar/busbar:latest" "$ghcr_ver (the :${V} digest)" "${ghcr_latest:-}" \ + "Users pulling from GHCR without a tag get a different image than ${TAG}. Same fix as the Docker Hub case." + else + echo "PASS: ghcr.io/getbusbar/busbar:latest == :${V} (${ghcr_latest})" + fi + # And the two registries must agree, or "latest" means two different things depending on + # which registry you happened to pull from. + if [ -n "$dh_ver" ] && [ -n "$ghcr_ver" ] && [ "$dh_ver" != "$ghcr_ver" ]; then + record "ghcr.io vs docker.io for :${V}" "$dh_ver" "$ghcr_ver" \ + "The same tag resolves to DIFFERENT images on the two registries, so which bytes a user runs depends on which registry they pulled from. Fix: docker.yml copies the manifest cross-registry; re-run it for ${TAG}." + fi + + # -- P3: the GitHub /releases/latest redirect, and its assets -------------------------- + # Every download button on getbusbar.com, and install.sh, follow this. It 404s outright if + # the newest Release is not flagged 'latest', and silently serves the PREVIOUS release's + # bytes if the newest one never published. + loc="$(curl -fsS --max-time 30 -o /dev/null -w '%{redirect_url}' \ + "https://github.com/GetBusbar/busbar/releases/latest" || true)" + if [ "${loc##*/releases/tag/}" = "$TAG" ]; then + echo "PASS: github.com/GetBusbar/busbar/releases/latest -> ${TAG}" + else + record "github.com/GetBusbar/busbar/releases/latest" ".../releases/tag/${TAG}" "${loc:-}" \ + "install.sh and every download button on getbusbar.com resolve through this redirect, so all of them are handing users the wrong release. Fix: mark Release ${TAG} as 'latest' (it is probably still a draft or flagged prerelease)." + fi + # A redirect that lands in the right place still proves nothing if the assets behind it are + # not there. THIS IS THE 1.5.3 FIVE-OF-SEVEN CASE: the Release existed, was flagged latest, + # and `curl install.sh | sh` still 404'd on Apple Silicon because that platform's tarball + # was never uploaded. The expected names come from the release's OWN manifest at the tag, + # never from a list typed here. + # The manifest is fetched at the TAG first so the expectation tracks the release that + # produced the assets. Tags cut before the manifest existed do not carry it, and it has not + # reached `main` yet either, so the chain degrades tag -> main -> dev. Reading it from a + # branch is weaker (the platform set could have moved since the tag) but it is the same + # trade check (c) already makes, and the platform set changes rarely - whereas HARDCODING + # a list here would be the exact defect the manifest was introduced to remove. + got_manifest=0 + for ref in "${TAG}" main dev; do + if curl -fsS --max-time 30 \ + "https://raw.githubusercontent.com/GetBusbar/busbar/${ref}/.github/release-targets.json" \ + -o /tmp/pt-targets.json 2>/dev/null; then + [ "$ref" = "$TAG" ] || echo "note: ${TAG} does not carry .github/release-targets.json; using ${ref}'s copy." + got_manifest=1 + break + fi + done + if [ "$got_manifest" = 1 ]; then + mapfile -t want < <(python3 - "$TAG" <<'PY' + import json, sys + spec = json.load(open("/tmp/pt-targets.json")) + for t in spec["targets"]: + print("busbar-%s.%s" % (t["target"], t["archive"])) + PY + ) + if [ "${#want[@]}" -lt 5 ]; then + record "release-targets manifest at ${TAG}" ">= 5 platform archives" "${#want[@]}" \ + "Refusing to 'verify' the latest-download path against an empty expectation list: that passes for a release that published nothing." + fi + for a in "${want[@]}"; do + u="https://github.com/GetBusbar/busbar/releases/latest/download/${a}" + code="$(curl -sSL --max-time 60 --range 0-0 -o /dev/null -w '%{http_code}' "$u" || echo 000)" + case "$code" in + 200|206) echo "PASS: /releases/latest/download/${a} -> ${code}" ;; + *) record "/releases/latest/download/${a}" "HTTP 200/206" "HTTP ${code}" \ + "This is the platform-specific 404 that broke \`curl -fsSL https://getbusbar.com/install.sh | sh\` on Apple Silicon for the whole 1.5.3 release. A user on that platform gets nothing. Fix: find that target's leg in release.yml's \`upload-assets\` matrix, fix it, and re-upload the asset to ${TAG}." ;; + esac + done + else + record ".github/release-targets.json (tried ${TAG}, main, dev)" "fetchable over raw.githubusercontent" "" \ + "Without it this check cannot know which platforms ${TAG} owed, and guessing is how a missing platform got waved through in the first place. Fix: restore .github/release-targets.json - release.yml's own \`targets\` job reads the same file, so if it is really gone the release matrix is broken too." + fi + + # -- P4: Homebrew tap ------------------------------------------------------------------ + fver="$(curl -fsSL --max-time 30 \ + "https://raw.githubusercontent.com/GetBusbar/homebrew-busbar/main/Formula/busbar.rb" 2>/dev/null \ + | grep -m1 -E '^ *version "' | sed -E 's/.*version "([^"]+)".*/\1/' || true)" + if [ "$fver" = "$V" ]; then + echo "PASS: Homebrew tap formula version == ${V}" + else + record "Homebrew tap Formula/busbar.rb version" "$V" "${fver:-}" \ + "\`brew install getbusbar/busbar/busbar\` -- the documented command - installs an old binary. Fix: run/repair the tap's bump.yml workflow." + fi + + # -- P5: published Helm chart appVersion ----------------------------------------------- + appver="$(curl -fsS --max-time 60 "https://getbusbar.github.io/helm-charts/index.yaml" 2>/dev/null \ + | awk '/^ busbar:/{f=1} f && /appVersion:/{print $2; exit}' | tr -d '"' || true)" + if [ "$appver" = "$V" ]; then + echo "PASS: helm-charts busbar appVersion == ${V}" + else + record "GetBusbar/helm-charts busbar chart appVersion" "$V" "${appver:-}" \ + "\`helm install busbar getbusbar/busbar\` deploys an old gateway. Fix: run/repair helm-charts' release workflow." + fi + + # -- P6: what the site presents as current --------------------------------------------- + # Anchored: an unanchored substring match once passed v1.5.2 against a page advertising + # v1.5.20. + # NEVER `curl ... | grep -q` UNDER pipefail. `grep -q` exits the instant it matches, which + # closes the pipe, which kills curl with SIGPIPE (exit 23), which `pipefail` then reports + # as a failed pipeline - so the check goes RED EXACTLY WHEN THE ASSERTION HOLDS and green + # only when the page is missing the version. That inversion is live in check (g) below and + # is fixed there too. Fetch to a file, then grep the file. + # "COULD NOT READ THE PAGE" AND "THE PAGE IS STALE" ARE DIFFERENT INCIDENTS and must not + # share a message. The first run of this check on a GitHub runner reported "v1.5.3 not + # present on the page" for a page that plainly shows v1.5.3 from a laptop: the runner's + # datacenter IP gets a different response (bot challenge / edge block) than a browser. A + # check that says "the site is stale" when it means "I was blocked" sends whoever reads + # the issue to redeploy a site that is fine, and the second time it does that, everyone + # stops believing it. So: send the User-Agent a real visitor sends, and report the HTTP + # code and body size when the page cannot be read as a page, distinct from staleness. + dl_code="$(curl -sSL --max-time 30 \ + -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36' \ + -H 'Accept: text/html,application/xhtml+xml' \ + -o /tmp/pt-download.html -w '%{http_code}' "https://getbusbar.com/download/" 2>/dev/null || echo 000)" + dl_size="$(wc -c < /tmp/pt-download.html 2>/dev/null || echo 0)" + # 403/429 IS NOT THE SAME INCIDENT AS 404/5xx, and conflating them is why this check has + # been red on main every day. A 403 is aimed at US: getbusbar.com sits behind an edge that + # challenges datacenter IPs, so a GitHub runner is refused while a browser is served + # normally. That means the check COULD NOT RUN, not that the assertion failed - and + # reporting "the site is stale" for a site that is fine is how a gate gets ignored. It is a + # VISIBLE SKIP, announced three ways (log, annotation, step summary), never a silent one. + # Anything else non-200 is broken for real users too, and stays a hard failure. + case "${dl_code}:$([ "$dl_size" -lt 500 ] && echo small || echo ok)" in + 200:ok) dl_state=readable ;; + 403:*|429:*) dl_state=blocked ;; + *) dl_state=broken ;; + esac + if [ "$dl_state" = "blocked" ]; then + echo "VISIBLE SKIP: getbusbar.com/download/ returned HTTP ${dl_code} to this runner, so the advertised-version assertion could NOT be evaluated. This is an edge/bot block against GitHub's datacenter IPs, not a stale page: a browser is served normally." + echo "::warning::(pointers) PARTIAL: getbusbar.com/download/ returns HTTP ${dl_code} to GitHub Actions runners, so the 'site advertises the current version' assertion did not run. Every other pointer WAS checked and is authoritative. Fix is MARKETING-SIDE: allow GitHub Actions egress through the Cloudflare bot rules for /download/ (or expose a small unchallenged JSON route carrying the current version), so this assertion can be made from CI at all." + { echo "### Moving pointers: PARTIAL - getbusbar.com/download/ returns HTTP ${dl_code} to this runner, so the advertised-version check did not run"; } >> "$GITHUB_STEP_SUMMARY" + elif [ "$dl_state" = "broken" ]; then + record "getbusbar.com/download/ availability" "HTTP 200 and a real HTML page" "HTTP ${dl_code}, ${dl_size} bytes" \ + "The download page is not being served at all. Unlike a 403, this is broken for every visitor, not just for CI. Fix: redeploy the marketing site." + elif grep -qE "v${V}([^0-9]|\$)" /tmp/pt-download.html; then + echo "PASS: getbusbar.com/download/ advertises v${V}" + else + record "getbusbar.com/download/ advertised version" "v${V}" \ + "" \ + "The site tells visitors the current release is something other than ${TAG}. Fix: redeploy the marketing site." + fi + + # -- P7-P10: INDEPENDENT-SEMVER channels. The invariant is registry-latest == that repo's + # own newest published tag, NOT == busbar's version. ---------------------------------- + check_independent() { # check_independent