docker.yml

name: Docker

# Build and publish the busbar container image (Docker Hub + GHCR) as a multi-arch
# manifest (linux/amd64 + linux/arm64), FROM scratch over static musl binaries.
#
# Triggers:
#   - v* tags: publish with the semver tag cascade (X.Y.Z, X.Y, X, latest)
#   - workflow_dispatch: publish a `test` tag only — end-to-end pipeline check
#     without cutting a release.
#
# Supply chain: both registries carry a GitHub build-provenance attestation
# (stored by GitHub — no extra tags on the repo). Verify with:
#   gh attestation verify oci://index.docker.io/getbusbar/busbar:<tag> --repo GetBusbar/busbar
# Cosign signatures are additionally pushed for the GHCR image only — cosign
# stores signatures as sha256-*.sig tags, which we keep off the Docker Hub tag
# list deliberately (clean public page; GHCR carries them for cosign users):
#   cosign verify ghcr.io/getbusbar/busbar:<tag> \
#     --certificate-identity-regexp 'https://github.com/GetBusbar/busbar/\.github/workflows/docker\.yml@.*' \
#     --certificate-oidc-issuer https://token.actions.githubusercontent.com

on:
  push:
    tags:
      - "v*"
      - "!v*-*" # NEVER publish a pre-release tag (e.g. v1.5.2-rc.1) — qa staging markers only.
  workflow_dispatch:
    inputs:
      version:
        description: >-
          Publish as this version (e.g. 1.1.0) with the full semver tag cascade +
          latest. Only use when the dispatched ref is source-identical to that
          release. Leave empty to publish only a `test` tag.
        required: false
        default: ""
      retag_from:
        description: >-
          Point `latest` at an ALREADY-PUBLISHED version (e.g. 1.1.1) with a
          manifest-only retag — no rebuild, and the immutable X.Y.Z tag is never
          re-pushed. Skips the build. Leave empty for a normal build.
        required: false
        default: ""

env:
  DOCKERHUB_IMAGE: getbusbar/busbar
  GHCR_IMAGE: ghcr.io/getbusbar/busbar
  # Pinned ref for the external first-party headroom-hook plugin repo (GetBusbar/headroom-hook),
  # bundled pre-installed into the image below. `dev` (not `main`) carries the sibling-relative-path
  # dependency fix this build needs — main stays release-only, ongoing work lands on dev, same split
  # as busbar's own main/dev. Same pin as .github/workflows/release.yml's HEADROOM_HOOK_REF — keep
  # the two in sync when re-pinning.
  HEADROOM_HOOK_REF: dev

permissions:
  contents: read
  packages: write      # push to GHCR
  id-token: write      # keyless cosign signing
  attestations: write  # build-provenance attestation

jobs:
  # Manifest-only retag: point `latest` at an already-published version, no rebuild.
  # Used when version tags are immutable — the immutable X.Y.Z is never re-pushed.
  retag-latest:
    name: point latest at ${{ inputs.retag_from }}
    if: ${{ inputs.retag_from != '' }}
    runs-on: ubuntu-latest
    steps:
      - uses: docker/setup-buildx-action@v4
      - name: Log in to Docker Hub
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Log in to GHCR
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Retag latest -> ${{ inputs.retag_from }} (multi-arch preserved)
        env:
          V: ${{ inputs.retag_from }}
        run: |
          set -euo pipefail
          # Docker Hub: point latest at the (immutable) version. Idempotent.
          docker buildx imagetools create -t "${DOCKERHUB_IMAGE}:latest" "${DOCKERHUB_IMAGE}:${V}"
          # GHCR may be MISSING this version if a prior combined build-push aborted mid-way
          # (e.g. Docker Hub tag-immutability rejected `latest` before GHCR finished). Mirror the
          # version from Docker Hub into GHCR (cross-registry manifest copy, no rebuild), then move
          # GHCR latest onto it. `imagetools create` is a no-op-safe overwrite if it already exists.
          docker buildx imagetools create -t "${GHCR_IMAGE}:${V}" "${DOCKERHUB_IMAGE}:${V}"
          docker buildx imagetools create -t "${GHCR_IMAGE}:latest" "${GHCR_IMAGE}:${V}"
          echo "== Docker Hub latest =="; docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:latest" | grep -iE 'name|platform'
          echo "== GHCR ${V} =="; docker buildx imagetools inspect "${GHCR_IMAGE}:${V}" | grep -iE 'name|platform'

  # Static musl binaries, one per architecture, built on native runners (no QEMU).
  build-binaries:
    name: musl ${{ matrix.arch }}
    if: ${{ inputs.retag_from == '' }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: true
      matrix:
        include:
          - arch: amd64
            os: ubuntu-latest
            target: x86_64-unknown-linux-musl
          - arch: arm64
            os: ubuntu-24.04-arm
            target: aarch64-unknown-linux-musl
    steps:
      - uses: actions/checkout@v7

      - name: Install musl toolchain
        run: sudo apt-get update && sudo apt-get install -y musl-tools

      - uses: dtolnay/rust-toolchain@stable
        with:
          targets: ${{ matrix.target }}

      - uses: Swatinem/rust-cache@v2
        with:
          key: ${{ matrix.target }}

      # PGO release build (scripts/pgo-build.sh): instrumented build -> on-host training (native-arch
      # runner, so the musl-static instrumented binary executes locally) -> optimized build.
      # FAIL-CLOSED: PGO is MANDATORY. If any PGO phase fails the script exits non-zero and this
      # step (and the job) FAILS - it never falls back to a plain `cargo build --release`. On
      # success the script writes a proof marker at target/pgo/<target>/release/busbar.pgo-verified
      # which the verification step below asserts, so a non-PGO binary can never ship green.
      - name: Build static binary (PGO, required/fail-closed)
        env:
          # The busbar release ed25519 PUBLIC key, embedded at build time (plugin-sign's
          # option_env!) so the IMAGE binary verifies first-party signed plugins exactly like the
          # release.yml binaries do. Omitting it here was the gap that shipped Docker images unable
          # to verify any plugin (headroom-hook bundled-image smoke test, 2026-08-02).
          BUSBAR_RELEASE_PUBKEY: ${{ vars.BUSBAR_RELEASE_PUBKEY }}
          PGO_TARGET: ${{ matrix.target }}
          # musl-gcc drives both the C compiles (bundled SQLite) and the final link.
          CC: musl-gcc
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: musl-gcc
          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER: musl-gcc
          # jemalloc-sys (new in 1.4.0) runs jemalloc's autoconf `configure`, which sees the
          # gnu-build / musl-host triplet mismatch and believes it is CROSS-COMPILING — so its
          # RUN-based atomics probes default to "no" and the C build dies with
          # `atomic.h: #error "Don't have atomics implemented on this platform"`. It is NOT a real
          # failure: every atomics backend compiles+links fine under musl-gcc (verified). Seed the
          # autoconf cache vars with the correct answers so configure uses them instead of the
          # cross-compile default. (Without this the FROM-scratch musl image cannot build with jemalloc.)
          je_cv_c11_atomics: "yes"
          je_cv_gcc_atomic_atomics: "yes"
          je_cv_gcc_u8_atomic_atomics: "yes"
          je_cv_gcc_sync_atomics: "yes"
          je_cv_gcc_u8_sync_atomics: "yes"
        run: |
          set -euo pipefail
          scripts/pgo-build.sh
          file "target/pgo/${{ matrix.target }}/release/busbar"
          # Fail loudly if the binary is not fully static - FROM scratch has no loader.
          if ldd "target/pgo/${{ matrix.target }}/release/busbar" 2>&1 | grep -qv 'not a dynamic\|statically linked'; then
            echo "binary is dynamically linked; refusing to ship in FROM scratch" && exit 1
          fi

      # POSITIVE PGO GATE: pgo-build.sh writes this marker ONLY after a non-empty merged profile
      # was fed to a successful -Cprofile-use build. Asserting it here (not just the script's exit
      # code) means a release cannot ship a non-PGO binary and still pass. Any missing/empty/
      # unverified marker fails the job and blocks the release.
      - name: Verify PGO was applied (marker gate)
        run: |
          set -euo pipefail
          marker="target/pgo/${{ matrix.target }}/release/busbar.pgo-verified"
          if [ ! -s "$marker" ]; then
            echo "::error::PGO proof marker missing or empty at $marker - refusing to ship a non-PGO binary" >&2
            exit 1
          fi
          echo "--- PGO proof marker ---"
          cat "$marker"
          grep -q '^pgo-verified=1$' "$marker" || { echo "::error::marker not marked verified" >&2; exit 1; }
          bytes="$(grep '^profile_bytes=' "$marker" | cut -d= -f2)"
          raw="$(grep '^profraw_count=' "$marker" | cut -d= -f2)"
          if [ -z "$bytes" ] || [ "$bytes" -le 0 ] 2>/dev/null; then
            echo "::error::merged profile was empty (profile_bytes=$bytes) - build was not PGO-optimized" >&2
            exit 1
          fi
          if [ -z "$raw" ] || [ "$raw" -le 0 ] 2>/dev/null; then
            echo "::error::no .profraw files fed the profile (profraw_count=$raw) - build was not PGO-optimized" >&2
            exit 1
          fi
          echo "PGO verified: ${bytes} bytes of merged profile from ${raw} .profraw file(s)."

      - uses: actions/upload-artifact@v7
        with:
          name: busbar-${{ matrix.arch }}
          path: target/pgo/${{ matrix.target }}/release/busbar
          if-no-files-found: error

      # ── headroom-hook: build + sign for this same musl target, so the image below can ship it
      # pre-installed (see the Dockerfile). Same first-party source repo and checkout as
      # .github/workflows/release.yml's hook-plugins job — see the comments there for the full
      # rationale (pin choice). headroom-hook's Cargo.toml declares its busbar-plugin-sdk (and
      # dev-dependency) local path deps as sibling-relative, same convention webrequest-hook
      # already uses, so no CI-side patching is needed — the container mounts below (`/src/headroom-hook`
      # + `/src/busbarAI`) give it that sibling layout regardless of the HOST-side checkout nesting.
      # Only headroom is bundled: it needs zero operator config
      # (local-only BM25 compression), whereas webrequest-hook requires an operator-supplied
      # forwarding URL and so has no sensible zero-config default — it is NOT pre-installed in the
      # image.
      # `path:` must stay INSIDE the workspace (actions/checkout@v7 refuses a `../`-escaping path:
      # "Repository path '...' is not under '...'") -- busbar's own checkout above has no `path:`
      # (lands at the workspace root), so this becomes a subdirectory of that checkout rather than
      # a true sibling. Safe here because busbar's own build (PGO, above) already completed before
      # this checkout happens, so nothing scans this directory as part of busbar's own workspace.
      - name: Checkout headroom-hook (first-party plugin source, pinned)
        uses: actions/checkout@v7
        with:
          repository: GetBusbar/headroom-hook
          ref: ${{ env.HEADROOM_HOOK_REF }}
          path: headroom-hook

      # Build headroom-hook's cdylib INSIDE a musl-native `rust:alpine` container rather than
      # cross-compiling from this (glibc) Ubuntu host with musl-gcc, for two verified reasons:
      #
      # 1. headroom-core pulls in `tokenizers` -> `esaxx-rs`, which needs a C++ compiler. Ubuntu's
      #    `musl-tools` package provides `musl-gcc` (C only) — no musl-targeted C++ toolchain, and
      #    plain `g++` on this host targets glibc, not musl. Alpine ships a real musl-native g++.
      #
      # 2. VERIFIED FAILURE: `aarch64-unknown-linux-musl` does not support `crate-type = ["cdylib"]`
      #    under Rust's DEFAULT (crt-static-on) linking mode for this target — the build silently
      #    drops the cdylib output ("dropping unsupported crate type `cdylib`") and produces only an
      #    .rlib. Building with `RUSTFLAGS="-C target-feature=-crt-static"` fixes this (produces a
      #    real cdylib) but switches it to DYNAMIC linking against musl's libc/libgcc_s/libstdc++ —
      #    which the FROM-scratch busbar image does not otherwise contain, so dlopen() would fail at
      #    runtime with those libs missing. Building inside `rust:alpine` (musl-native) means the
      #    exact matching runtime libs (from the SAME musl/gcc userland the plugin was linked
      #    against) can be extracted from the build container itself and bundled into the image
      #    (see the "Extract runtime libs" step and the Dockerfile) — this was tested end-to-end
      #    (a real `FROM scratch` image, only the busbar binary + these libs + the plugin tarball,
      #    successfully dlopen'd headroom and passed `busbar --validate`).
      #
      # JUDGMENT CALL: -crt-static is disabled (and the runtime libs bundled) UNIFORMLY for BOTH
      # musl targets here, not just aarch64, even though x86_64-unknown-linux-musl did not show the
      # "unsupported crate type" failure in ad hoc testing — that test was inconclusive (a QEMU
      # emulation crash unrelated to cdylib support cut it short before reaching a clean signal).
      # Applying the same, PROVEN mechanism to both architectures is the safer choice: it does not
      # depend on an assumption about x86_64-musl's default-linking cdylib support that was never
      # cleanly confirmed either way.
      - name: Build headroom-hook cdylib + extract matching runtime libs (musl-native container)
        run: |
          set -euo pipefail
          mkdir -p headroom-libs
          docker run --rm \
            -v "$PWD/headroom-hook":/src/headroom-hook \
            -v "$PWD":/src/busbarAI:ro \
            -v "$PWD/headroom-libs":/out \
            rust:alpine sh -exc '
              apk add --no-cache musl-dev sqlite-dev pkgconfig gcc g++ git perl make
              export CARGO_HOME=/tmp/cargo-home
              cd /src/headroom-hook
              RUSTFLAGS="-C target-feature=-crt-static" cargo build --release
              cp target/release/libheadroom_hook.so /out/libheadroom_hook.so
              cp /usr/lib/libstdc++.so.6 /usr/lib/libgcc_s.so.1 /out/
              cp /lib/libc.musl-*.so.1 /out/
              chmod -R a+rwX /out
            '
          ls -l headroom-libs
        shell: bash

      - name: Build busbar-plugin-pack (host target, build tool only)
        run: cargo build --release -p busbar-plugin-pack

      - name: Package + sign the headroom-hook plugin tarball
        env:
          BUSBAR_SIGN_KEY: ${{ secrets.BUSBAR_SIGN_KEY }}
        run: |
          set -euo pipefail
          # Same version resolution as the publish job's "Resolve version" step (duplicated here,
          # not shared, since this runs in a different job): the workflow_dispatch `version` input,
          # else the pushed v* tag, else a placeholder for a bare `test`-tag dispatch build.
          ver="${{ inputs.version }}"
          if [ -z "$ver" ] && [ "$GITHUB_REF_TYPE" = "tag" ]; then ver="${GITHUB_REF_NAME#v}"; fi
          [ -n "$ver" ] || ver="0.0.0-test"
          outdir="plugin-dist"; mkdir -p "$outdir"
          unsigned_flag=""
          if [ -z "${BUSBAR_SIGN_KEY:-}" ]; then
            echo "::warning::BUSBAR_SIGN_KEY secret is not provisioned — packaging an UNSIGNED headroom plugin tarball (loadable only under plugins.trust.allow_unsigned). See the TODO(release-keys) seam in release.yml."
            unsigned_flag="--allow-unsigned"
          fi
          target/release/busbar-plugin-pack pack \
            --lib "headroom-libs/libheadroom_hook.so" \
            --name "busbar-headroom" --alias "headroom" --kind hook \
            --version "$ver" --publisher busbar \
            --needs-prompt rw \
            --description "busbar headroom hook plugin: deterministic rule-based prompt-compression rewrite gate" \
            --license Apache-2.0 \
            --out "${outdir}/busbar-headroom-${{ matrix.arch }}.tar.gz" \
            $unsigned_flag
          ls -l "$outdir"
        shell: bash

      # The runtime libs (libc.musl-*.so.1, libgcc_s.so.1, libstdc++.so.6) the cdylib above needs at
      # dlopen time — bundled alongside the plugin tarball so the Dockerfile can COPY them into
      # /lib. Named per-arch so the publish job (single image build spanning both arches) can stage
      # each into its own arch-specific Docker build context directory.
      - uses: actions/upload-artifact@v7
        with:
          name: busbar-headroom-libs-${{ matrix.arch }}
          path: |
            headroom-libs/libstdc++.so.6
            headroom-libs/libgcc_s.so.1
            headroom-libs/libc.musl-*.so.1
          if-no-files-found: error

      - uses: actions/upload-artifact@v7
        with:
          name: busbar-headroom-${{ matrix.arch }}
          path: plugin-dist/busbar-headroom-${{ matrix.arch }}.tar.gz
          if-no-files-found: error

  publish:
    name: build & push image
    needs: build-binaries
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Stage binaries + plugins into build context
        uses: actions/download-artifact@v8
        with:
          pattern: busbar-*
          path: staged

      - name: Arrange per-arch layout
        run: |
          mkdir -p binaries/amd64 binaries/arm64 plugins/amd64/lib plugins/arm64/lib
          mv staged/busbar-amd64/busbar binaries/amd64/busbar
          mv staged/busbar-arm64/busbar binaries/arm64/busbar
          chmod +x binaries/amd64/busbar binaries/arm64/busbar
          # headroom-hook, pre-installed into the image (see Dockerfile) — one signed tarball per arch,
          # built by build-binaries above. webrequest-hook is intentionally NOT bundled: see the
          # comment on the "headroom-hook" build steps in build-binaries for why.
          mv staged/busbar-headroom-amd64/busbar-headroom-amd64.tar.gz plugins/amd64/busbar-headroom.tar.gz
          mv staged/busbar-headroom-arm64/busbar-headroom-arm64.tar.gz plugins/arm64/busbar-headroom.tar.gz
          # Runtime libs the plugin cdylib needs at dlopen time (musl libc/libgcc_s/libstdc++, built
          # in the SAME musl-native container as the plugin — see build-binaries' "Build headroom-hook
          # cdylib..." step for why these are required and how this was verified end to end).
          mv staged/busbar-headroom-libs-amd64/* plugins/amd64/lib/
          mv staged/busbar-headroom-libs-arm64/* plugins/arm64/lib/

      - uses: docker/setup-buildx-action@v4

      - name: Log in to Docker Hub
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Log in to GHCR
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      # The version being published: the pushed v* tag, or the workflow_dispatch
      # `version` input. Empty (bare dispatch) → publish only a `test` tag.
      - name: Resolve version
        id: ver
        run: |
          VER="${{ inputs.version }}"
          if [ -z "$VER" ] && [ "$GITHUB_REF_TYPE" = "tag" ]; then VER="$GITHUB_REF_NAME"; fi
          echo "ver=${VER#v}" >> "$GITHUB_OUTPUT"

      - name: Compute tags and labels
        id: meta
        uses: docker/metadata-action@v6
        with:
          images: |
            ${{ env.DOCKERHUB_IMAGE }}
            ${{ env.GHCR_IMAGE }}
          # version resolved → X.Y.Z + latest (exact pins only, no floating
          # major/minor aliases — one consistent tag shape); bare dispatch → `test` only.
          tags: |
            type=semver,pattern={{version}},value=v${{ steps.ver.outputs.ver }},enable=${{ steps.ver.outputs.ver != '' }}
            type=raw,value=test,enable=${{ steps.ver.outputs.ver == '' }}
          labels: |
            org.opencontainers.image.title=busbar
            org.opencontainers.image.description=The reliability layer for LLM traffic — one endpoint, six wire protocols, fault-attributed circuit breaking, in-flight failover.
            org.opencontainers.image.url=https://getbusbar.com
            org.opencontainers.image.documentation=https://getbusbar.com/docs/getting-started/
            org.opencontainers.image.licenses=Apache-2.0

      - name: Build and push
        id: push
        uses: docker/build-push-action@v7
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}

      - name: Install cosign
        uses: sigstore/cosign-installer@v3

      # GHCR only: cosign stores signatures as sha256-*.sig tags, which we keep
      # off the Docker Hub tag list (clean public page). Hub verification runs
      # through the GitHub build-provenance attestation instead.
      - name: Sign GHCR image (keyless)
        env:
          DIGEST: ${{ steps.push.outputs.digest }}
        run: cosign sign --yes "${GHCR_IMAGE}@${DIGEST}"

      - name: Attest build provenance (Docker Hub)
        uses: actions/attest-build-provenance@v4
        with:
          subject-name: index.docker.io/${{ env.DOCKERHUB_IMAGE }}
          subject-digest: ${{ steps.push.outputs.digest }}
          push-to-registry: false

      - name: Attest build provenance (GHCR)
        uses: actions/attest-build-provenance@v4
        with:
          subject-name: ${{ env.GHCR_IMAGE }}
          subject-digest: ${{ steps.push.outputs.digest }}
          push-to-registry: true