feat: add priority lanes to worker scheduler

.env.example

@@ -30,6 +30,10 @@ PORT=8080
 # Use sslmode=require in production; sslmode=disable is acceptable for local dev.
 DATABASE_URL=postgres://stellabill:changeme@localhost:5432/stellabill_dev?sslmode=disable
 
+# [OPTIONAL] Read replica PostgreSQL connection string.
+# If not set, falls back to DATABASE_URL (primary).
+DATABASE_REPLICA_URL=postgres://stellabill:changeme@localhost:5432/stellabill_replica?sslmode=disable
+
 # -----------------------------------------------------------------------------
 # Authentication & authorisation
 # -----------------------------------------------------------------------------
@@ -54,6 +58,15 @@ ADMIN_TOKEN=CHANGE_ME_admin_Token1!
 # Example: https://app.example.com,https://admin.example.com
 ALLOWED_ORIGINS=http://localhost:3000
 OUTBOX_PUBLISHER_CA_FILE=
+
+# [OPTIONAL] Encrypt sensitive outbox payloads with subscriber JWKs (JWE).
+OUTBOX_JWE_ENABLED=false
+OUTBOX_JWE_SENSITIVE_EVENT_TYPES=webhook.received,payment.processed
+
+# [OPTIONAL] Probability (0.0–1.0) of injecting a context cancellation into
+# each outbox publish call. Only active when ENV=staging. Set to 0 to disable.
+# Example: CHAOS_OUTBOX_PROB=0.1 (10 % chance per publish).
+CHAOS_OUTBOX_PROB=0
 # -----------------------------------------------------------------------------
 # HTTP server tuning
 # -----------------------------------------------------------------------------
@@ -119,6 +132,18 @@ TRACING_EXPORTER=stdout
 # [OPTIONAL] Service name reported in trace spans.
 TRACING_SERVICE_NAME=stellabill-backend
 
+# [OPTIONAL] Enable bounded in-process tail decisions. Default: false.
+TRACING_TAIL_ENABLED=false
+
+# [OPTIONAL] Always retain traces whose server root takes at least this many
+# milliseconds. Range: 1-600000. Default: 1000.
+TRACING_TAIL_LATENCY_MS=1000
+
+# [OPTIONAL] Baseline fraction of ordinary traces retained when tail sampling
+# is enabled. Errors, 5xx responses, and slow requests are always retained.
+# Range: 0.0-1.0. Default: 0.05.
+TRACING_TAIL_ERROR_RATE=0.05
+
 # -----------------------------------------------------------------------------
 # Database connection pool
 # -----------------------------------------------------------------------------
@@ -158,6 +183,15 @@ AUDIT_HMAC_SECRET=CHANGE_ME_audit_Hmac1!
 # [OPTIONAL] File path for the audit log (JSON Lines). Default: audit.log.
 AUDIT_LOG_PATH=audit.log
 
+# -----------------------------------------------------------------------------
+# Legacy API deprecation
+# -----------------------------------------------------------------------------
+
+# [OPTIONAL] HTTP-date or RFC3339 timestamp emitted as the Sunset header on
+# legacy /api/* aliases. Leave unset to omit Sunset while keeping Deprecation
+# and successor Link headers.
+LEGACY_API_SUNSET="Thu, 31 Dec 2026 23:59:59 GMT"
+
 # -----------------------------------------------------------------------------
 # Feature flags
 # -----------------------------------------------------------------------------

.github/pr_body.txt

@@ -1,36 +1,36 @@
-feat: add contract-to-backend reconciliation endpoint and reports
-
-Summary
-- Adds a backend ↔ contract reconciliation subsystem and an admin HTTP endpoint to run on-demand checks.
-- Provides models, a comparator (field-by-field), adapters (in-memory + HTTP), and an in-memory store for report persistence.
-- Adds unit tests for comparator, adapters, and handler.
-- Adds documentation and a GitHub Actions workflow that runs `go test ./...` on push/PR.
-
-Files of interest
-- `internal/reconciliation/*` — comparator, models, adapters, store, and tests
-- `internal/handlers/reconciliation.go` — admin POST `/api/admin/reconcile` (accepts backend subscriptions)
-- `internal/routes/routes.go` — route wiring, adapter selection via `CONTRACT_SNAPSHOT_URL`, admin GET `/api/admin/reports`
-- `docs/reconciliation.md` — short doc + security notes
-- `scripts/install_go_and_run_tests.ps1` — helper to install Go and run reconciliation tests locally
-- `.github/workflows/reconciliation-ci.yml` — CI that runs the full test suite
-
-How it works
-- POST JSON array of backend subscriptions to `/api/admin/reconcile` (admin-only).
-- The handler fetches contract snapshots via configured adapter:
-  - If `CONTRACT_SNAPSHOT_URL` is set -> HTTP adapter fetches JSON snapshots from that URL (set `CONTRACT_SNAPSHOT_AUTH` for auth header).
-  - Otherwise uses an in-memory adapter for dev.
-- Comparator checks: status, amount+currency, interval, per-key balances, missing snapshots, and stale snapshots (>24h).
-- Reports are saved to an in-memory store and can be retrieved via GET `/api/admin/reports`.
-
-Security notes
-- Endpoint is protected by `auth.RequirePermission(auth.PermManageSubscriptions)`; ensure only admin roles can call it.
-- For HTTP adapter use TLS and set `CONTRACT_SNAPSHOT_AUTH` for authentication.
-- Redact PII and use a persistent, access-controlled store in production — the current store is in-memory for dev/tests.
-
-Testing
-- Unit tests added under `internal/reconciliation` and `internal/handlers`.
-- CI workflow runs `go test ./...` on push/PR.
-
-Next steps
-- Replace in-memory store with DB-backed store (migration, repo, tests) for production.
-- Wire a real contract snapshot endpoint and add integration tests. Provide API details (URL, auth, JSON schema) and I will implement the adapter and tests.
+feat: add contract-to-backend reconciliation endpoint and reports
+
+Summary
+- Adds a backend ↔ contract reconciliation subsystem and an admin HTTP endpoint to run on-demand checks.
+- Provides models, a comparator (field-by-field), adapters (in-memory + HTTP), and an in-memory store for report persistence.
+- Adds unit tests for comparator, adapters, and handler.
+- Adds documentation and a GitHub Actions workflow that runs `go test ./...` on push/PR.
+
+Files of interest
+- `internal/reconciliation/*` — comparator, models, adapters, store, and tests
+- `internal/handlers/reconciliation.go` — admin POST `/api/admin/reconcile` (accepts backend subscriptions)
+- `internal/routes/routes.go` — route wiring, adapter selection via `CONTRACT_SNAPSHOT_URL`, admin GET `/api/admin/reports`
+- `docs/reconciliation.md` — short doc + security notes
+- `scripts/install_go_and_run_tests.ps1` — helper to install Go and run reconciliation tests locally
+- `.github/workflows/reconciliation-ci.yml` — CI that runs the full test suite
+
+How it works
+- POST JSON array of backend subscriptions to `/api/admin/reconcile` (admin-only).
+- The handler fetches contract snapshots via configured adapter:
+  - If `CONTRACT_SNAPSHOT_URL` is set -> HTTP adapter fetches JSON snapshots from that URL (set `CONTRACT_SNAPSHOT_AUTH` for auth header).
+  - Otherwise uses an in-memory adapter for dev.
+- Comparator checks: status, amount+currency, interval, per-key balances, missing snapshots, and stale snapshots (>24h).
+- Reports are saved to an in-memory store and can be retrieved via GET `/api/admin/reports`.
+
+Security notes
+- Endpoint is protected by `auth.RequirePermission(auth.PermManageSubscriptions)`; ensure only admin roles can call it.
+- For HTTP adapter use TLS and set `CONTRACT_SNAPSHOT_AUTH` for authentication.
+- Redact PII and use a persistent, access-controlled store in production — the current store is in-memory for dev/tests.
+
+Testing
+- Unit tests added under `internal/reconciliation` and `internal/handlers`.
+- CI workflow runs `go test ./...` on push/PR.
+
+Next steps
+- Replace in-memory store with DB-backed store (migration, repo, tests) for production.
+- Wire a real contract snapshot endpoint and add integration tests. Provide API details (URL, auth, JSON schema) and I will implement the adapter and tests.

.github/workflows/benchmark-regression-gate.yml

@@ -0,0 +1,250 @@
+name: Benchmark Regression Gate
+
+on:
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+# Prevent concurrent runs on the same PR so baselines are never written
+# and read at the same time, which would corrupt the comparison.
+concurrency:
+  group: benchmark-regression-gate-${{ github.ref }}
+  cancel-in-progress: false
+
+env:
+  # Fail CI if any tracked benchmark regresses by more than this amount.
+  REGRESSION_THRESHOLD_PERCENT: "10"
+
+jobs:
+  benchmark-regression-gate:
+    # Pin to a stable runner class so hardware variance does not produce
+    # false positives.  ubuntu-22.04 is a fixed GA image (not `latest`).
+    runs-on: ubuntu-22.04
+
+    permissions:
+      contents: read
+      actions: read      # needed to download artifacts from the main branch
+
+    steps:
+      # ---------------------------------------------------------------
+      # 1.  Check out the PR head with full history so we can also
+      #     check out origin/main in a worktree.
+      # ---------------------------------------------------------------
+      - name: Checkout PR head
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0   # full history required for worktree
+
+      # ---------------------------------------------------------------
+      # 2.  Set up Go (version comes from go.mod so it stays in sync).
+      # ---------------------------------------------------------------
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      # ---------------------------------------------------------------
+      # 3.  Install benchstat – the authoritative statistical comparator
+      #     from the Go performance team.  It computes p-values and
+      #     confidence intervals so single-sample noise is ignored.
+      # ---------------------------------------------------------------
+      - name: Install benchstat
+        run: go install golang.org/x/perf/cmd/benchstat@latest
+
+      # ---------------------------------------------------------------
+      # 4.  Download dependencies for the PR head.
+      # ---------------------------------------------------------------
+      - name: Download dependencies (PR head)
+        run: go mod download
+
+      # ---------------------------------------------------------------
+      # 5.  Run the benchmark suite on the PR head.
+      #     -count=10 gives benchstat enough samples to compute a
+      #     meaningful confidence interval and reject statistical noise.
+      # ---------------------------------------------------------------
+      - name: Run benchmarks on PR head
+        run: |
+          go test \
+            -bench=. \
+            -benchmem \
+            -count=10 \
+            -run=^$ \
+            -timeout=20m \
+            ./internal/handlers/... \
+            | tee /tmp/bench_head.txt
+          echo "PR head benchmark output:"
+          cat /tmp/bench_head.txt
+
+      # ---------------------------------------------------------------
+      # 6.  Try to restore a stored baseline produced from the last
+      #     successful push to main.  If none exists (first run, or the
+      #     artifact expired) we skip the comparison and succeed so that
+      #     new repositories are not permanently broken.
+      # ---------------------------------------------------------------
+      - name: Restore baseline artifact
+        id: restore-baseline
+        continue-on-error: true
+        uses: actions/download-artifact@v4
+        with:
+          name: benchmark-baseline-main
+          path: /tmp/baseline
+
+      # ---------------------------------------------------------------
+      # 7.  Decide whether a baseline is available.
+      # ---------------------------------------------------------------
+      - name: Check baseline availability
+        id: check-baseline
+        run: |
+          if [ -f /tmp/baseline/bench_baseline.txt ]; then
+            echo "baseline_exists=true" >> "$GITHUB_OUTPUT"
+            echo "Baseline file found – regression gate is active."
+          else
+            echo "baseline_exists=false" >> "$GITHUB_OUTPUT"
+            echo "No baseline artifact found.  Skipping regression comparison (first run or expired artifact)."
+          fi
+
+      # ---------------------------------------------------------------
+      # 8.  Run the comparison with benchstat.
+      #     --threshold is intentionally NOT used here; we parse the
+      #     output ourselves so we can report per-benchmark details and
+      #     use a strict 10 % ceiling (benchstat's built-in threshold
+      #     option only gates on statistical significance, not magnitude).
+      # ---------------------------------------------------------------
+      - name: Compare benchmarks with benchstat
+        if: steps.check-baseline.outputs.baseline_exists == 'true'
+        id: compare
+        run: |
+          echo "## Benchmark Regression Report" >> "$GITHUB_STEP_SUMMARY"
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+          echo '```' >> "$GITHUB_STEP_SUMMARY"
+          benchstat /tmp/baseline/bench_baseline.txt /tmp/bench_head.txt \
+            | tee /tmp/benchstat_output.txt \
+            | tee -a "$GITHUB_STEP_SUMMARY"
+          echo '```' >> "$GITHUB_STEP_SUMMARY"
+
+      # ---------------------------------------------------------------
+      # 9.  Parse benchstat output and fail if any benchmark regressed
+      #     by more than REGRESSION_THRESHOLD_PERCENT.
+      #
+      #     benchstat prints lines like:
+      #       BenchmarkListPlans_Small   1.10 ± 2%   1.25 ± 3%   +13.64% (p=0.000 n=10)
+      #     We extract the final percentage column and compare to the
+      #     threshold.  Lines that lack a percentage (new / removed
+      #     benchmarks) are handled as edge cases below.
+      # ---------------------------------------------------------------
+      - name: Enforce regression threshold
+        if: steps.check-baseline.outputs.baseline_exists == 'true'
+        run: |
+          THRESHOLD=${{ env.REGRESSION_THRESHOLD_PERCENT }}
+          FAILED=0
+
+          echo "Checking for regressions > ${THRESHOLD}% …"
+
+          while IFS= read -r line; do
+            # Skip header / blank / informational lines
+            [[ "$line" =~ ^(name|goos|goarch|pkg|cpu|PASS|ok|---) ]] && continue
+            [[ -z "$line" ]] && continue
+
+            # Extract the trailing POSITIVE delta column, e.g. "+13.64%".
+            # Negative (improvement) tokens are intentionally skipped.
+            delta=$(echo "$line" | grep -oE '\+[0-9]+\.[0-9]+%' | tail -1 || true)
+            [[ -z "$delta" ]] && continue
+
+            # Strip '+' and '%' to get the magnitude
+            magnitude=$(echo "$delta" | tr -d '+' | tr -d '%')
+
+            # Compare using awk for floating-point arithmetic
+            is_regression=$(awk -v mag="$magnitude" -v thr="$THRESHOLD" \
+              'BEGIN { print (mag > thr) ? "yes" : "no" }')
+
+            if [[ "$is_regression" == "yes" ]]; then
+              echo "❌  REGRESSION: $line"
+              FAILED=$((FAILED + 1))
+            fi
+          done < /tmp/benchstat_output.txt
+
+          echo ""
+          if [[ $FAILED -gt 0 ]]; then
+            echo "❌  $FAILED benchmark(s) regressed by more than ${THRESHOLD}%." >&2
+            echo "" >&2
+            echo "To investigate locally:" >&2
+            echo "  git checkout main && go test -bench=. -count=10 -run=^$ ./internal/handlers/... | tee base.txt" >&2
+            echo "  git checkout - && go test -bench=. -count=10 -run=^$ ./internal/handlers/... | tee head.txt" >&2
+            echo "  benchstat base.txt head.txt" >&2
+            exit 1
+          else
+            echo "✅  No benchmark regressed by more than ${THRESHOLD}%."
+          fi
+
+      # ---------------------------------------------------------------
+      # 10. Persist benchmark results as an artifact so they are visible
+      #     in the Actions UI regardless of pass / fail.
+      # ---------------------------------------------------------------
+      - name: Upload PR head benchmark results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results-pr-${{ github.event.pull_request.number }}
+          path: /tmp/bench_head.txt
+          retention-days: 30
+
+      # ---------------------------------------------------------------
+      # 11. Emit a summary when no baseline is available so reviewers
+      #     know why the gate was skipped.
+      # ---------------------------------------------------------------
+      - name: Summary (no baseline)
+        if: steps.check-baseline.outputs.baseline_exists != 'true'
+        run: |
+          echo "## Benchmark Regression Gate – Skipped" >> "$GITHUB_STEP_SUMMARY"
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+          echo "No baseline artifact found for \`main\`.  This is expected on first run." >> "$GITHUB_STEP_SUMMARY"
+          echo "A baseline will be created after this PR merges and the \`update-benchmark-baseline\` job runs." >> "$GITHUB_STEP_SUMMARY"
+
+  # -----------------------------------------------------------------
+  # Separate job: only runs on pushes to main to update the baseline.
+  # Runs on push to main (triggered separately from the PR gate above).
+  # -----------------------------------------------------------------
+  update-benchmark-baseline:
+    if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main'
+    runs-on: ubuntu-22.04
+
+    permissions:
+      contents: read
+      actions: write   # needed to upload artifacts
+
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Download dependencies
+        run: go mod download
+
+      - name: Run benchmarks on main (baseline)
+        run: |
+          go test \
+            -bench=. \
+            -benchmem \
+            -count=10 \
+            -run=^$ \
+            -timeout=20m \
+            ./internal/handlers/... \
+            | tee /tmp/bench_baseline.txt
+          echo "Baseline benchmark output:"
+          cat /tmp/bench_baseline.txt
+
+      - name: Upload baseline artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-baseline-main
+          path: /tmp/bench_baseline.txt
+          # Keep for 90 days so PRs opened against an old main still
+          # have a baseline to compare against.
+          retention-days: 90
+          overwrite: true