AI gate #3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AI gate | |
| on: | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - 'crates/phase-ai/**' | |
| - 'data/card-data.json' | |
| - 'scripts/lib/mtgjson-fetch.sh' | |
| - '.cargo/config.toml' | |
| - '.github/actions/ai-card-data-cache/**' | |
| - '.github/workflows/ai-gate.yml' | |
| schedule: | |
| - cron: "0 9 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| issues: write | |
| jobs: | |
| ai-gate: | |
| name: Paired-seed AI gate | |
| if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| # 30m was too tight and flaked out on slow runners. Measured budget on a | |
| # cache miss: ~6.5m for the "Generate card data" step (5m `tool` build + gen) | |
| # + a cold debug build + the debug run itself (30 games at ~1.1m/game on a | |
| # slow runner ≈ 34m). Hosted-runner speed varies ~2x, so a fast runner | |
| # finishes well under 30m while a slow one overruns. 60m preserves the full | |
| # `--games 10` coverage; the ceiling only bills the failure case. | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| cache-shared-key: rust-ai-gate | |
| - name: Restore AI card data caches | |
| id: card-data-cache | |
| uses: ./.github/actions/ai-card-data-cache | |
| - name: Generate card data | |
| if: steps.card-data-cache.outputs.cardgen-cache-hit != 'true' | |
| run: | | |
| cargo run --profile tool --features cli --bin oracle-gen -- data/ --stats --names-out data/card-names.json > data/card-data.json | |
| - name: Run quick AI gate | |
| run: cargo ai-gate --games 10 | |
| ai-gate-nightly: | |
| name: Nightly AI gate drift monitor | |
| if: github.event_name == 'schedule' | |
| runs-on: ubuntu-latest | |
| # `--full-suite --games 100` consistently exceeded the prior 90m ceiling | |
| # (never completed since at least 2026-06-13). Raised to 300m — generous | |
| # headroom under GitHub's 360m hosted-runner cap — to get a first completion | |
| # and a real-duration baseline to right-size from. The ceiling only bills the | |
| # failure case; a job that finishes in 2h stops at 2h. | |
| timeout-minutes: 300 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| cache-shared-key: rust-ai-gate | |
| - name: Restore AI card data caches | |
| id: card-data-cache | |
| uses: ./.github/actions/ai-card-data-cache | |
| - name: Generate card data | |
| if: steps.card-data-cache.outputs.cardgen-cache-hit != 'true' | |
| run: | | |
| cargo run --profile tool --features cli --bin oracle-gen -- data/ --stats --names-out data/card-names.json > data/card-data.json | |
| - name: Run full AI gate | |
| id: gate | |
| continue-on-error: true | |
| run: | | |
| # bash opens the redirect before cargo runs, so `target/` has to exist | |
| # first. On a cardgen cache hit the `Generate card data` step above is | |
| # skipped, and nothing else has invoked cargo yet — the redirect then | |
| # dies with "No such file or directory" and the gate never executes. | |
| mkdir -p target | |
| cargo ai-gate --full-suite --games 100 > target/ai-gate-report.md | |
| - name: Open or update drift issue | |
| if: steps.gate.outcome == 'failure' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| title="Nightly AI gate drift" | |
| if [ ! -s target/ai-gate-report.md ]; then | |
| echo "AI gate failed without a drift report" >&2 | |
| exit 1 | |
| fi | |
| body="$(cat target/ai-gate-report.md)" | |
| existing="$(gh issue list --label ai-gate-drift --state open --json number --jq '.[0].number')" | |
| if [ -n "$existing" ]; then | |
| gh issue comment "$existing" --body "$body" | |
| else | |
| gh issue create --title "$title" --label ai-gate-drift --body "$body" | |
| fi | |
| - name: Fail only on infrastructure errors | |
| if: steps.gate.outcome == 'cancelled' | |
| run: exit 1 | |
| ai-perf-gate: | |
| name: Decision-cost perf gate | |
| if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| # Same runner-variance overrun as the paired gate above. Measured budget on | |
| # a cache miss: ~6.5m card-data gen + cold debug build + the perf suite | |
| # (PERF_SAMPLE_COUNT=5 cold child processes × 3 scenarios × PERF_ACTION_CAP | |
| # 3000 actions ≈ 21m in debug on a slow runner). The per-counter median | |
| # assertion (#4878) is variance-robust; the wall-clock is not. 60m gives | |
| # headroom without changing the fixed workload. | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| cache-shared-key: rust-ai-gate | |
| - name: Restore AI card data caches | |
| id: card-data-cache | |
| uses: ./.github/actions/ai-card-data-cache | |
| - name: Generate card data | |
| if: steps.card-data-cache.outputs.cardgen-cache-hit != 'true' | |
| run: | | |
| cargo run --profile tool --features cli --bin oracle-gen -- data/ --stats --names-out data/card-names.json > data/card-data.json | |
| # server-release profile (authoritative, set by the `cargo ai-perf-gate` alias): | |
| # counter VALUES are profile-independent, and the shared rust-ai-gate cache stays | |
| # coherent because every job in this workflow builds the same profile — the | |
| # win-rate jobs populate it and the perf jobs reuse it. Expect one cold build on | |
| # the first run after the dev -> server-release move. | |
| # Runs PERF_SAMPLE_COUNT independent sample processes; compares the per-counter median (#4878). | |
| - name: Run decision-cost perf gate | |
| run: cargo ai-perf-gate | |
| ai-perf-gate-nightly: | |
| name: Nightly decision-cost perf drift monitor | |
| if: github.event_name == 'schedule' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| cache-shared-key: rust-ai-gate | |
| - name: Restore AI card data caches | |
| id: card-data-cache | |
| uses: ./.github/actions/ai-card-data-cache | |
| - name: Generate card data | |
| if: steps.card-data-cache.outputs.cardgen-cache-hit != 'true' | |
| run: | | |
| cargo run --profile tool --features cli --bin oracle-gen -- data/ --stats --names-out data/card-names.json > data/card-data.json | |
| # server-release profile (authoritative, set by the `cargo ai-perf-gate` alias): | |
| # counter VALUES are profile-independent, and the shared rust-ai-gate cache stays | |
| # coherent because every job in this workflow builds the same profile — the | |
| # win-rate jobs populate it and the perf jobs reuse it. Expect one cold build on | |
| # the first run after the dev -> server-release move. | |
| # Runs PERF_SAMPLE_COUNT independent sample processes; compares the per-counter median (#4878). | |
| # `cargo` build progress goes to stderr and spawned children are Stdio::null on | |
| # stdout, so the redirect captures only the binary's clean markdown table. | |
| - name: Run decision-cost perf gate | |
| id: gate | |
| continue-on-error: true | |
| run: | | |
| # bash opens the redirect before cargo runs, so `target/` has to exist | |
| # first. On a cardgen cache hit the `Generate card data` step above is | |
| # skipped, and nothing else has invoked cargo yet — the redirect then | |
| # dies with "No such file or directory" and the gate never executes. | |
| mkdir -p target | |
| cargo ai-perf-gate > target/ai-perf-gate-report.md | |
| - name: Open or update drift issue | |
| if: steps.gate.outcome == 'failure' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| title="Nightly decision-cost perf drift" | |
| if [ ! -s target/ai-perf-gate-report.md ]; then | |
| echo "Decision-cost perf gate failed without a drift report" >&2 | |
| exit 1 | |
| fi | |
| body="$(cat target/ai-perf-gate-report.md)" | |
| existing="$(gh issue list --label ai-perf-drift --state open --json number --jq '.[0].number')" | |
| if [ -n "$existing" ]; then | |
| gh issue comment "$existing" --body "$body" | |
| else | |
| gh issue create --title "$title" --label ai-perf-drift --body "$body" | |
| fi | |
| - name: Fail only on infrastructure errors | |
| if: steps.gate.outcome == 'cancelled' | |
| run: exit 1 |