forked from phase-rs/phase
-
Notifications
You must be signed in to change notification settings - Fork 0
207 lines (181 loc) · 8.35 KB
/
Copy pathai-gate.yml
File metadata and controls
207 lines (181 loc) · 8.35 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
name: AI gate
on:
pull_request:
branches: [main]
paths:
- 'crates/phase-ai/**'
- 'data/card-data.json'
- 'scripts/lib/mtgjson-fetch.sh'
- '.cargo/config.toml'
- '.github/actions/ai-card-data-cache/**'
- '.github/workflows/ai-gate.yml'
schedule:
- cron: "0 9 * * *"
workflow_dispatch:
permissions:
contents: read
issues: write
jobs:
ai-gate:
name: Paired-seed AI gate
if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
# 30m was too tight and flaked out on slow runners. Measured budget on a
# cache miss: ~6.5m for the "Generate card data" step (5m `tool` build + gen)
# + a cold debug build + the debug run itself (30 games at ~1.1m/game on a
# slow runner ≈ 34m). Hosted-runner speed varies ~2x, so a fast runner
# finishes well under 30m while a slow one overruns. 60m preserves the full
# `--games 10` coverage; the ceiling only bills the failure case.
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-shared-key: rust-ai-gate
- name: Restore AI card data caches
id: card-data-cache
uses: ./.github/actions/ai-card-data-cache
- name: Generate card data
if: steps.card-data-cache.outputs.cardgen-cache-hit != 'true'
run: |
cargo run --profile tool --features cli --bin oracle-gen -- data/ --stats --names-out data/card-names.json > data/card-data.json
- name: Run quick AI gate
run: cargo ai-gate --games 10
ai-gate-nightly:
name: Nightly AI gate drift monitor
if: github.event_name == 'schedule'
runs-on: ubuntu-latest
# `--full-suite --games 100` consistently exceeded the prior 90m ceiling
# (never completed since at least 2026-06-13). Raised to 300m — generous
# headroom under GitHub's 360m hosted-runner cap — to get a first completion
# and a real-duration baseline to right-size from. The ceiling only bills the
# failure case; a job that finishes in 2h stops at 2h.
timeout-minutes: 300
steps:
- uses: actions/checkout@v4
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-shared-key: rust-ai-gate
- name: Restore AI card data caches
id: card-data-cache
uses: ./.github/actions/ai-card-data-cache
- name: Generate card data
if: steps.card-data-cache.outputs.cardgen-cache-hit != 'true'
run: |
cargo run --profile tool --features cli --bin oracle-gen -- data/ --stats --names-out data/card-names.json > data/card-data.json
- name: Run full AI gate
id: gate
continue-on-error: true
run: |
# bash opens the redirect before cargo runs, so `target/` has to exist
# first. On a cardgen cache hit the `Generate card data` step above is
# skipped, and nothing else has invoked cargo yet — the redirect then
# dies with "No such file or directory" and the gate never executes.
mkdir -p target
cargo ai-gate --full-suite --games 100 > target/ai-gate-report.md
- name: Open or update drift issue
if: steps.gate.outcome == 'failure'
env:
GH_TOKEN: ${{ github.token }}
run: |
title="Nightly AI gate drift"
if [ ! -s target/ai-gate-report.md ]; then
echo "AI gate failed without a drift report" >&2
exit 1
fi
body="$(cat target/ai-gate-report.md)"
existing="$(gh issue list --label ai-gate-drift --state open --json number --jq '.[0].number')"
if [ -n "$existing" ]; then
gh issue comment "$existing" --body "$body"
else
gh issue create --title "$title" --label ai-gate-drift --body "$body"
fi
- name: Fail only on infrastructure errors
if: steps.gate.outcome == 'cancelled'
run: exit 1
ai-perf-gate:
name: Decision-cost perf gate
if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
# Same runner-variance overrun as the paired gate above. Measured budget on
# a cache miss: ~6.5m card-data gen + cold debug build + the perf suite
# (PERF_SAMPLE_COUNT=5 cold child processes × 3 scenarios × PERF_ACTION_CAP
# 3000 actions ≈ 21m in debug on a slow runner). The per-counter median
# assertion (#4878) is variance-robust; the wall-clock is not. 60m gives
# headroom without changing the fixed workload.
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-shared-key: rust-ai-gate
- name: Restore AI card data caches
id: card-data-cache
uses: ./.github/actions/ai-card-data-cache
- name: Generate card data
if: steps.card-data-cache.outputs.cardgen-cache-hit != 'true'
run: |
cargo run --profile tool --features cli --bin oracle-gen -- data/ --stats --names-out data/card-names.json > data/card-data.json
# server-release profile (authoritative, set by the `cargo ai-perf-gate` alias):
# counter VALUES are profile-independent, and the shared rust-ai-gate cache stays
# coherent because every job in this workflow builds the same profile — the
# win-rate jobs populate it and the perf jobs reuse it. Expect one cold build on
# the first run after the dev -> server-release move.
# Runs PERF_SAMPLE_COUNT independent sample processes; compares the per-counter median (#4878).
- name: Run decision-cost perf gate
run: cargo ai-perf-gate
ai-perf-gate-nightly:
name: Nightly decision-cost perf drift monitor
if: github.event_name == 'schedule'
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-shared-key: rust-ai-gate
- name: Restore AI card data caches
id: card-data-cache
uses: ./.github/actions/ai-card-data-cache
- name: Generate card data
if: steps.card-data-cache.outputs.cardgen-cache-hit != 'true'
run: |
cargo run --profile tool --features cli --bin oracle-gen -- data/ --stats --names-out data/card-names.json > data/card-data.json
# server-release profile (authoritative, set by the `cargo ai-perf-gate` alias):
# counter VALUES are profile-independent, and the shared rust-ai-gate cache stays
# coherent because every job in this workflow builds the same profile — the
# win-rate jobs populate it and the perf jobs reuse it. Expect one cold build on
# the first run after the dev -> server-release move.
# Runs PERF_SAMPLE_COUNT independent sample processes; compares the per-counter median (#4878).
# `cargo` build progress goes to stderr and spawned children are Stdio::null on
# stdout, so the redirect captures only the binary's clean markdown table.
- name: Run decision-cost perf gate
id: gate
continue-on-error: true
run: |
# bash opens the redirect before cargo runs, so `target/` has to exist
# first. On a cardgen cache hit the `Generate card data` step above is
# skipped, and nothing else has invoked cargo yet — the redirect then
# dies with "No such file or directory" and the gate never executes.
mkdir -p target
cargo ai-perf-gate > target/ai-perf-gate-report.md
- name: Open or update drift issue
if: steps.gate.outcome == 'failure'
env:
GH_TOKEN: ${{ github.token }}
run: |
title="Nightly decision-cost perf drift"
if [ ! -s target/ai-perf-gate-report.md ]; then
echo "Decision-cost perf gate failed without a drift report" >&2
exit 1
fi
body="$(cat target/ai-perf-gate-report.md)"
existing="$(gh issue list --label ai-perf-drift --state open --json number --jq '.[0].number')"
if [ -n "$existing" ]; then
gh issue comment "$existing" --body "$body"
else
gh issue create --title "$title" --label ai-perf-drift --body "$body"
fi
- name: Fail only on infrastructure errors
if: steps.gate.outcome == 'cancelled'
run: exit 1