Skip to content

Commit 32010be

Browse files
jwm4jwm4
authored
Merge branch 'main' into submit/fullsend-ai-fullsend
2 parents 4d6b066 + 145888d commit 32010be

4 files changed

Lines changed: 140 additions & 23 deletions

File tree

‎.github/workflows/leaderboard-comment.yml‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
name: Leaderboard Validation Comment
2+
3+
# Post validation results as a PR comment after the leaderboard workflow completes.
4+
# Uses workflow_run to run in the upstream repo context with write permissions,
5+
# enabling comments on fork PRs (same pattern as coverage-comment.yml).
6+
7+
on:
8+
workflow_run:
9+
workflows: ["Leaderboard Management"]
10+
types: [completed]
11+
12+
jobs:
13+
post-comment:
14+
name: Post Validation Comment
15+
runs-on: ubuntu-latest
16+
if: github.event.workflow_run.event == 'pull_request'
17+
permissions:
18+
pull-requests: write
19+
actions: read
20+
21+
steps:
22+
- name: Get PR number from workflow run
23+
id: pr_info
24+
env:
25+
GH_TOKEN: ${{ github.token }}
26+
run: |
27+
HEAD_REPO="${{ github.event.workflow_run.head_repository.full_name }}"
28+
HEAD_BRANCH="${{ github.event.workflow_run.head_branch }}"
29+
TARGET_REPO="${{ github.repository }}"
30+
31+
if [ "$HEAD_REPO" = "$TARGET_REPO" ]; then
32+
BRANCH_QUERY="${HEAD_BRANCH}"
33+
else
34+
BRANCH_QUERY="${HEAD_REPO%%/*}:${HEAD_BRANCH}"
35+
fi
36+
37+
PR_NUMBER=$(gh pr view \
38+
--repo "$TARGET_REPO" \
39+
"$BRANCH_QUERY" \
40+
--json number --jq .number 2>/dev/null || echo "")
41+
42+
if [ -z "$PR_NUMBER" ]; then
43+
echo "::warning::Could not find PR for ${BRANCH_QUERY} in ${TARGET_REPO}"
44+
exit 0
45+
fi
46+
47+
echo "pr_number=${PR_NUMBER}" >> "$GITHUB_OUTPUT"
48+
49+
- name: Download validation data
50+
if: steps.pr_info.outputs.pr_number
51+
uses: actions/download-artifact@v4
52+
with:
53+
name: leaderboard-validation
54+
path: validation-data
55+
run-id: ${{ github.event.workflow_run.id }}
56+
github-token: ${{ secrets.GITHUB_TOKEN }}
57+
58+
- name: Post validation comment
59+
if: steps.pr_info.outputs.pr_number
60+
uses: actions/github-script@v9
61+
env:
62+
TRUSTED_PR_NUMBER: ${{ steps.pr_info.outputs.pr_number }}
63+
with:
64+
script: |
65+
const fs = require('fs');
66+
const data = JSON.parse(fs.readFileSync('validation-data/validation.json', 'utf8'));
67+
68+
const artifactPR = data.pr_number;
69+
const trustedPR = parseInt(process.env.TRUSTED_PR_NUMBER);
70+
if (parseInt(artifactPR) !== trustedPR) {
71+
core.setFailed(`PR number mismatch: artifact=${artifactPR}, expected=${trustedPR}`);
72+
return;
73+
}
74+
75+
const claimed = data.claimed_score || 'N/A';
76+
const actual = data.actual_score || 'N/A';
77+
const diff = actual !== 'N/A' && actual !== ''
78+
? Math.abs(parseFloat(actual) - parseFloat(claimed)).toFixed(1)
79+
: 'N/A';
80+
81+
const status = parseFloat(diff) <= 2.0 ? '✅ **PASSED**' : '❌ **FAILED**';
82+
const body = `## Leaderboard Validation\n\n${status}\n\n` +
83+
`**Claimed**: ${claimed}/100\n` +
84+
`**Verified**: ${actual}/100\n` +
85+
`**Diff**: ${diff} points (±2 tolerance)`;
86+
87+
const comments = await github.rest.issues.listComments({
88+
owner: context.repo.owner,
89+
repo: context.repo.repo,
90+
issue_number: trustedPR,
91+
});
92+
93+
const existing = comments.data.find(c =>
94+
c.user.login === 'github-actions[bot]' &&
95+
c.body.includes('Leaderboard Validation')
96+
);
97+
98+
if (existing) {
99+
await github.rest.issues.updateComment({
100+
owner: context.repo.owner,
101+
repo: context.repo.repo,
102+
comment_id: existing.id,
103+
body: body,
104+
});
105+
} else {
106+
await github.rest.issues.createComment({
107+
owner: context.repo.owner,
108+
repo: context.repo.repo,
109+
issue_number: trustedPR,
110+
body: body,
111+
});
112+
}

‎.github/workflows/leaderboard.yml‎

Lines changed: 20 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ jobs:
2626
runs-on: ubuntu-latest
2727
permissions:
2828
contents: read # Checkout PR branch
29-
pull-requests: write # Post validation results as PR comment
30-
issues: write # Required by github-script (PRs are issues in GH API)
3129
steps:
3230
# Fix for outdated forks: Always use upstream for validation tools/schema
3331
# The PR branch may be from a fork that hasn't synced with upstream,
@@ -203,34 +201,34 @@ jobs:
203201
exit 1
204202
fi
205203
206-
- name: Post validation results
204+
- name: Save validation results
207205
if: always()
208-
uses: actions/github-script@v9
209206
env:
210207
CLAIMED_SCORE: ${{ steps.extract.outputs.claimed_score }}
211208
ACTUAL_SCORE: ${{ env.ACTUAL_SCORE }}
212209
REPO_NAME: ${{ steps.extract.outputs.repo_name }}
213210
CLAIMED_RESEARCH_VERSION: ${{ steps.extract.outputs.research_version }}
214211
ACTUAL_RESEARCH_VERSION: ${{ env.ACTUAL_RESEARCH_VERSION }}
212+
PR_NUMBER: ${{ github.event.pull_request.number }}
213+
run: |
214+
mkdir -p /tmp/validation-data
215+
jq -n \
216+
--arg claimed "$CLAIMED_SCORE" \
217+
--arg actual "$ACTUAL_SCORE" \
218+
--arg repo_name "$REPO_NAME" \
219+
--arg claimed_rv "$CLAIMED_RESEARCH_VERSION" \
220+
--arg actual_rv "$ACTUAL_RESEARCH_VERSION" \
221+
--arg pr_number "$PR_NUMBER" \
222+
'{claimed_score: $claimed, actual_score: $actual, repo_name: $repo_name, claimed_research_version: $claimed_rv, actual_research_version: $actual_rv, pr_number: $pr_number}' \
223+
> /tmp/validation-data/validation.json
224+
225+
- name: Upload validation results
226+
if: always()
227+
uses: actions/upload-artifact@v4
215228
with:
216-
script: |
217-
// SAFE: All values from environment variables
218-
const claimed = process.env.CLAIMED_SCORE || 'N/A';
219-
const actual = process.env.ACTUAL_SCORE || 'N/A';
220-
const diff = actual !== 'N/A' ? Math.abs(parseFloat(actual) - parseFloat(claimed)).toFixed(1) : 'N/A';
221-
222-
const status = parseFloat(diff) <= 2.0 ? '✅ **PASSED**' : '❌ **FAILED**';
223-
const body = `## Leaderboard Validation\n\n${status}\n\n` +
224-
`**Claimed**: ${claimed}/100\n` +
225-
`**Verified**: ${actual}/100\n` +
226-
`**Diff**: ${diff} points (±2 tolerance)`;
227-
228-
github.rest.issues.createComment({
229-
owner: context.repo.owner,
230-
repo: context.repo.repo,
231-
issue_number: context.issue.number,
232-
body: body
233-
});
229+
name: leaderboard-validation
230+
path: /tmp/validation-data/validation.json
231+
retention-days: 1
234232

235233
# Job 2: Update leaderboard data (after merge to main, or manual trigger)
236234
update:

‎CHANGELOG.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
## [2.39.3](https://github.com/ambient-code/agentready/compare/v2.39.2...v2.39.3) (2026-05-20)
2+
3+
4+
### Bug Fixes
5+
6+
* post leaderboard validation comments via workflow_run for fork PRs ([308afc4](https://github.com/ambient-code/agentready/commit/308afc46773b7f68ef2d39c1c2211f93e43f5521))
7+
18
## [2.39.2](https://github.com/ambient-code/agentready/compare/v2.39.1...v2.39.2) (2026-05-20)
29

310

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "agentready"
3-
version = "2.39.2"
3+
version = "2.39.3"
44
description = "Assess and bootstrap git repositories for AI-assisted development with automated remediation"
55
authors = [{name = "Jeremy Eder", email = "jeder@redhat.com"}]
66
readme = "README.md"

0 commit comments

Comments
 (0)