From efa17e5bb3122fd701214886589c5b58fad30c8a Mon Sep 17 00:00:00 2001 From: Benjamin Taylor Date: Tue, 21 Jul 2026 11:15:52 -0500 Subject: [PATCH 1/5] ci(triage): add issue-triage automation (stale + on-open label/dedup + backfill) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port of the CopilotKit issue-triage automation, adapted to ag-ui's label taxonomy. Three workflows sharing one analysis module (scripts/triage/analyze.js; repo-agnostic via context.repo): - triage-stale.yml daily cron, no LLM. Marks needs-info issues stale after 14d silence, closes at +7d. Exempts Roadmap/proposal; PRs untouched. (Needs a needs-info label created to act.) - triage-on-open.yml issues:opened. One combined LLM call classifies + flags duplicates. Applies allow-listed labels (>=0.75) and an advisory dup comment (>=0.8). Never closes. - triage-backfill.yml manual, dry-run by default. Same analysis over the existing backlog, capped by max_issues. Allow-list holds ag-ui's real content labels only: bug, enhancement, documentation, question, Integration, SDK, framework, Agent Framework. Curation/disposition labels (proposal, Roadmap, good first issue, help wanted…) excluded. Safety: default-deny allow-list, confidence gates, flag-never-close, spam/low-signal cost gate, SHA-pinned actions w/ persist-credentials:false, clean-skip when ANTHROPIC_API_KEY absent. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/triage-backfill.yml | 78 +++++++++++++++++++ .github/workflows/triage-on-open.yml | 56 ++++++++++++++ .github/workflows/triage-stale.yml | 38 ++++++++++ scripts/triage/README.md | 46 ++++++++++++ scripts/triage/analyze.js | 103 ++++++++++++++++++++++++++ 5 files changed, 321 insertions(+) create mode 100644 .github/workflows/triage-backfill.yml create mode 100644 .github/workflows/triage-on-open.yml create mode 100644 .github/workflows/triage-stale.yml create mode 100644 scripts/triage/README.md create mode 100644 scripts/triage/analyze.js diff --git a/.github/workflows/triage-backfill.yml b/.github/workflows/triage-backfill.yml new file mode 100644 index 0000000000..9de16ecac8 --- /dev/null +++ b/.github/workflows/triage-backfill.yml @@ -0,0 +1,78 @@ +# .github/workflows/triage-backfill.yml +# One-time / occasional drain of the ACCRETED backlog — runs the SAME shared +# analysis (scripts/triage/analyze.js) over EXISTING open issues. Manual only, +# DRY-RUN by default (preview in the job summary before anything applies), capped. +name: triage / backfill (manual) +on: + workflow_dispatch: + inputs: + max_issues: + description: "Max issues to process this run (oldest first)" + type: number + default: 50 + dry_run: + description: "Preview only (no labels/comments applied)" + type: boolean + default: true +permissions: + issues: write +concurrency: + group: triage-backfill + cancel-in-progress: false +jobs: + backfill: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false # only reads scripts/triage/analyze.js; no git ops + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + with: + script: | + const analyzeIssue = require(`${process.env.GITHUB_WORKSPACE}/scripts/triage/analyze.js`); + const { owner, repo } = context.repo; + const dryRun = ${{ inputs.dry_run }}; + const max = Number("${{ inputs.max_issues }}") || 50; + if (!process.env.ANTHROPIC_API_KEY) { core.warning("ANTHROPIC_API_KEY not set — nothing to do."); return; } + const sleep = (ms) => new Promise(r => setTimeout(r, ms)); + + const labels = await github.paginate(github.rest.issues.listLabelsForRepo, { owner, repo, per_page: 100 }); + const valid = new Set(labels.map(l => l.name)); + const APPLYABLE = new Set(["bug","enhancement","documentation","question","Integration","SDK","framework","Agent Framework"]); // content labels only; see triage-on-open.yml for rationale + const labelList = labels.map(l => `- ${l.name}${l.description ? `: ${l.description}` : ""}`).join("\n"); + + const all = await github.paginate(github.rest.issues.listForRepo, { owner, repo, state: "open", sort: "created", direction: "asc", per_page: 100 }); + const issues = all.filter(i => !i.pull_request).slice(0, max); + const rows = []; + + for (const issue of issues) { + let r; + try { r = await analyzeIssue({ github, owner, repo, issue, labelList }); } + catch (e) { core.warning(`#${issue.number}: ${e.message}`); continue; } + await sleep(2500); // analyze() does a search — stay under the 30/min search limit + if (r.skipped) continue; + + const toApply = r.labels.filter(l => valid.has(l) && APPLYABLE.has(l) && !(issue.labels || []).some(x => (x.name || x) === l)); + const willLabel = r.labelConfidence >= 0.75 && toApply.length; + const dup = (r.duplicateOf && r.dupConfidence >= 0.8 && r.candidates.includes(r.duplicateOf) && valid.has("duplicate")) ? r.duplicateOf : null; + if (!willLabel && !dup) continue; + + rows.push({ n: issue.number, labels: willLabel ? toApply.join(",") : "", dup: dup ? `#${dup}` : "" }); + if (!dryRun) { + if (willLabel) await github.rest.issues.addLabels({ owner, repo, issue_number: issue.number, labels: toApply }); + if (dup) { + await github.rest.issues.addLabels({ owner, repo, issue_number: issue.number, labels: ["duplicate"] }); + await github.rest.issues.createComment({ owner, repo, issue_number: issue.number, body: `Possible duplicate of #${dup} — a maintainer will confirm; say so if it's not and we'll remove the label. (Automated triage flag.)` }); + } + } + } + + core.summary + .addHeading(`Triage backfill — ${dryRun ? "DRY RUN (nothing applied)" : "APPLIED"} · ${issues.length} scanned · ${rows.length} actionable`) + .addTable([ + [{ data: "Issue", header: true }, { data: "Labels", header: true }, { data: "Dup-of", header: true }], + ...rows.map(r => [`#${r.n}`, r.labels || "—", r.dup || "—"]), + ]) + .write(); diff --git a/.github/workflows/triage-on-open.yml b/.github/workflows/triage-on-open.yml new file mode 100644 index 0000000000..274df08f72 --- /dev/null +++ b/.github/workflows/triage-on-open.yml @@ -0,0 +1,56 @@ +# .github/workflows/triage-on-open.yml +# Consolidated on-open triage: label + dedup-flag in ONE combined LLM call +# (shared logic in scripts/triage/analyze.js). Advisory / high-confidence, +# reopen-friendly; flags duplicates but never auto-closes. No key -> clean skip. +name: triage / on-open +on: + issues: + types: [opened] +permissions: + issues: write +concurrency: + group: triage-on-open-${{ github.event.issue.number }} + cancel-in-progress: true +jobs: + triage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false # only reads scripts/triage/analyze.js; no git ops + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + with: + script: | + const analyzeIssue = require(`${process.env.GITHUB_WORKSPACE}/scripts/triage/analyze.js`); + const { owner, repo } = context.repo; + const issue = context.payload.issue; + + const labels = await github.paginate(github.rest.issues.listLabelsForRepo, { owner, repo, per_page: 100 }); + const valid = new Set(labels.map(l => l.name)); + // Allow-list (default-deny): the classifier may apply ONLY these — content + // labels that exist in this repo. Excludes disposition/curation labels + // (proposal, Roadmap, good first issue, help wanted, release, invalid, + // wontfix…) — those are human calls. "enhancement" is this repo's canonical + // feature type (heavily used); "Feature Request" is a rarely-used synonym we + // deliberately omit to avoid the classifier waffling between the two. + const APPLYABLE = new Set(["bug","enhancement","documentation","question","Integration","SDK","framework","Agent Framework"]); + const labelList = labels.map(l => `- ${l.name}${l.description ? `: ${l.description}` : ""}`).join("\n"); + + const r = await analyzeIssue({ github, owner, repo, issue, labelList }); + if (r.skipped) { core.info(`skipped: ${r.skipped}`); return; } + + const toApply = r.labels.filter(l => valid.has(l) && APPLYABLE.has(l)); + if (r.labelConfidence >= 0.75 && toApply.length) { + await github.rest.issues.addLabels({ owner, repo, issue_number: issue.number, labels: toApply }); + core.info(`labeled: ${toApply.join(", ")} (conf ${r.labelConfidence})`); + } + if (r.duplicateOf && r.dupConfidence >= 0.8 && r.candidates.includes(r.duplicateOf) && valid.has("duplicate")) { + await github.rest.issues.addLabels({ owner, repo, issue_number: issue.number, labels: ["duplicate"] }); + await github.rest.issues.createComment({ + owner, repo, issue_number: issue.number, + body: `Possible duplicate of #${r.duplicateOf} — a maintainer will confirm; if it isn't, just say so and we'll remove the label. (Automated triage flag; nothing closes automatically.)`, + }); + core.info(`flagged possible dup of #${r.duplicateOf} (conf ${r.dupConfidence})`); + } diff --git a/.github/workflows/triage-stale.yml b/.github/workflows/triage-stale.yml new file mode 100644 index 0000000000..5f193e0023 --- /dev/null +++ b/.github/workflows/triage-stale.yml @@ -0,0 +1,38 @@ +# .github/workflows/triage-stale.yml +# Stop-the-bleeding MVP #1 — drain the "awaiting info" dead weight. +# Deterministic (no LLM). Scoped to needs-info only, so it never nukes real issues. +name: triage / stale +on: + schedule: + - cron: "0 9 * * *" # daily 09:00 UTC + workflow_dispatch: +permissions: + issues: write +concurrency: + group: triage-stale + cancel-in-progress: false +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 + with: + # MVP: only auto-progress issues we explicitly asked info on and got silence. + only-labels: "needs-info" + days-before-stale: 14 + days-before-close: 7 + stale-issue-label: "stale" + stale-issue-message: > + Marking stale — this has been awaiting the requested info for 14 days + and will close in 7 if there's no update. Just comment to keep it open, + and reopen anytime if it closes. + close-issue-message: > + Closing due to inactivity while awaiting more information. Please reopen + or comment with the details and we'll pick it back up. + remove-stale-when-updated: true + exempt-issue-labels: "Roadmap,proposal" + # PRs are a later arc — do not touch them here. + days-before-pr-stale: -1 + days-before-pr-close: -1 + # Be gentle on rate limits. + operations-per-run: 60 diff --git a/scripts/triage/README.md b/scripts/triage/README.md new file mode 100644 index 0000000000..76887501b3 --- /dev/null +++ b/scripts/triage/README.md @@ -0,0 +1,46 @@ +# Issue triage automation + +Stop-the-bleeding automation for incoming issues. Three workflows, one shared +analysis module. Everything is **advisory / high-confidence / reopen-friendly** — +nothing closes an issue on an LLM's say-so. + +| Workflow | Trigger | LLM? | What it does | +| --------------------- | ---------------------------- | ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `triage-stale.yml` | daily cron | no | Marks `needs-info` issues stale after 14d of silence, closes 7d later. Scoped to `needs-info` only, so it never touches active issues. Exempts `Roadmap,proposal`. PRs untouched. | +| `triage-on-open.yml` | `issues: opened` | yes | One combined classify + dedup pass. Applies allow-listed labels (conf ≥ 0.75) and flags likely duplicates with an advisory comment (conf ≥ 0.8). Never closes. | +| `triage-backfill.yml` | manual (`workflow_dispatch`) | yes | Same analysis over the existing open backlog. **Dry-run by default** — previews in the job summary; only applies when you uncheck dry-run. Capped by `max_issues`. | + +`analyze.js` is the single source of truth for the LLM logic (search for +candidates → one combined Anthropic call → return proposals). Both the on-open +and backfill workflows call it, so policy and safety controls live in one place. + +## Setup + +1. **`ANTHROPIC_API_KEY`** repo secret — required only by the two LLM workflows. + Without it they clean-skip (log and exit, no failures). `triage-stale` needs + nothing. Recommend a workspace-scoped key with a monthly spend cap. +2. **Curate the label allow-list.** The classifier may apply _only_ the labels in + `APPLYABLE` (top of `triage-on-open.yml` / `triage-backfill.yml`). It's + default-deny and currently holds the repo's real content labels only — + `bug, enhancement, documentation, question, Integration, SDK, framework, Agent Framework`. + `enhancement` is this repo's canonical feature type; the rarely-used `Feature Request` + synonym is deliberately omitted. Curation/disposition labels (`proposal`, `Roadmap`, + `good first issue`, `help wanted`, `release`, `invalid`, `wontfix`…) are excluded — + those are human calls. Adjust per repo. +3. **Create a `needs-info` label** (this repo doesn't have one yet). Until it exists + and gets applied, `triage-stale` is a clean no-op — it only acts on `needs-info`. +4. **First run:** dispatch `triage-backfill` with dry-run **on** and a small + `max_issues` to eyeball the proposals before letting it apply anything. + +## Safety model + +- **Constrained action space** — the LLM never posts free text. It returns + structured JSON; the workflow applies validated labels and templated comments. +- **Default-deny labels** — only allow-listed labels can be applied. +- **Confidence gates** — labels ≥ 0.75, dedup ≥ 0.8; dedup also requires the + target to be one of the candidates we actually searched. +- **Flag, never close** — duplicates get a label + a "a maintainer will confirm" + comment. Humans close. +- **Spam/low-signal gate** — already-flagged or empty-body-from-outsider issues + skip the LLM call entirely (cost guard). +- **Pinned actions** — checkout / github-script / stale are SHA-pinned. diff --git a/scripts/triage/analyze.js b/scripts/triage/analyze.js new file mode 100644 index 0000000000..8cbd98937c --- /dev/null +++ b/scripts/triage/analyze.js @@ -0,0 +1,103 @@ +// scripts/triage/analyze.js +// Shared issue-analysis used by triage-on-open + triage-backfill (single source of truth). +// Does ONE GitHub search + ONE combined Anthropic call; returns proposals only — +// no labels/comments are applied here (the caller applies per its policy). +// No npm deps: uses the passed `github` octokit + global fetch. Needs ANTHROPIC_API_KEY in env. + +const MODEL = "claude-haiku-4-5-20251001"; + +async function ask(prompt, maxTokens) { + const res = await fetch("https://api.anthropic.com/v1/messages", { + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": process.env.ANTHROPIC_API_KEY, + "anthropic-version": "2023-06-01", + }, + body: JSON.stringify({ + model: MODEL, + max_tokens: maxTokens, + messages: [{ role: "user", content: prompt }], + }), + }); + if (!res.ok) throw new Error(`Anthropic ${res.status}`); + const data = await res.json(); + const text = data.content?.[0]?.text || "{}"; + try { + return JSON.parse(text.match(/\{[\s\S]*\}/)?.[0] ?? "{}"); + } catch { + return {}; + } +} + +// Cheap spam / low-signal gate — avoids spending an LLM call on obvious junk (cost guard). +function lowSignal(issue) { + const labels = (issue.labels || []).map((l) => l.name || l); + if (labels.some((n) => n === "spam" || n === "invalid")) + return "already-flagged"; + const assoc = issue.author_association || "NONE"; + const outsider = assoc === "NONE" || assoc === "FIRST_TIMER"; + const bodyLen = (issue.body || "").replace(/\s/g, "").length; + if (outsider && bodyLen < 15) return "empty-body-from-outsider"; + return null; +} + +module.exports = async function analyzeIssue({ + github, + owner, + repo, + issue, + labelList, +}) { + if (!process.env.ANTHROPIC_API_KEY) return { skipped: "no-api-key" }; + const skip = lowSignal(issue); + if (skip) return { skipped: skip }; + + // Candidate duplicates — lexical recall (upgrade to an embeddings index later if needed). + const kw = (issue.title || "") + .replace(/[^\w\s]/g, " ") + .split(/\s+/) + .filter(Boolean) + .slice(0, 8) + .join(" "); + let candidates = []; + try { + const found = await github.rest.search.issuesAndPullRequests({ + q: `repo:${owner}/${repo} is:issue ${kw}`, + per_page: 12, + }); + candidates = found.data.items + .filter((i) => i.number !== issue.number && !i.pull_request) + .slice(0, 8); + } catch (_) { + /* search rate-limited/failed — proceed label-only */ + } + + // ONE combined call: classification + dedup together. + const candList = candidates.length + ? candidates.map((c) => `#${c.number} [${c.state}]: ${c.title}`).join("\n") + : "(none)"; + const out = await ask( + [ + "Triage this GitHub issue. Do BOTH tasks and return a single JSON object.", + "1) LABELS — choose best-fit labels from ONLY this list (canonical types: bug, feature request, question, documentation; plus a clear area label if one applies). Never invent labels; empty array if unsure.", + labelList, + "2) DUPLICATE — is the issue a duplicate of one candidate (same underlying problem, not merely similar area)?", + `Candidates:\n${candList}`, + "", + `Title: ${issue.title}`, + `Body:\n${(issue.body || "").slice(0, 4000)}`, + "", + 'Respond with ONLY JSON: {"labels": string[], "labelConfidence": number 0-1, "duplicateOf": number|null, "dupConfidence": number 0-1}', + ].join("\n"), + 400, + ); + + return { + labels: Array.isArray(out.labels) ? out.labels : [], + labelConfidence: Number(out.labelConfidence ?? 0), + duplicateOf: typeof out.duplicateOf === "number" ? out.duplicateOf : null, + dupConfidence: Number(out.dupConfidence ?? 0), + candidates: candidates.map((c) => c.number), + }; +}; From ac5571b3247274038f5a448ac3d3c3b496c28ba3 Mon Sep 17 00:00:00 2001 From: Benjamin Taylor Date: Tue, 21 Jul 2026 11:21:10 -0500 Subject: [PATCH 2/5] ci(triage): harden workflows for zizmor/GHAS (scoped perms, no template injection) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ag-ui runs zizmor via GitHub Advanced Security without CopilotKit's suppression config, so it surfaces findings CK's check hid. Address them at the source: - excessive-permissions (error): move `issues: write` from workflow level to the job, with an explanatory comment (also clears undocumented-permissions). - anonymous-definition: give each job a `name:`. - template-injection: pass workflow_dispatch inputs (dry_run/max_issues) via env and read from process.env instead of interpolating ${{ }} into the script. - secrets-outside-env (warning): inline-ignore with justification — a single read-only classification key doesn't warrant a dedicated GH Environment. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/triage-backfill.yml | 14 ++++++++++---- .github/workflows/triage-on-open.yml | 7 +++++-- .github/workflows/triage-stale.yml | 6 ++++-- scripts/triage/README.md | 7 ++++++- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/.github/workflows/triage-backfill.yml b/.github/workflows/triage-backfill.yml index 9de16ecac8..bd68c84902 100644 --- a/.github/workflows/triage-backfill.yml +++ b/.github/workflows/triage-backfill.yml @@ -14,27 +14,33 @@ on: description: "Preview only (no labels/comments applied)" type: boolean default: true -permissions: - issues: write concurrency: group: triage-backfill cancel-in-progress: false jobs: backfill: + name: backfill triage over open issues runs-on: ubuntu-latest + # Least privilege: only add labels/comments to existing issues. + permissions: + issues: write steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false # only reads scripts/triage/analyze.js; no git ops - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: + # zizmor: ignore[secrets-outside-env] single read-only classification key; a dedicated GH Environment adds setup friction with marginal benefit here ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + # Inputs via env (not inlined into the script) to avoid template-injection. + DRY_RUN: ${{ inputs.dry_run }} + MAX_ISSUES: ${{ inputs.max_issues }} with: script: | const analyzeIssue = require(`${process.env.GITHUB_WORKSPACE}/scripts/triage/analyze.js`); const { owner, repo } = context.repo; - const dryRun = ${{ inputs.dry_run }}; - const max = Number("${{ inputs.max_issues }}") || 50; + const dryRun = process.env.DRY_RUN === "true"; + const max = Number(process.env.MAX_ISSUES) || 50; if (!process.env.ANTHROPIC_API_KEY) { core.warning("ANTHROPIC_API_KEY not set — nothing to do."); return; } const sleep = (ms) => new Promise(r => setTimeout(r, ms)); diff --git a/.github/workflows/triage-on-open.yml b/.github/workflows/triage-on-open.yml index 274df08f72..6bcc01eb3d 100644 --- a/.github/workflows/triage-on-open.yml +++ b/.github/workflows/triage-on-open.yml @@ -6,20 +6,23 @@ name: triage / on-open on: issues: types: [opened] -permissions: - issues: write concurrency: group: triage-on-open-${{ github.event.issue.number }} cancel-in-progress: true jobs: triage: + name: label + dedup-flag on open runs-on: ubuntu-latest + # Least privilege: only add labels/comments to the opened issue. + permissions: + issues: write steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false # only reads scripts/triage/analyze.js; no git ops - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: + # zizmor: ignore[secrets-outside-env] single read-only classification key; a dedicated GH Environment adds setup friction with marginal benefit here ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} with: script: | diff --git a/.github/workflows/triage-stale.yml b/.github/workflows/triage-stale.yml index 5f193e0023..d4a9cf6ca1 100644 --- a/.github/workflows/triage-stale.yml +++ b/.github/workflows/triage-stale.yml @@ -6,14 +6,16 @@ on: schedule: - cron: "0 9 * * *" # daily 09:00 UTC workflow_dispatch: -permissions: - issues: write concurrency: group: triage-stale cancel-in-progress: false jobs: stale: + name: stale needs-info issues runs-on: ubuntu-latest + # Least privilege: only label/close stale needs-info issues. + permissions: + issues: write steps: - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 with: diff --git a/scripts/triage/README.md b/scripts/triage/README.md index 76887501b3..c990ad1f2a 100644 --- a/scripts/triage/README.md +++ b/scripts/triage/README.md @@ -43,4 +43,9 @@ and backfill workflows call it, so policy and safety controls live in one place. comment. Humans close. - **Spam/low-signal gate** — already-flagged or empty-body-from-outsider issues skip the LLM call entirely (cost guard). -- **Pinned actions** — checkout / github-script / stale are SHA-pinned. +- **Pinned actions** — checkout / github-script / stale are SHA-pinned, with + `persist-credentials: false` (no git ops). +- **Least privilege** — `issues: write` is scoped to the job, not the workflow; + no other token scopes are granted. +- **No template injection** — `workflow_dispatch` inputs are passed via `env` + and read from `process.env`, never interpolated into the inline script. From 95acd729288df74b08d3eb12dac6c6159108f336 Mon Sep 17 00:00:00 2001 From: Benjamin Taylor Date: Thu, 23 Jul 2026 09:42:55 -0500 Subject: [PATCH 3/5] ci(triage): make model provider pluggable (Azure OpenAI/Foundry preferred, Anthropic fallback) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit analyze.js gains a small provider abstraction selected by env: - Azure OpenAI / Foundry: AZURE_OPENAI_API_KEY (secret) + AZURE_OPENAI_ENDPOINT / AZURE_OPENAI_DEPLOYMENT (repo Variables); AZURE_OPENAI_API_VERSION optional. - Anthropic: ANTHROPIC_API_KEY (secret); ANTHROPIC_MODEL optional. - TRIAGE_PROVIDER forces one; else inferred from present creds (Azure wins if both). Both providers get the same prompt and return JSON-as-text, extracted uniformly, so the whole Foundry catalog (GPT/Llama/…) works. Workflows pass the new env through (non-secret Azure config via vars); backfill guard now checks providerConfigured(). Lets triage runs draw on shared Azure credits and drops the Anthropic spend-cap item. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/triage-backfill.yml | 11 +++- .github/workflows/triage-on-open.yml | 11 +++- scripts/triage/README.md | 11 ++-- scripts/triage/analyze.js | 73 ++++++++++++++++++++++----- 4 files changed, 87 insertions(+), 19 deletions(-) diff --git a/.github/workflows/triage-backfill.yml b/.github/workflows/triage-backfill.yml index bd68c84902..7a20d3894c 100644 --- a/.github/workflows/triage-backfill.yml +++ b/.github/workflows/triage-backfill.yml @@ -30,7 +30,14 @@ jobs: persist-credentials: false # only reads scripts/triage/analyze.js; no git ops - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: - # zizmor: ignore[secrets-outside-env] single read-only classification key; a dedicated GH Environment adds setup friction with marginal benefit here + # Model provider — set ONE (Azure preferred so runs draw on shared credits; Anthropic is the fallback). + # Non-secret Azure config lives in repo "Variables"; only the API keys are secrets. + # zizmor: ignore[secrets-outside-env] read-only inference key; dedicated GH Environment adds friction w/ marginal benefit here + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_OPENAI_DEPLOYMENT }} + AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }} + # zizmor: ignore[secrets-outside-env] read-only inference key (fallback provider) ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} # Inputs via env (not inlined into the script) to avoid template-injection. DRY_RUN: ${{ inputs.dry_run }} @@ -41,7 +48,7 @@ jobs: const { owner, repo } = context.repo; const dryRun = process.env.DRY_RUN === "true"; const max = Number(process.env.MAX_ISSUES) || 50; - if (!process.env.ANTHROPIC_API_KEY) { core.warning("ANTHROPIC_API_KEY not set — nothing to do."); return; } + if (!analyzeIssue.providerConfigured()) { core.warning("No model provider configured (set Azure or Anthropic secrets) — nothing to do."); return; } const sleep = (ms) => new Promise(r => setTimeout(r, ms)); const labels = await github.paginate(github.rest.issues.listLabelsForRepo, { owner, repo, per_page: 100 }); diff --git a/.github/workflows/triage-on-open.yml b/.github/workflows/triage-on-open.yml index 6bcc01eb3d..8c815080c4 100644 --- a/.github/workflows/triage-on-open.yml +++ b/.github/workflows/triage-on-open.yml @@ -1,7 +1,7 @@ # .github/workflows/triage-on-open.yml # Consolidated on-open triage: label + dedup-flag in ONE combined LLM call # (shared logic in scripts/triage/analyze.js). Advisory / high-confidence, -# reopen-friendly; flags duplicates but never auto-closes. No key -> clean skip. +# reopen-friendly; flags duplicates but never auto-closes. No provider -> clean skip. name: triage / on-open on: issues: @@ -22,7 +22,14 @@ jobs: persist-credentials: false # only reads scripts/triage/analyze.js; no git ops - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: - # zizmor: ignore[secrets-outside-env] single read-only classification key; a dedicated GH Environment adds setup friction with marginal benefit here + # Model provider — set ONE (Azure preferred so runs draw on shared credits; Anthropic is the fallback). + # Non-secret Azure config lives in repo "Variables"; only the API keys are secrets. + # zizmor: ignore[secrets-outside-env] read-only inference key; dedicated GH Environment adds friction w/ marginal benefit here + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_OPENAI_DEPLOYMENT }} + AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }} + # zizmor: ignore[secrets-outside-env] read-only inference key (fallback provider) ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} with: script: | diff --git a/scripts/triage/README.md b/scripts/triage/README.md index c990ad1f2a..6507311f73 100644 --- a/scripts/triage/README.md +++ b/scripts/triage/README.md @@ -16,9 +16,14 @@ and backfill workflows call it, so policy and safety controls live in one place. ## Setup -1. **`ANTHROPIC_API_KEY`** repo secret — required only by the two LLM workflows. - Without it they clean-skip (log and exit, no failures). `triage-stale` needs - nothing. Recommend a workspace-scoped key with a monthly spend cap. +1. **Model provider** — the two LLM workflows need ONE provider; with none they + clean-skip (log and exit, no failures). `triage-stale` needs nothing. + - **Azure OpenAI / Foundry (preferred — draws on shared credits):** secret + `AZURE_OPENAI_API_KEY`; repo **Variables** `AZURE_OPENAI_ENDPOINT` and + `AZURE_OPENAI_DEPLOYMENT` (optional `AZURE_OPENAI_API_VERSION`, default `2024-10-21`). + - **Anthropic (fallback):** secret `ANTHROPIC_API_KEY` (optional `ANTHROPIC_MODEL`). + - Force one with the `TRIAGE_PROVIDER` variable (`azure` | `anthropic`); otherwise + it's inferred from whichever credentials are present (Azure wins if both are set). 2. **Curate the label allow-list.** The classifier may apply _only_ the labels in `APPLYABLE` (top of `triage-on-open.yml` / `triage-backfill.yml`). It's default-deny and currently holds the repo's real content labels only — diff --git a/scripts/triage/analyze.js b/scripts/triage/analyze.js index 8cbd98937c..2f48ee34bc 100644 --- a/scripts/triage/analyze.js +++ b/scripts/triage/analyze.js @@ -1,12 +1,63 @@ // scripts/triage/analyze.js // Shared issue-analysis used by triage-on-open + triage-backfill (single source of truth). -// Does ONE GitHub search + ONE combined Anthropic call; returns proposals only — +// Does ONE GitHub search + ONE combined model call; returns proposals only — // no labels/comments are applied here (the caller applies per its policy). -// No npm deps: uses the passed `github` octokit + global fetch. Needs ANTHROPIC_API_KEY in env. +// +// Provider-agnostic (no npm deps; global fetch + the passed `github` octokit): +// • Azure OpenAI / Foundry (preferred): AZURE_OPENAI_API_KEY (secret) plus +// AZURE_OPENAI_ENDPOINT + AZURE_OPENAI_DEPLOYMENT (repo Variables); +// AZURE_OPENAI_API_VERSION optional (default below). +// • Anthropic (fallback): ANTHROPIC_API_KEY (secret); ANTHROPIC_MODEL optional. +// Selection: TRIAGE_PROVIDER ("azure"|"anthropic") forces it; otherwise inferred +// from whichever credentials are present (Azure wins when both are set). -const MODEL = "claude-haiku-4-5-20251001"; +const ANTHROPIC_MODEL = process.env.ANTHROPIC_MODEL || "claude-haiku-4-5-20251001"; +const AZURE_API_VERSION = process.env.AZURE_OPENAI_API_VERSION || "2024-10-21"; +function provider() { + const explicit = (process.env.TRIAGE_PROVIDER || "").toLowerCase(); + if (explicit === "azure" || explicit === "anthropic") return explicit; + if ( + process.env.AZURE_OPENAI_API_KEY && + process.env.AZURE_OPENAI_ENDPOINT && + process.env.AZURE_OPENAI_DEPLOYMENT + ) + return "azure"; + if (process.env.ANTHROPIC_API_KEY) return "anthropic"; + return null; +} + +function extractJson(text) { + try { + return JSON.parse((text || "").match(/\{[\s\S]*\}/)?.[0] ?? "{}"); + } catch { + return {}; + } +} + +// ONE model call. Both providers get the same prompt and return JSON as text, +// which we extract uniformly — keeps the whole Foundry catalog (GPT/Llama/…) in play. async function ask(prompt, maxTokens) { + if (provider() === "azure") { + const base = process.env.AZURE_OPENAI_ENDPOINT.replace(/\/+$/, ""); + const url = `${base}/openai/deployments/${process.env.AZURE_OPENAI_DEPLOYMENT}/chat/completions?api-version=${AZURE_API_VERSION}`; + const res = await fetch(url, { + method: "POST", + headers: { + "content-type": "application/json", + "api-key": process.env.AZURE_OPENAI_API_KEY, + }, + body: JSON.stringify({ + messages: [{ role: "user", content: prompt }], + max_tokens: maxTokens, + temperature: 0, + }), + }); + if (!res.ok) throw new Error(`Azure OpenAI ${res.status}`); + const data = await res.json(); + return extractJson(data.choices?.[0]?.message?.content); + } + const res = await fetch("https://api.anthropic.com/v1/messages", { method: "POST", headers: { @@ -15,22 +66,17 @@ async function ask(prompt, maxTokens) { "anthropic-version": "2023-06-01", }, body: JSON.stringify({ - model: MODEL, + model: ANTHROPIC_MODEL, max_tokens: maxTokens, messages: [{ role: "user", content: prompt }], }), }); if (!res.ok) throw new Error(`Anthropic ${res.status}`); const data = await res.json(); - const text = data.content?.[0]?.text || "{}"; - try { - return JSON.parse(text.match(/\{[\s\S]*\}/)?.[0] ?? "{}"); - } catch { - return {}; - } + return extractJson(data.content?.[0]?.text); } -// Cheap spam / low-signal gate — avoids spending an LLM call on obvious junk (cost guard). +// Cheap spam / low-signal gate — avoids spending a model call on obvious junk (cost guard). function lowSignal(issue) { const labels = (issue.labels || []).map((l) => l.name || l); if (labels.some((n) => n === "spam" || n === "invalid")) @@ -49,7 +95,7 @@ module.exports = async function analyzeIssue({ issue, labelList, }) { - if (!process.env.ANTHROPIC_API_KEY) return { skipped: "no-api-key" }; + if (!provider()) return { skipped: "no-model-provider" }; const skip = lowSignal(issue); if (skip) return { skipped: skip }; @@ -101,3 +147,6 @@ module.exports = async function analyzeIssue({ candidates: candidates.map((c) => c.number), }; }; + +// Lets callers (e.g. the backfill early-guard) check config without making a call. +module.exports.providerConfigured = () => provider() !== null; From 54f84dcbbf8b4e94e42b7f17e59de6995d4f2825 Mon Sep 17 00:00:00 2001 From: Benjamin Taylor Date: Thu, 23 Jul 2026 10:16:08 -0500 Subject: [PATCH 4/5] ci(triage): harden runners (egress audit, scoped perms, timeouts, endpoint pin) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From a 3-front security audit (Actions supply-chain, LLM prompt-injection, secrets) against current best practices. The LLM/output surface was already contained; these close the runner/cost-layer gaps: - Harden-Runner (SHA-pinned) as the first step in every job, egress-policy: audit — baseline the network, then flip to block with the commented allow-list so the model key can only reach GitHub + the model host (the tj-actions lesson: pinning alone doesn't stop a bumped/transitive dep from exfiltrating with unrestricted egress). - Least privilege: top-level `permissions: {}` default + job-scoped `contents: read` (needed by checkout) + `issues: write`. Nothing else. - `timeout-minutes` on every job (a hung model fetch shouldn't burn the 6h default). - analyze.js refuses to POST the key to any non-*.azure.com host (guards the mutable endpoint Variable); Anthropic host is already hardcoded. Already covered, not re-added: zizmor in CI + Renovate exist in both repos. Operator follow-ups (documented in README): provider spend cap, flip egress to block after baseline, confirm Renovate doesn't auto-merge action bumps. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/triage-backfill.yml | 16 +++++++++++++++- .github/workflows/triage-on-open.yml | 16 +++++++++++++++- .github/workflows/triage-stale.yml | 6 ++++++ scripts/triage/README.md | 24 ++++++++++++++++++++---- scripts/triage/analyze.js | 16 +++++++++++++++- 5 files changed, 71 insertions(+), 7 deletions(-) diff --git a/.github/workflows/triage-backfill.yml b/.github/workflows/triage-backfill.yml index 7a20d3894c..82407f7c6e 100644 --- a/.github/workflows/triage-backfill.yml +++ b/.github/workflows/triage-backfill.yml @@ -14,6 +14,8 @@ on: description: "Preview only (no labels/comments applied)" type: boolean default: true +# Default to no privileges; each job opts in to exactly what it needs. +permissions: {} concurrency: group: triage-backfill cancel-in-progress: false @@ -21,10 +23,22 @@ jobs: backfill: name: backfill triage over open issues runs-on: ubuntu-latest - # Least privilege: only add labels/comments to existing issues. + # Least privilege: read the repo (for analyze.js) + write issue labels/comments. permissions: + contents: read issues: write + timeout-minutes: 30 # backfill iterates many issues; still bounded steps: + - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit # baseline first — review the run's network report, then flip to "block" with the allow-list below + # block-mode allow-list (uncomment + set the model host when flipping): + # allowed-endpoints: > + # github.com:443 + # api.github.com:443 + # objects.githubusercontent.com:443 + # api.anthropic.com:443 + # .openai.azure.com:443 # or *.services.ai.azure.com for Foundry - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false # only reads scripts/triage/analyze.js; no git ops diff --git a/.github/workflows/triage-on-open.yml b/.github/workflows/triage-on-open.yml index 8c815080c4..9b7482ed94 100644 --- a/.github/workflows/triage-on-open.yml +++ b/.github/workflows/triage-on-open.yml @@ -6,6 +6,8 @@ name: triage / on-open on: issues: types: [opened] +# Default to no privileges; each job opts in to exactly what it needs. +permissions: {} concurrency: group: triage-on-open-${{ github.event.issue.number }} cancel-in-progress: true @@ -13,10 +15,22 @@ jobs: triage: name: label + dedup-flag on open runs-on: ubuntu-latest - # Least privilege: only add labels/comments to the opened issue. + # Least privilege: read the repo (for analyze.js) + write issue labels/comments. permissions: + contents: read issues: write + timeout-minutes: 10 # a hung model fetch shouldn't burn the 6h default steps: + - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit # baseline first — review the run's network report, then flip to "block" with the allow-list below + # block-mode allow-list (uncomment + set the model host when flipping): + # allowed-endpoints: > + # github.com:443 + # api.github.com:443 + # objects.githubusercontent.com:443 + # api.anthropic.com:443 + # .openai.azure.com:443 # or *.services.ai.azure.com for Foundry - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false # only reads scripts/triage/analyze.js; no git ops diff --git a/.github/workflows/triage-stale.yml b/.github/workflows/triage-stale.yml index d4a9cf6ca1..7ee1fb363d 100644 --- a/.github/workflows/triage-stale.yml +++ b/.github/workflows/triage-stale.yml @@ -6,6 +6,8 @@ on: schedule: - cron: "0 9 * * *" # daily 09:00 UTC workflow_dispatch: +# Default to no privileges; each job opts in to exactly what it needs. +permissions: {} concurrency: group: triage-stale cancel-in-progress: false @@ -16,7 +18,11 @@ jobs: # Least privilege: only label/close stale needs-info issues. permissions: issues: write + timeout-minutes: 10 steps: + - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit # no secrets/external calls here; GitHub API only. Block-mode: github.com:443 + api.github.com:443 - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 with: # MVP: only auto-progress issues we explicitly asked info on and got silence. diff --git a/scripts/triage/README.md b/scripts/triage/README.md index 6507311f73..c5bb722306 100644 --- a/scripts/triage/README.md +++ b/scripts/triage/README.md @@ -48,9 +48,25 @@ and backfill workflows call it, so policy and safety controls live in one place. comment. Humans close. - **Spam/low-signal gate** — already-flagged or empty-body-from-outsider issues skip the LLM call entirely (cost guard). -- **Pinned actions** — checkout / github-script / stale are SHA-pinned, with - `persist-credentials: false` (no git ops). -- **Least privilege** — `issues: write` is scoped to the job, not the workflow; - no other token scopes are granted. +- **Pinned actions** — checkout / github-script / stale / harden-runner are + SHA-pinned, with `persist-credentials: false` (no git ops). +- **Least privilege** — every workflow defaults to `permissions: {}`; each job + opts into only `contents: read` + `issues: write`. Nothing else is granted. - **No template injection** — `workflow_dispatch` inputs are passed via `env` and read from `process.env`, never interpolated into the inline script. +- **Egress monitoring** — [StepSecurity Harden-Runner](https://github.com/step-security/harden-runner) + runs first in every job (`egress-policy: audit`). Review the network report, + then flip to `block` with the allow-list commented in each workflow so the + model key can only reach GitHub + your model host. +- **Endpoint pinning** — `analyze.js` refuses to send the key to any non-Azure + host (guards against the endpoint `var` being repointed). +- **Bounded runtime** — `timeout-minutes` on every job caps a hung model call. + +### Not yet automated (operator actions) + +- **Provider spend cap** — set a hard quota/spend limit on the model provider + (Azure deployment TPM quota / Anthropic workspace monthly limit). This is the + real backstop against issue-flood cost-DoS (an event-driven workflow can't hold + a global rate limit); do it when you configure the provider. +- **Flip Harden-Runner to `block`** after one `audit` baseline run. +- **Confirm Renovate doesn't auto-merge** action SHA bumps (review them by hand). diff --git a/scripts/triage/analyze.js b/scripts/triage/analyze.js index 2f48ee34bc..030239f4d2 100644 --- a/scripts/triage/analyze.js +++ b/scripts/triage/analyze.js @@ -11,7 +11,8 @@ // Selection: TRIAGE_PROVIDER ("azure"|"anthropic") forces it; otherwise inferred // from whichever credentials are present (Azure wins when both are set). -const ANTHROPIC_MODEL = process.env.ANTHROPIC_MODEL || "claude-haiku-4-5-20251001"; +const ANTHROPIC_MODEL = + process.env.ANTHROPIC_MODEL || "claude-haiku-4-5-20251001"; const AZURE_API_VERSION = process.env.AZURE_OPENAI_API_VERSION || "2024-10-21"; function provider() { @@ -39,6 +40,19 @@ function extractJson(text) { // which we extract uniformly — keeps the whole Foundry catalog (GPT/Llama/…) in play. async function ask(prompt, maxTokens) { if (provider() === "azure") { + // Defense-in-depth: the endpoint is a mutable, non-secret repo Variable. Refuse + // to send the API key to anything but an Azure host (covers *.openai.azure.com, + // *.cognitiveservices.azure.com, *.services.ai.azure.com). Harden-Runner egress + // block-mode is the hard control; this is the cheap in-code guard. + let host; + try { + host = new URL(process.env.AZURE_OPENAI_ENDPOINT).host; + } catch { + throw new Error("AZURE_OPENAI_ENDPOINT is not a valid URL"); + } + if (!host.endsWith(".azure.com")) { + throw new Error(`Refusing to send key to non-Azure host: ${host}`); + } const base = process.env.AZURE_OPENAI_ENDPOINT.replace(/\/+$/, ""); const url = `${base}/openai/deployments/${process.env.AZURE_OPENAI_DEPLOYMENT}/chat/completions?api-version=${AZURE_API_VERSION}`; const res = await fetch(url, { From f633e24486f0b8fed81433e1395e04334638e248 Mon Sep 17 00:00:00 2001 From: Benjamin Taylor Date: Thu, 23 Jul 2026 10:19:58 -0500 Subject: [PATCH 5/5] ci(triage): document each permission scope (clears zizmor undocumented-permissions) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-scope rationale comments on the job permissions blocks so the GHAS/zizmor undocumented-permissions notices clear — a truly clean static-analysis pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/triage-backfill.yml | 4 ++-- .github/workflows/triage-on-open.yml | 4 ++-- .github/workflows/triage-stale.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/triage-backfill.yml b/.github/workflows/triage-backfill.yml index 82407f7c6e..659e060828 100644 --- a/.github/workflows/triage-backfill.yml +++ b/.github/workflows/triage-backfill.yml @@ -25,8 +25,8 @@ jobs: runs-on: ubuntu-latest # Least privilege: read the repo (for analyze.js) + write issue labels/comments. permissions: - contents: read - issues: write + contents: read # checkout reads scripts/triage/analyze.js + issues: write # apply labels + post the templated dup comment timeout-minutes: 30 # backfill iterates many issues; still bounded steps: - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 diff --git a/.github/workflows/triage-on-open.yml b/.github/workflows/triage-on-open.yml index 9b7482ed94..6a8dc08abe 100644 --- a/.github/workflows/triage-on-open.yml +++ b/.github/workflows/triage-on-open.yml @@ -17,8 +17,8 @@ jobs: runs-on: ubuntu-latest # Least privilege: read the repo (for analyze.js) + write issue labels/comments. permissions: - contents: read - issues: write + contents: read # checkout reads scripts/triage/analyze.js + issues: write # apply labels + post the templated dup comment timeout-minutes: 10 # a hung model fetch shouldn't burn the 6h default steps: - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 diff --git a/.github/workflows/triage-stale.yml b/.github/workflows/triage-stale.yml index 7ee1fb363d..c9f3b6de4c 100644 --- a/.github/workflows/triage-stale.yml +++ b/.github/workflows/triage-stale.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest # Least privilege: only label/close stale needs-info issues. permissions: - issues: write + issues: write # label/close stale needs-info issues timeout-minutes: 10 steps: - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0