From 98459c9ca6b595c18526fc4791c30683665fff06 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 25 Jul 2026 13:22:28 +0000 Subject: [PATCH 1/2] feat(automations): complete Dependabot triage follow-ups --- .../standard/dependabot-triage/SKILL.md | 22 +++++----- .../__tests__/dependabot-triage.test.ts | 40 +++++++++++++++---- .../server/automations/dependabot-triage.ts | 29 +++++--------- 3 files changed, 53 insertions(+), 38 deletions(-) diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/dependabot-triage/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/dependabot-triage/SKILL.md index a270ddeac..965a1c1b5 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/dependabot-triage/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/dependabot-triage/SKILL.md @@ -10,7 +10,7 @@ You are a dependency-security triage specialist. Use GitHub's Dependabot alert d - Use the GitHub access already available in the task environment. Prefer `gh api` for alert retrieval, keep scheduled/background runs read-only, and honor any repository scope, Slack channel, run mode, or automation policy supplied in the request. Human-triggered or automation-started follow-up execution work belongs in the dependency-update specialist, not in this triage scan itself. + Use the GitHub access already available in the task environment. Prefer `gh api` for alert retrieval, keep scheduled/background runs read-only, and honor any repository scope, Slack channel, run mode, or automation policy supplied in the request. Human-triggered or automation-started follow-up execution work belongs in the dependency-update specialist, not in this triage scan itself. Make each scan completion-oriented: track the current alert count, account for related open PRs, and launch the smallest remediation that can clear a cohesive alert bundle. @@ -22,7 +22,8 @@ You are a dependency-security triage specialist. Use GitHub's Dependabot alert d - Inspect current open Dependabot alerts for each repository in scope. Prefer repository-level API calls so every finding already maps to a single launch target. + Inspect current open Dependabot alerts for each repository in scope and count them. Prefer repository-level API calls so every finding already maps to a single launch target. + Inspect each repository's open pull requests for dependency updates or Dependabot alert references. Treat a PR that covers an alert as in-flight remediation, not a reason to create duplicate work; record which alerts remain uncovered. Collect only the evidence needed to rank each alert: repository, alert URL or number, dependency package name, ecosystem, manifest path, dependency scope if available, severity, summary, vulnerable range, first patched version, and whether the alert appears direct or transitive. Prioritize alerts by severity, reachable/runtime impact, direct dependency ownership, availability of a clear patched version, number of repositories or manifests affected, and confidence that a focused update can land without broad churn. Deprioritize already-closed alerts, duplicated grouped updates that would create unnecessary version churn, low-confidence ecosystem mismatches, and alerts whose repository ownership or update path is unclear. @@ -33,18 +34,17 @@ You are a dependency-security triage specialist. Use GitHub's Dependabot alert d Start with the repository scope, overall risk, and the highest-priority update candidate or no-op result. - Submit actionable candidates with `submit_automation_work_items`. Submit up to 3 `act` work items for the best cohesive candidates, keep each one scoped to one repository from `repository_scope`, submit at most one work item for each `targetEnvironmentId`, and only target repositories that appear in the `Repository environments` section. - Do not submit suggestion work items (they are rejected), do not fall back to bare-repo execution, and do not post a Slack launch announcement after the item is submitted. The later execution task stays silent while work is in flight and uses Slack only when it needs input, hits a blocker, or has a meaningful result. + Submit actionable candidates with `submit_automation_work_items`. Submit up to 3 `act` work items for the best cohesive candidates, keep each one scoped to one repository from `repository_scope`, submit at most one work item for each `targetEnvironmentId`, and only target repositories that appear in the `Repository environments` section. When multiple related alerts share a lockfile or dependency surface, one task should aim to clear every actionable alert in that cohesive bundle, not merely the highest-severity alert. + Do not submit suggestion work items (they are rejected) or fall back to bare-repo execution. Do not duplicate work already covered by a related open PR; include the PR and any uncovered alerts in the status message instead. The later execution task stays silent while work is in flight and uses Slack only when it needs input, hits a blocker, or has a meaningful result. Write action-first titles such as `Update lodash in apps/web to resolve high-severity Dependabot alert` or `Bundle pnpm lockfile refresh for vulnerable ws alerts in worker`. Every work item must target exactly one repository from `repository_scope`. When a `Repository environments` section is present, copy the matching `targetEnvironmentId` only for repositories explicitly listed there. Do not invent environment IDs or reuse one repository's environment ID for another repository. Use `security` for alerts that represent real vulnerability remediation work, `chore` for lower-risk dependency maintenance bundles, and `improvement` only when the strongest follow-up is tooling or validation hardening instead of the update itself. - In `investigationContext`, include `$update-dependencies`, the alert URL or number, alert summary, package name, ecosystem, manifest path, vulnerable range, first patched version, severity, the exact GitHub CLI commands used during triage, whether the affected package likely touches a running service or user-facing web surface, and what the follow-up task must verify before shipping the update. - Make the `executionPrompt` start with `$update-dependencies` and prefer the smallest cohesive execution scope that is likely to clear the alert: one package when enough, or one tightly related manifest/workspace group when the remediation needs aligned updates. Do not submit a broad sweep. + In `investigationContext`, include `$update-dependencies`, the alert URL or number, alert summary, package name, ecosystem, manifest path, vulnerable range, first patched version, severity, related open PRs and the alerts they cover, the exact GitHub CLI commands used during triage, whether the affected package likely touches a running service or user-facing web surface, and what the follow-up task must verify before shipping the update. + Make the `executionPrompt` start with `$update-dependencies` and prefer the smallest cohesive execution scope that is likely to clear the alert bundle: one package when enough, or one tightly related manifest/workspace group when the remediation needs aligned updates. Require the follow-up task to re-check the targeted alerts before delivery and leave no actionable alerts from that bundle open, unless an existing related PR already covers them. Do not submit a broad sweep. Prefer focused follow-ups: one alert or one tightly related alert bundle per task. It is fine to include multiple affected workspaces or a small related dependency bundle when that is what the remediation actually requires. Do not submit broad "update everything" chores from this skill. - If `submit_automation_work_items` succeeds for one or more work items, do not call `post_to_slack_channel` and do not post a separate Slack summary unless the request explicitly says the scan itself should report launch outcomes. - If `slack_channel_id` is present and there is a GitHub setup/auth blocker (for example missing or suspended access to Dependabot alerts), post a concise report there with `post_to_slack_channel` so the broken run does not disappear silently. Treat repository-level gaps such as Dependabot alerts being disabled for a repository, a repository returning zero open alerts, or a repository falling outside configured environment coverage as non-blocking no-op findings for this run, not as Slack-worthy blockers. When the run is otherwise clean — no actionable alerts, no eligible configured-environment candidates, no configured repositories, or only non-launchable findings — stay quiet: do not post to Slack, and end with a terse internal note. A clean read-only run is not worth a channel message. - Keep any `post_to_slack_channel` blocker report plain-language and manager-readable, and do not paste raw GitHub CLI commands, `gh api` invocations, or command transcripts into Slack. The exact commands belong only in work item `investigationContext`, never in the channel report. - End the task response with a terse internal note when follow-up items were submitted or the run was clean, or the concise blocker report when a Slack post was needed. + If `slack_channel_id` is present, always post one concise scan summary there with `post_to_slack_channel`: total open alert count, per-repository counts, relevant open PRs (or that none are open), and whether a remediation task was started, deferred because an existing PR covers the alerts, or blocked. This is a status summary, not a launch announcement. + Keep the `post_to_slack_channel` summary plain-language and manager-readable, and do not paste raw GitHub CLI commands, `gh api` invocations, or command transcripts into Slack. The exact commands belong only in work item `investigationContext`, never in the channel report. + End the task response with a terse internal note after posting the scan summary, including when there are no actionable alerts or GitHub access is blocked. @@ -54,5 +54,5 @@ You are a dependency-security triage specialist. Use GitHub's Dependabot alert d The scan stayed read-only for scheduled/background runs. Actionable scheduled findings were submitted as environment-backed `act` work items. The final report or submitted work items were concise, prioritized, plain-language, and free of raw command transcripts, so they are safe to post in Slack. -Clean scans stayed silent in Slack; only setup/auth blockers were reported there. +Every configured Slack scan posted the current alert count, related open PR state, and remediation disposition. diff --git a/packages/sdk/src/server/automations/__tests__/dependabot-triage.test.ts b/packages/sdk/src/server/automations/__tests__/dependabot-triage.test.ts index 5efa940ca..8f2ecafbc 100644 --- a/packages/sdk/src/server/automations/__tests__/dependabot-triage.test.ts +++ b/packages/sdk/src/server/automations/__tests__/dependabot-triage.test.ts @@ -71,7 +71,7 @@ describe('dependabotTriageJob buildScanTask', () => { mockLoadAutomationThreadFeedbackContext.mockResolvedValue(null); }); - it('scopes the scan to environment-backed GitHub repositories and stamps the provider', async () => { + it('scans all active GitHub repositories while keeping remediation environment-backed', async () => { mockGetActiveGitHubRepositoryFullNames.mockResolvedValue([ 'acme/api', 'acme/no-environment', @@ -89,10 +89,15 @@ describe('dependabotTriageJob buildScanTask', () => { throw new Error('expected a scan build'); } + const [payload] = result.payloads; + if (!payload) { + throw new Error('expected a scan payload'); + } + expect(result.payloads).toHaveLength(1); - expect(result.payloads[0]).toMatchObject({ + expect(payload).toMatchObject({ repo: ALL_REPOSITORIES, - selectedRepositories: ['acme/api'], + selectedRepositories: ['acme/api', 'acme/no-environment'], sourceControlProvider: 'github', suggestionSource: 'dependabot_triage', }); @@ -103,9 +108,22 @@ describe('dependabotTriageJob buildScanTask', () => { 'acme/api', 'acme/no-environment', ]); + expect(payload.description).toContain( + 'count its current open Dependabot alerts', + ); + expect(payload.description).toContain('inspect its open pull requests'); + expect(payload.description).toContain( + 'always post one concise status message', + ); + expect(payload.description).toContain( + 'total number of open Dependabot alerts, the per-repository counts', + ); + expect(payload.description).toContain( + 'do not submit duplicate work for alerts it covers', + ); }); - it('skips when no active GitHub repository is backed by an environment', async () => { + it('scans and reports repositories without environments without permitting remediation launches', async () => { mockGetActiveGitHubRepositoryFullNames.mockResolvedValue([ 'acme/no-environment', ]); @@ -115,10 +133,18 @@ describe('dependabotTriageJob buildScanTask', () => { const result = await config.buildScanTask(buildScanTaskParams()); - expect(result).toEqual({ - kind: 'skip', - reason: 'No active GitHub repositories have configured environments', + expect(result.kind).toBe('scan'); + + if (result.kind !== 'scan') { + throw new Error('expected a scan build'); + } + + expect(result.payloads[0]).toMatchObject({ + selectedRepositories: ['acme/no-environment'], }); + expect(result.payloads[0]?.description).toContain( + 'Only consider repositories that appear in the "Repository environments" list below', + ); }); it('skips when GitHub is not configured', async () => { diff --git a/packages/sdk/src/server/automations/dependabot-triage.ts b/packages/sdk/src/server/automations/dependabot-triage.ts index 53c4c87ca..0399f0a95 100644 --- a/packages/sdk/src/server/automations/dependabot-triage.ts +++ b/packages/sdk/src/server/automations/dependabot-triage.ts @@ -1,7 +1,6 @@ import { buildRepositoryCoverage, formatRepositoryEnvironmentLines, - getEnvironmentBackedCoverage, type RepositoryCoverage, } from '@roomote/cloud-agents/server'; import { ALL_REPOSITORIES } from '@roomote/types'; @@ -42,17 +41,20 @@ function buildDependabotTriagePrompt({ const repositoryEnvironmentSection = repositoryEnvironmentScope ? `\nRepository environments:\n${repositoryEnvironmentScope}\n` : ''; - const followUpInstructions = `If you find actionable candidates, submit up to 3 \`act\` automation work items with \`submit_automation_work_items\`. Do not submit any suggestion work items; they are rejected. Submit at most one work item for each \`targetEnvironmentId\`. Only consider repositories that appear in the "Repository environments" list below. Do not fall back to bare-repo launches. Pick the highest-priority cohesive update bundles across the eligible repositories. Start with the narrowest security fix that is likely to work, but if the alert realistically needs an aligned lockfile refresh, multiple affected workspaces, or a small related dependency bundle, submit that broader cohesive remediation instead of deferring. Do not turn one alert into a broad maintenance sweep. + const followUpInstructions = `For every repository in scope, count its current open Dependabot alerts and inspect its open pull requests for dependency updates or alert references that could already address them. Treat a related open PR as in-flight remediation: do not submit duplicate work for alerts it covers, but include it in the scan summary and identify any alerts it does not cover. + +If you find actionable candidates, submit up to 3 \`act\` automation work items with \`submit_automation_work_items\`. Do not submit any suggestion work items; they are rejected. Submit at most one work item for each \`targetEnvironmentId\`. Only consider repositories that appear in the "Repository environments" list below. Do not fall back to bare-repo launches. Pick the highest-priority cohesive remediation bundle across the eligible repositories. For a repository with multiple related lockfile or dependency alerts, create one task that aims to resolve every actionable open alert in that cohesive bundle, not only the first alert inspected. Start with the narrowest security fix that is likely to work, but if the alerts realistically need an aligned lockfile refresh, multiple affected workspaces, or a small related dependency bundle, submit that broader cohesive remediation instead of deferring. Do not turn unrelated alerts into a broad maintenance sweep. Each submitted act item must: - target exactly one repository from repository_scope - copy the matching \`targetEnvironmentId\` from the "Repository environments" list - include \`executionPrompt\` that starts with \`$update-dependencies\` -- include investigationContext with the alert URL or number, alert summary, package, ecosystem, manifest path, severity, vulnerable range, first patched version, the exact GitHub CLI commands used during triage, and the validation the execution task must perform before opening a PR +- state that success means re-checking the targeted alert bundle and leaving no actionable alerts from that bundle open, unless an existing related PR already covers them +- include investigationContext with the alert URL or number, alert summary, package, ecosystem, manifest path, severity, vulnerable range, first patched version, related open PRs and the alerts they cover, the exact GitHub CLI commands used during triage, and the validation the execution task must perform before opening a PR -If \`submit_automation_work_items\` succeeds for one or more act items, do not call \`${promptContext.postToolName}\` and do not post a launch announcement. Each execution task starts silently and creates ${promptContext.surfaceLabel} output only later if it needs input, hits a blocker, or finishes with a result. End the task response with a terse internal note that action items were submitted. +After triage, always post one concise status message to the configured ${promptContext.surfaceLabel} channel with \`${promptContext.postToolName}\`. State the total number of open Dependabot alerts, the per-repository counts, related open PRs (or that none are open), and whether a remediation task was started, deferred because an existing PR covers the alerts, or blocked. Keep it manager-readable and do not paste raw GitHub CLI commands, \`gh api\` invocations, or command transcripts. Each execution task starts silently and creates ${promptContext.surfaceLabel} output only later if it needs input, hits a blocker, or finishes with a result. End the task response with a terse internal note after posting the status. -If there are no actionable alerts, no eligible configured-environment candidates, or no configured environment coverage, do not post to ${promptContext.surfaceLabel}; end with a terse internal note. Treat repository-level gaps such as Dependabot alerts being disabled for a repository, a repository returning zero open alerts, or a repository falling outside configured environment coverage as non-blocking no-op findings for this run, not as GitHub setup/auth blockers worth a ${promptContext.surfaceLabel} post. A clean read-only run is not worth a channel message. Post a concise report to the configured ${promptContext.surfaceLabel} channel with \`${promptContext.postToolName}\` only for GitHub setup/auth blockers (for example missing or suspended Dependabot alert access), so configuration failures do not disappear silently. Keep any such report plain-language and manager-readable, and do not paste the raw GitHub CLI commands, \`gh api\` invocations, or command transcripts into ${promptContext.surfaceLabel}; the exact commands belong only in work item \`investigationContext\`.`; +Treat repository-level gaps such as Dependabot alerts being disabled for a repository, a repository returning zero open alerts, or a repository falling outside configured environment coverage as reportable scan outcomes, not as GitHub setup/auth blockers. If GitHub setup or alert access is blocked, post the same concise status message with the blocker and the counts that could be determined.`; return `$dependabot-triage @@ -90,19 +92,6 @@ export const dependabotTriageJob = createScheduledTriageJob({ const selectedRepositories = await getActiveGitHubRepositoryFullNames(); const repositoryCoverage = await buildRepositoryCoverage(selectedRepositories); - // Dependabot follow-ups must run validation before opening PRs, so the - // scan only targets repositories backed by a configured environment. - const environmentBackedRepositories = getEnvironmentBackedCoverage( - repositoryCoverage, - ).map((coverage) => coverage.repositoryFullName); - - if (environmentBackedRepositories.length === 0) { - return { - kind: 'skip', - reason: 'No active GitHub repositories have configured environments', - }; - } - const recentThreadFeedback = await loadAutomationThreadFeedbackContext({ automationKey: 'dependabot_triage', slackChannelId: channelId, @@ -114,12 +103,12 @@ export const dependabotTriageJob = createScheduledTriageJob({ payloads: [ { repo: ALL_REPOSITORIES, - selectedRepositories: environmentBackedRepositories, + selectedRepositories, sourceControlProvider: 'github', description: buildDependabotTriagePrompt({ channelId, destination, - repositoryFullNames: environmentBackedRepositories, + repositoryFullNames: selectedRepositories, repositoryCoverage, manualTrigger, recentThreadFeedback, From f0fe328ab287e8eabfd41c9ea8663e0b41077616 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 25 Jul 2026 13:31:11 +0000 Subject: [PATCH 2/2] fix(automations): skip empty Dependabot scans --- .../automations/__tests__/dependabot-triage.test.ts | 13 +++++++++++++ .../sdk/src/server/automations/dependabot-triage.ts | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/packages/sdk/src/server/automations/__tests__/dependabot-triage.test.ts b/packages/sdk/src/server/automations/__tests__/dependabot-triage.test.ts index 8f2ecafbc..a0e462123 100644 --- a/packages/sdk/src/server/automations/__tests__/dependabot-triage.test.ts +++ b/packages/sdk/src/server/automations/__tests__/dependabot-triage.test.ts @@ -147,6 +147,19 @@ describe('dependabotTriageJob buildScanTask', () => { ); }); + it('skips when there are no active GitHub repositories', async () => { + mockGetActiveGitHubRepositoryFullNames.mockResolvedValue([]); + + const result = await config.buildScanTask(buildScanTaskParams()); + + expect(result).toEqual({ + kind: 'skip', + reason: 'No active GitHub repositories', + }); + expect(mockBuildRepositoryCoverage).not.toHaveBeenCalled(); + expect(mockLoadAutomationThreadFeedbackContext).not.toHaveBeenCalled(); + }); + it('skips when GitHub is not configured', async () => { mockHasActiveGitHubInstallation.mockResolvedValue(false); diff --git a/packages/sdk/src/server/automations/dependabot-triage.ts b/packages/sdk/src/server/automations/dependabot-triage.ts index 0399f0a95..d7365e643 100644 --- a/packages/sdk/src/server/automations/dependabot-triage.ts +++ b/packages/sdk/src/server/automations/dependabot-triage.ts @@ -90,6 +90,13 @@ export const dependabotTriageJob = createScheduledTriageJob({ // leaves the run's source-control provider ambiguous and GitHub token // minting then fails on the non-GitHub repository names. const selectedRepositories = await getActiveGitHubRepositoryFullNames(); + if (selectedRepositories.length === 0) { + return { + kind: 'skip', + reason: 'No active GitHub repositories', + }; + } + const repositoryCoverage = await buildRepositoryCoverage(selectedRepositories); const recentThreadFeedback = await loadAutomationThreadFeedbackContext({