Skip to content

Commit 6ad4a6b

Browse files
committed
fix(sweep): dedup the re-gate fan-out so a burst collapses to one effective sweep
The ~2-min cron enqueues a fan-out job each tick, but fanOutAgentRegateSweepJobs had no global dedup. When a burst of fan-out jobs ran at once — a deploy-restart cron catch-up, or fan-out jobs that queued behind a heavy per-PR re-review backlog and then drained together — EACH one enqueued a per-repo sweep before the per-repo dispatch-stamp in-flight guard could engage, producing redundant overlapping sweeps (observed ~3x on the metagraphed dry-run). Those redundant sweeps tripled the per-PR load, which delayed the next fan-out, which then burst in turn — a self-sustaining cascade. Add an atomic fan-out dedup: claimRegateFanoutSlot does a conditional UPDATE on the global_agent_controls singleton (new last_regate_fanout_at column, migration 0063) that matches only when the last fan-out is unset or older than the dedup window. D1 serializes writes, so a burst collapses to exactly ONE winner per window; the rest get 0 changes and skip (audited deduped). One effective fan-out per window keeps the per-PR load bounded, which stops the backlog that was delaying subsequent fan-outs — breaking the cascade at its source. Fail-open on a driver error so the fleet never stalls.
1 parent 398d942 commit 6ad4a6b

6 files changed

Lines changed: 94 additions & 2 deletions

File tree

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
-- Fan-out dedup marker: collapse a burst of re-gate fan-out jobs to ONE effective fan-out per window.
2+
--
3+
-- BEFORE: the ~2-min cron enqueues an agent-regate-sweep fan-out job each tick; fanOutAgentRegateSweepJobs runs it
4+
-- with no global dedup. When a burst of fan-out jobs runs at once — a deploy-restart cron catch-up, or fan-out
5+
-- jobs that queued behind a heavy per-PR re-review backlog and then drained together — EACH one enqueues a
6+
-- per-repo sweep before the per-repo dispatch-stamp in-flight guard (0062 / #audit-sweep-dispatch-stamp) can
7+
-- engage, producing redundant overlapping sweeps (observed ~3x on the metagraphed dry-run).
8+
--
9+
-- AFTER: claimRegateFanoutSlot performs an atomic conditional UPDATE on this singleton column — D1 serializes
10+
-- writes, so only ONE concurrent fan-out's UPDATE matches the "unset or older than the dedup window" predicate
11+
-- and proceeds; the rest get 0 changes and skip. One effective fan-out per window keeps the per-PR load bounded,
12+
-- which in turn stops the backlog that was delaying subsequent fan-outs (the cascade that caused the bursts).
13+
--
14+
-- Reuses the global_agent_controls singleton (0059); nullable / no default → backward-compatible (NULL = no
15+
-- fan-out has claimed the slot yet, so the first one proceeds).
16+
ALTER TABLE global_agent_controls ADD COLUMN last_regate_fanout_at TEXT;

src/db/repositories.ts

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1948,6 +1948,27 @@ export async function isGlobalAgentFrozen(env: Env): Promise<boolean> {
19481948
}
19491949
}
19501950

1951+
/** Atomic re-gate fan-out dedup (#audit-fanout-dedup): claim the global fan-out slot for this window. The
1952+
* conditional UPDATE on the singleton matches only when the last fan-out is unset or older than `windowMs`. D1
1953+
* serializes writes, so when a BURST of fan-out jobs runs at once (a deploy-restart cron catch-up, or fan-out
1954+
* jobs that queued behind a per-PR backlog and drained together) exactly ONE wins the slot (changes === 1); the
1955+
* rest get 0 changes and skip, collapsing the burst to a single effective fan-out. Fail-open on a driver error
1956+
* (return true → the sweep still runs, degrading to the pre-dedup behaviour rather than stalling the fleet). */
1957+
export async function claimRegateFanoutSlot(env: Env, now: string, windowMs: number): Promise<boolean> {
1958+
const threshold = new Date(Date.parse(now) - windowMs).toISOString();
1959+
try {
1960+
const result = await env.DB.prepare(
1961+
"UPDATE global_agent_controls SET last_regate_fanout_at = ?1 WHERE id = 'singleton' AND (last_regate_fanout_at IS NULL OR last_regate_fanout_at < ?2)",
1962+
)
1963+
.bind(now, threshold)
1964+
.run();
1965+
/* v8 ignore next -- D1 update metadata normally includes changes; the ?? 0 fallback protects driver anomalies. */
1966+
return Number(result.meta.changes ?? 0) === 1;
1967+
} catch {
1968+
return true;
1969+
}
1970+
}
1971+
19511972
/** Flip the DB-backed global kill-switch (operator emergency brake; no redeploy required). */
19521973
export async function setGlobalAgentFrozen(env: Env, frozen: boolean, updatedBy?: string | null): Promise<void> {
19531974
await env.DB.prepare(

src/queue/processors.ts

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import {
4141
persistAdvisory,
4242
markPullRequestsRegated,
4343
getLatestRegatedAt,
44+
claimRegateFanoutSlot,
4445
recordAgentCommandFeedback,
4546
recordAuditEvent,
4647
recordGateBlockOutcome,
@@ -115,7 +116,7 @@ import { isAuthorizedGitHubSessionLogin, parseGitHubLoginList } from "../auth/se
115116
import { commandAuthorizationAllowedRoles, commandAuthorizationNeedsMinerDetection } from "../settings/command-authorization";
116117
import { autonomyRequiresApproval, isAgentConfigured, resolveAutonomy } from "../settings/autonomy";
117118
import { isGlobalAgentPause, resolveAgentActionMode } from "../settings/agent-execution";
118-
import { isRegateSweepDraining, selectRegateCandidates } from "../settings/agent-sweep";
119+
import { SWEEP_FANOUT_DEDUP_MS, isRegateSweepDraining, selectRegateCandidates } from "../settings/agent-sweep";
119120
import { MAINTENANCE_RESERVED_HEADROOM, delayUntil, shouldWaitForGitHubRateLimit } from "../github/rate-limit";
120121
import { downgradeCloseToHold, downgradeMergeToHold, isProtectedAutomationAuthor, planAgentMaintenanceActions, type PlannedAgentAction } from "../settings/agent-actions";
121122
import { executeAgentMaintenanceActions, pendingClosureLabelApplied } from "../services/agent-action-executor";
@@ -432,8 +433,20 @@ async function fanOutRepoSignalSnapshotJobs(env: Env, requestedBy: "schedule" |
432433
// sweep job for every repo that opted the agent in (an acting autonomy level). Mirrors the signal-snapshot
433434
// fan-out so each repo's sweep runs as its own bounded, retryable queue message.
434435
async function fanOutAgentRegateSweepJobs(env: Env, requestedBy: "schedule" | "api" | "test"): Promise<void> {
435-
const repositories = await listRepositories(env);
436436
const now = nowIso();
437+
// Atomic fan-out dedup (#audit-fanout-dedup): collapse a BURST of fan-out jobs to a SINGLE effective fan-out per
438+
// window, so a deploy-restart cron catch-up (or fan-out jobs delayed behind a per-PR backlog then drained
439+
// together) cannot each enqueue a redundant per-repo sweep before the per-repo dispatch-stamp guard engages.
440+
if (!(await claimRegateFanoutSlot(env, now, SWEEP_FANOUT_DEDUP_MS))) {
441+
await recordAuditEvent(env, {
442+
eventType: "agent.sweep.fanout",
443+
outcome: "denied",
444+
detail: "re-gate fan-out deduped: another fan-out already claimed this window",
445+
metadata: { requestedBy, deduped: true },
446+
});
447+
return;
448+
}
449+
const repositories = await listRepositories(env);
437450
const configured: string[] = [];
438451
let skippedDraining = 0;
439452
for (const repo of repositories) {

src/settings/agent-sweep.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@ export const SWEEP_MAX_PRS = 25;
1616
// approved PRs unmerged for up to an hour.
1717
export const SWEEP_FRESHNESS_MS = 2 * 60 * 1000;
1818

19+
// Fan-out dedup window (#audit-fanout-dedup): a burst of fan-out jobs within this window collapses to ONE
20+
// effective fan-out. Kept BELOW the ~2-min cron cadence so a legitimate next-tick fan-out is never skipped, but
21+
// well above the few-seconds spread of a burst (a deploy-restart cron catch-up, or fan-out jobs that queued
22+
// behind a per-PR backlog and drained together).
23+
export const SWEEP_FANOUT_DEDUP_MS = 90 * 1000;
24+
1925
/**
2026
* Select the open PRs a single repo sweep should recompute: drop drafts and anything a webhook touched within
2127
* `freshnessWindowMs` of `now` (don't race an in-flight review), then take the `max` PRs the sweep has gone

test/unit/db-parsers.test.ts

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { describe, expect, it } from "vitest";
22
import {
3+
claimRegateFanoutSlot,
34
countRecentDeadLetters,
45
getLatestScorePreview,
56
getRepoAuthorPullRequestHistory,
@@ -113,6 +114,22 @@ describe("database row parser hardening", () => {
113114
expect(rows.find((p) => p.number === 6)?.lastRegatedAt ?? null).toBeNull(); // #6 not in the batch → untouched
114115
});
115116

117+
it("claimRegateFanoutSlot collapses a burst to one winner per window (#audit-fanout-dedup)", async () => {
118+
const env = createTestEnv();
119+
const W = 90 * 1000;
120+
expect(await claimRegateFanoutSlot(env, "2026-06-25T01:00:00.000Z", W)).toBe(true); // first claim wins (marker NULL)
121+
expect(await claimRegateFanoutSlot(env, "2026-06-25T01:00:05.000Z", W)).toBe(false); // +5s, inside window → loses
122+
expect(await claimRegateFanoutSlot(env, "2026-06-25T01:00:50.000Z", W)).toBe(false); // +50s, still inside → loses
123+
expect(await claimRegateFanoutSlot(env, "2026-06-25T01:01:31.000Z", W)).toBe(true); // +91s, outside window → wins again
124+
expect(await claimRegateFanoutSlot(env, "2026-06-25T01:01:40.000Z", W)).toBe(false); // back inside the new window → loses
125+
});
126+
127+
it("claimRegateFanoutSlot fails open (returns true) on a DB error so the fleet never stalls", async () => {
128+
const env = createTestEnv();
129+
const broken = { ...env, DB: null } as unknown as typeof env;
130+
expect(await claimRegateFanoutSlot(broken, "2026-06-25T01:00:00.000Z", 90 * 1000)).toBe(true);
131+
});
132+
116133
it("REGRESSION: a later GitHub sync does NOT clobber last_regated_at (omitted from the upsert SET clause)", async () => {
117134
const env = createTestEnv();
118135
await upsertPullRequestFromGitHub(env, "owner/repo", { number: 6, title: "First", state: "open", user: { login: "bob" }, labels: [] });

test/unit/queue.test.ts

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -891,6 +891,25 @@ describe("queue processors", () => {
891891
expect(JSON.parse(fanout?.metadata_json ?? "{}")).toMatchObject({ repoCount: 1, skippedDraining: 1 });
892892
});
893893

894+
it("INVARIANT (#audit-fanout-dedup): a BURST of fan-outs collapses to ONE — the second claims nothing and audits denied", async () => {
895+
const sent: import("../../src/types").JobMessage[] = [];
896+
const env = createTestEnv({ JOBS: { async send(m: import("../../src/types").JobMessage) { sent.push(m); } } as unknown as Queue });
897+
await upsertInstallation(env, { action: "created", installation: { id: 9400, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } });
898+
await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9400);
899+
await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo", autonomy: { merge: "auto" } });
900+
await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 7, title: "PR7", state: "open", user: { login: "c" }, head: { sha: "a7" }, labels: [], body: "" });
901+
vi.setSystemTime(new Date("2026-05-28T02:00:00.000Z"));
902+
903+
await processJob(env, { type: "agent-regate-sweep", requestedBy: "schedule" }); // first fan-out claims the window
904+
expect(sent.some((m) => m.type === "agent-regate-sweep" && m.repoFullName === "owner/agent-repo")).toBe(true);
905+
906+
sent.length = 0;
907+
await processJob(env, { type: "agent-regate-sweep", requestedBy: "schedule" }); // burst sibling in the same window → deduped
908+
expect(sent.filter((m) => m.type === "agent-regate-sweep")).toEqual([]); // enqueues no redundant sweep
909+
const denied = await env.DB.prepare("select count(*) as n from audit_events where event_type='agent.sweep.fanout' and outcome='denied'").first<{ n: number }>();
910+
expect(denied?.n).toBe(1);
911+
});
912+
894913
it("the sweep stamps the marker INLINE when the repo has no installation (audit-only, still converges) (#audit-sweep-fanout)", async () => {
895914
const sent: import("../../src/types").JobMessage[] = [];
896915
const env = createTestEnv({ JOBS: { async send(m: import("../../src/types").JobMessage) { sent.push(m); } } as unknown as Queue });

0 commit comments

Comments
 (0)