|
| 1 | +import { recordAuditEvent } from "../db/repositories"; |
| 2 | +import { errorMessage } from "../utils/json"; |
| 3 | + |
| 4 | +/** |
| 5 | + * #8997 — rescue a PR left wearing a decisive panel with no matching disposition. |
| 6 | + * |
| 7 | + * A deploy restart (SIGTERM, no drain) can kill a pass at the worst possible cut point: after the public-surface |
| 8 | + * publish commits (panel comment, gate check-run, CI aggregate all reflect the current head) but before |
| 9 | + * `maybeRunAgentMaintenance` ever claims its per-PR actuation lock for that head. The confirmed live shape |
| 10 | + * (#8965, 2026-07-26): a red-CI panel published at 14:55:24Z from the dying container, and the matching close |
| 11 | + * only executed at 15:07:53Z — ~12 minutes later, purely because an UNRELATED later sweep tick happened to |
| 12 | + * re-run the whole pass from scratch. The regular outage-repair priority check |
| 13 | + * (`surfaceRepairPriorityPullNumbers`, processors.ts) cannot see this: it flags a STALE surface |
| 14 | + * (`lastPublishedSurfaceSha !== headSha`), and this incident's surface is not stale — it published successfully |
| 15 | + * right before the kill. From that check's point of view the PR already looks healthy. |
| 16 | + * |
| 17 | + * This scan is the missing half: it looks for the disposition side directly. `maybeRunAgentMaintenance` records |
| 18 | + * a durable marker (`DISPOSITION_CONSIDERED_EVENT_TYPE`, processors.ts) the moment it claims the actuation lock |
| 19 | + * for a head — i.e. the moment a real disposition attempt begins. A PR whose surface is current for its head but |
| 20 | + * carries no such marker never got that far, for whatever reason (a restart being the confirmed one; a thrown |
| 21 | + * error before the claim is another). Re-enqueuing a regate is cheap and safe either way: `agent-regate-pr` |
| 22 | + * re-derives everything from live state, and a PR that in fact already has a real disposition in flight is |
| 23 | + * simply denied by the SAME actuation lock this scan is checking for, exactly as any other contending pass would |
| 24 | + * be (#9013's per-PR mutex). |
| 25 | + * |
| 26 | + * Deliberately independent of surfaceRepairPriorityPullNumbers rather than folded into it: that function's |
| 27 | + * return value directly drives the regular sweep's staleness-ordered fan-out and is asserted against by a large |
| 28 | + * existing test surface (dispatch shape, ordering, backlog-restriction semantics). Riding the SAME sweep tick as |
| 29 | + * its own bounded scan — mirroring reconcileMissingPrOutcomes/sweepStrandedPendingClosures (#9026/#9031) — closes |
| 30 | + * the identical gap without touching that already-tested surface at all. |
| 31 | + */ |
| 32 | + |
| 33 | +export const DISPOSITION_CONSIDERED_EVENT_TYPE = "agent.maintenance.disposition_considered"; |
| 34 | + |
| 35 | +/** How far back to look. Bounded so the scan stays cheap on every tick; a surface-without-disposition PR older |
| 36 | + * than this has almost certainly been superseded by a later commit or resolved by a human already. */ |
| 37 | +export const SURFACE_DISPOSITION_RECONCILE_LOOKBACK_MS = 24 * 60 * 60 * 1000; |
| 38 | + |
| 39 | +/** Cap per run, mirroring the other bounded repair scans on this same sweep tick. */ |
| 40 | +export const SURFACE_DISPOSITION_RECONCILE_LIMIT = 200; |
| 41 | + |
| 42 | +export type SurfaceDispositionReconcileResult = { scanned: number; requeued: number }; |
| 43 | + |
| 44 | +/** |
| 45 | + * Find open PRs with a current published surface but no disposition marker for that exact head, and re-enqueue |
| 46 | + * an `agent-regate-pr` job for each. Best-effort throughout: a repair pass that can itself break the tick it |
| 47 | + * rides on is worse than the gap it exists to close. |
| 48 | + */ |
| 49 | +export async function reconcileSurfaceWithoutDisposition(env: Env, nowMs: number = Date.now()): Promise<SurfaceDispositionReconcileResult> { |
| 50 | + const since = new Date(nowMs - SURFACE_DISPOSITION_RECONCILE_LOOKBACK_MS).toISOString(); |
| 51 | + let rows: Array<{ repoFullName: string; number: number; installationId: number; headSha: string }> = []; |
| 52 | + try { |
| 53 | + const result = await env.DB.prepare( |
| 54 | + `SELECT pr.repo_full_name AS repoFullName, pr.number AS number, repo.installation_id AS installationId, pr.head_sha AS headSha |
| 55 | + FROM pull_requests AS pr |
| 56 | + JOIN repositories AS repo ON repo.full_name = pr.repo_full_name |
| 57 | + WHERE pr.state = 'open' |
| 58 | + AND pr.head_sha IS NOT NULL |
| 59 | + AND pr.last_published_surface_sha = pr.head_sha |
| 60 | + AND pr.updated_at >= ?1 |
| 61 | + AND repo.installation_id IS NOT NULL |
| 62 | + AND NOT EXISTS ( |
| 63 | + SELECT 1 FROM audit_events AS disposition |
| 64 | + WHERE disposition.target_key = pr.repo_full_name || '#' || pr.number || '#' || pr.head_sha |
| 65 | + AND disposition.event_type = ?2 |
| 66 | + ) |
| 67 | + ORDER BY pr.updated_at |
| 68 | + LIMIT ?3`, |
| 69 | + ) |
| 70 | + .bind(since, DISPOSITION_CONSIDERED_EVENT_TYPE, SURFACE_DISPOSITION_RECONCILE_LIMIT) |
| 71 | + .all<{ repoFullName: string; number: number; installationId: number; headSha: string }>(); |
| 72 | + rows = result.results ?? []; |
| 73 | + } catch (error) { |
| 74 | + console.warn(JSON.stringify({ level: "warn", event: "surface_disposition_reconcile_scan_failed", message: errorMessage(error).slice(0, 160) })); |
| 75 | + return { scanned: 0, requeued: 0 }; |
| 76 | + } |
| 77 | + |
| 78 | + let requeued = 0; |
| 79 | + for (const row of rows) { |
| 80 | + const sent = await env.JOBS.send({ |
| 81 | + type: "agent-regate-pr", |
| 82 | + deliveryId: `surface-without-disposition:${row.repoFullName}#${row.number}#${row.headSha}`, |
| 83 | + repoFullName: row.repoFullName, |
| 84 | + prNumber: row.number, |
| 85 | + installationId: row.installationId, |
| 86 | + }) |
| 87 | + .then(() => true) |
| 88 | + .catch(() => false); |
| 89 | + if (!sent) continue; |
| 90 | + requeued += 1; |
| 91 | + await recordAuditEvent(env, { |
| 92 | + eventType: "agent.sweep.surface_without_disposition", |
| 93 | + actor: "loopover", |
| 94 | + targetKey: `${row.repoFullName}#${row.number}`, |
| 95 | + outcome: "queued", |
| 96 | + detail: "published surface has no matching disposition for this head; re-gating to close the gap", |
| 97 | + metadata: { repoFullName: row.repoFullName, pullNumber: row.number, headSha: row.headSha }, |
| 98 | + }).catch(() => undefined); |
| 99 | + } |
| 100 | + return { scanned: rows.length, requeued }; |
| 101 | +} |
0 commit comments