diff --git a/src/chrome/src/background.js b/src/chrome/src/background.js index c2b5922a1..514cf0e14 100644 --- a/src/chrome/src/background.js +++ b/src/chrome/src/background.js @@ -112,6 +112,11 @@ import { const providerManager = new ProviderManager(); const apocalypseController = createApocalypseController(chrome); const VISION_OFFSCREEN_URL = chrome.runtime.getURL('src/offscreen/offscreen.html'); +// The stale-run repair scan waits a beat after wake so a run resuming from +// eviction registers its in-memory state first; the Traces page can also +// request an immediate scan via WB_TRACE_REPAIR_STALE_RUNS. +const TRACE_REPAIR_STARTUP_DELAY_MS = 15_000; +setTimeout(() => { void workflowTrace.repairStaleRuns().catch(() => {}); }, TRACE_REPAIR_STARTUP_DELAY_MS); function normalizeVisionDownloadState(state) { return { @@ -1486,6 +1491,16 @@ chrome.contextMenus?.onClicked?.addListener?.((info, tab) => { handleContextMenuAsk(info, tab).catch(() => {}); }); +// Only this instance knows which runs are live in memory, so it owns the +// stale-run repair whenever it is reachable. +chrome.runtime.onMessage.addListener((msg, _sender, sendResponse) => { + if (msg?.type !== 'WB_TRACE_REPAIR_STALE_RUNS') return; + workflowTrace.repairStaleRuns() + .then(repaired => sendResponse({ ok: true, repaired })) + .catch(error => sendResponse({ ok: false, error: error?.message || String(error) })); + return true; +}); + chrome.runtime.onMessage.addListener((msg, _sender, sendResponse) => { if (msg?.type !== 'WB_SELECTION_SHORTCUT_LOCALIZATION') return; sendResponse({ ok: true, ...getSelectionShortcutLocalization(msg.locale) }); diff --git a/src/chrome/src/trace/error-codes.js b/src/chrome/src/trace/error-codes.js index 82f355c08..ff691f760 100644 --- a/src/chrome/src/trace/error-codes.js +++ b/src/chrome/src/trace/error-codes.js @@ -23,6 +23,7 @@ export const ERROR_CODES = Object.freeze([ 'TRANSPORT', 'TOOL_TIMEOUT', 'COST_LIMIT', + 'SERVICE_WORKER_EVICTED', 'UNKNOWN', ]); @@ -35,4 +36,4 @@ export function isErrorCode(code) { export function normalizeErrorCode(code) { if (isErrorCode(code)) return code; return 'UNKNOWN'; -} \ No newline at end of file +} diff --git a/src/chrome/src/trace/recorder.js b/src/chrome/src/trace/recorder.js index a8f145cea..6450c6f0d 100644 --- a/src/chrome/src/trace/recorder.js +++ b/src/chrome/src/trace/recorder.js @@ -4,6 +4,12 @@ import { formatErrorMessage } from '../error-format.js'; import { TRACE_FORMAT_VERSION, makeEvent } from './event-model.js'; import { normalizeErrorCode } from './error-codes.js'; import { normalizeRunHeader, effectiveDelegationDepth } from './run-header.js'; +import { + buildTraceRepairPlan, + isStaleRunningTrace, + normalizedThreshold, + TRACE_REPAIR_STALE_AFTER_MS, +} from './repair.js'; /** * Trace recorder — writes per-run traces (LLM requests/responses, tool calls, @@ -137,6 +143,50 @@ async function _peekSeq(db, runId) { return result; } +function _repairRunInTransaction(db, runId, { now, staleAfterMs }) { + return new Promise((resolve, reject) => { + const transaction = tx(db, ['runs', 'events'], 'readwrite'); + const runsStore = transaction.objectStore('runs'); + const eventsStore = transaction.objectStore('events'); + let run = null; + let events = null; + let pending = 2; + let repairedRunId = null; + let requestError = null; + + const fail = (error) => { + requestError = error || new Error('trace repair request failed'); + try { transaction.abort(); } catch {} + }; + const apply = () => { + pending -= 1; + if (pending > 0 || requestError) return; + const plan = buildTraceRepairPlan(run, events, { now, staleAfterMs }); + if (!plan) return; + // Last-instant liveness check: a run resumed between the candidate scan + // and this transaction has registered itself in memory again. + if (_runState.has(runId)) return; + for (const event of plan.events) eventsStore.put(event); + runsStore.put(plan.run); + repairedRunId = runId; + }; + + transaction.oncomplete = () => resolve(repairedRunId); + transaction.onerror = () => reject(requestError || transaction.error || new Error('trace repair failed')); + transaction.onabort = () => { + if (requestError) reject(requestError); + else if (!repairedRunId) resolve(null); + }; + + const runRequest = runsStore.get(runId); + runRequest.onsuccess = () => { run = runRequest.result || null; apply(); }; + runRequest.onerror = () => fail(runRequest.error); + const eventsRequest = eventsStore.index('runId').getAll(IDBKeyRange.only(runId)); + eventsRequest.onsuccess = () => { events = eventsRequest.result || []; apply(); }; + eventsRequest.onerror = () => fail(eventsRequest.error); + }); +} + function _newSeq(runId) { const st = _runState.get(runId); if (!st) return 0; @@ -498,6 +548,62 @@ export async function endRun(runId, { status = 'done', finalContent = null } = { } } +/** + * Repair trace records left running after a service-worker eviction. Each run + * is re-read and updated in one transaction so a concurrent normal completion + * wins, and a second repair pass cannot append duplicate terminal events. + */ +async function _listStaleRunCandidates(db, cutoff) { + // Only runs old enough to be stale can qualify (last activity is never + // older than startedAt), so scan just that slice of the startedAt index + // instead of every run on record. + const out = []; + await new Promise((resolve, reject) => { + const idx = tx(db, ['runs'], 'readonly').objectStore('runs').index('startedAt'); + const req = idx.openCursor(IDBKeyRange.upperBound(cutoff)); + req.onsuccess = () => { + const c = req.result; + if (!c) return resolve(); + const row = c.value; + if (row?.status === 'running' && !row.repairedBy) out.push(row); + c.continue(); + }; + req.onerror = () => reject(req.error || new Error('stale-run scan failed')); + }); + return out; +} + +export async function repairStaleRuns({ + now = Date.now(), + staleAfterMs = TRACE_REPAIR_STALE_AFTER_MS, +} = {}) { + if (typeof indexedDB === 'undefined') return []; + if (!Number.isFinite(Number(now))) return []; + try { + const db = await openDB(); + const cutoff = Number(now) - normalizedThreshold(staleAfterMs); + const candidates = await _listStaleRunCandidates(db, cutoff); + const repaired = []; + for (const candidate of candidates) { + if (!isStaleRunningTrace(candidate, { now, staleAfterMs })) continue; + // A live run in this service-worker instance is still owned by the + // agent. The durable marker handles races from another extension page. + if (_runState.has(candidate.runId)) continue; + await _flushRunWrites(candidate.runId); + try { + const repairedRunId = await _repairRunInTransaction(db, candidate.runId, { now, staleAfterMs }); + if (repairedRunId) repaired.push(repairedRunId); + } catch (e) { + console.warn('[trace] stale-run repair failed:', e); + } + } + return repaired; + } catch (e) { + console.warn('[trace] stale-run scan failed:', e); + return []; + } +} + // ----- Reader API (used by traces.html) -------------------------------------- export async function listRuns({ limit = 500, conversationId = null } = {}) { diff --git a/src/chrome/src/trace/repair.js b/src/chrome/src/trace/repair.js new file mode 100644 index 000000000..cde576d1b --- /dev/null +++ b/src/chrome/src/trace/repair.js @@ -0,0 +1,124 @@ +import { makeEvent } from './event-model.js'; +import { normalizeErrorCode } from './error-codes.js'; + +// A run can legitimately take a while, so repair only treats records older +// than this conservative window as abandoned. Callers/tests may provide a +// smaller explicit threshold when they need a deterministic boundary. +export const TRACE_REPAIR_STALE_AFTER_MS = 10 * 60 * 1000; +export const TRACE_REPAIR_MARKER = 'service-worker-eviction'; +export const TRACE_REPAIR_ERROR_CODE = 'SERVICE_WORKER_EVICTED'; +export const TRACE_REPAIR_REASON = 'service_worker_eviction'; +export const TRACE_REPAIR_MESSAGE = 'Trace run interrupted by service-worker eviction.'; + +function repairEvent(runId, seq, kind, data, now) { + const event = makeEvent(runId, seq, kind, data); + return event ? { ...event, ts: now } : null; +} + +function eventStep(event) { + return Number.isInteger(event?.data?.step) ? event.data.step : null; +} + +function openSteps(events) { + const open = new Map(); + for (const event of events) { + if (event?.kind === 'step_start') open.set(eventStep(event), eventStep(event)); + if (event?.kind === 'step_end') open.delete(eventStep(event)); + } + return [...open.values()].sort((a, b) => (a ?? 0) - (b ?? 0)); +} + +function openTurnStep(events) { + let step = null; + for (const event of events) { + if (event?.kind === 'turn_start') step = eventStep(event); + if (event?.kind === 'turn_end') step = null; + } + return step; +} + +export function normalizedThreshold(value) { + return Number.isFinite(Number(value)) ? Math.max(0, Number(value)) : TRACE_REPAIR_STALE_AFTER_MS; +} + +export function isStaleRunningTrace(run, { + now = Date.now(), + staleAfterMs = TRACE_REPAIR_STALE_AFTER_MS, + events = [], +} = {}) { + if (!run || run.status !== 'running' || run.repairedBy) return false; + const currentTime = Number(now); + let lastActivityAt = Number(run.startedAt); + if (!Number.isFinite(lastActivityAt) || !Number.isFinite(currentTime)) return false; + for (const event of Array.isArray(events) ? events : []) { + const eventTime = Number(event?.ts); + if (Number.isFinite(eventTime)) lastActivityAt = Math.max(lastActivityAt, eventTime); + } + return currentTime - lastActivityAt >= normalizedThreshold(staleAfterMs); +} + +/** + * Build the durable repair mutation for one abandoned trace. + * + * This is deliberately pure: the recorder applies the returned event/run + * values in one IndexedDB transaction, while tests can prove the interruption + * semantics without a browser or a real IndexedDB implementation. + */ +export function buildTraceRepairPlan(run, events, { + now = Date.now(), + staleAfterMs = TRACE_REPAIR_STALE_AFTER_MS, +} = {}) { + const orderedEvents = (Array.isArray(events) ? events : []) + .filter(Boolean) + .slice() + .sort((a, b) => (Number(a.seq) || 0) - (Number(b.seq) || 0)); + if (!isStaleRunningTrace(run, { now, staleAfterMs, events: orderedEvents })) return null; + const lastSeq = orderedEvents.reduce((max, event) => Math.max(max, Number(event.seq) || 0), 0); + const code = normalizeErrorCode(TRACE_REPAIR_ERROR_CODE); + const repairedEvents = []; + let nextSeq = lastSeq + 1; + + for (const step of openSteps(orderedEvents)) { + repairedEvents.push(repairEvent(run.runId, nextSeq++, 'step_end', { + step, + ok: false, + reason: TRACE_REPAIR_REASON, + code, + repaired: true, + }, now)); + } + + repairedEvents.push(repairEvent(run.runId, nextSeq++, 'error', { + step: null, + phase: 'repair', + message: TRACE_REPAIR_MESSAGE, + code, + }, now)); + + const turnStep = openTurnStep(orderedEvents); + if (turnStep !== null) { + repairedEvents.push(repairEvent(run.runId, nextSeq, 'turn_end', { + step: turnStep, + status: 'error', + reason: TRACE_REPAIR_REASON, + code, + repaired: true, + }, now)); + } + + const maxStep = orderedEvents.reduce((max, event) => Math.max(max, eventStep(event) ?? 0), 0); + const startedAt = Number(run.startedAt); + return { + events: repairedEvents.filter(Boolean), + run: { + ...run, + endedAt: now, + durationMs: Math.max(0, now - startedAt), + status: 'error', + stepCount: Math.max(Number(run.stepCount) || 0, maxStep), + repairedBy: TRACE_REPAIR_MARKER, + repairedAt: now, + repairReason: TRACE_REPAIR_REASON, + }, + }; +} diff --git a/src/chrome/src/ui/traces.js b/src/chrome/src/ui/traces.js index 8da742441..6f0c9ad3b 100644 --- a/src/chrome/src/ui/traces.js +++ b/src/chrome/src/ui/traces.js @@ -5,12 +5,14 @@ import { listRuns, getRun, getRunEvents, getScreenshot, - deleteRun, clearAllRuns, + deleteRun, clearAllRuns, repairStaleRuns, } from '../trace/recorder.js'; import { isKnownKind, isIgnorableKind } from '../trace/event-model.js'; import { t } from './i18n.js'; import { escapeHtml, escapeAttr } from './utils.js'; +const runtimeApi = globalThis.browser || globalThis.chrome; + const listEl = document.getElementById('run-list'); const mainPane = document.getElementById('main-pane'); const emptyState = document.getElementById('empty-state'); @@ -645,9 +647,31 @@ document.addEventListener('visibilitychange', () => { else if (hasRunningJob()) scheduleAutoRefresh(); }); +// Prefer letting the background own the stale-run scan: only its recorder +// instance knows which runs are live in memory. If it cannot be reached, +// nothing can be running anywhere, so a local pass is safe. +async function repairStaleRunsForPage({ timeoutMs = 5_000 } = {}) { + let timer; + try { + if (runtimeApi?.runtime?.sendMessage) { + const response = await Promise.race([ + runtimeApi.runtime.sendMessage({ type: 'WB_TRACE_REPAIR_STALE_RUNS' }), + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error('trace repair roundtrip timed out')), timeoutMs); + }), + ]); + if (response?.ok) return Array.isArray(response.repaired) ? response.repaired : []; + } + } catch {} finally { + clearTimeout(timer); + } + return repairStaleRuns().catch(() => []); +} + // Initial load: always do one refresh so the list populates, then only keep // polling if the freshly-loaded data shows a live run. (async () => { + await repairStaleRunsForPage(); await refresh(); if (initialRunId && await ensureRunLoaded(initialRunId)) { selectedRunId = initialRunId; diff --git a/src/firefox/src/background.js b/src/firefox/src/background.js index 6500e027d..78bb32b8b 100644 --- a/src/firefox/src/background.js +++ b/src/firefox/src/background.js @@ -100,6 +100,11 @@ import { const providerManager = new ProviderManager(); const apocalypseController = createApocalypseController(browser); let emergencyDownloads = null; +// The stale-run repair scan waits a beat after wake so a run resuming from +// eviction registers its in-memory state first; the Traces page can also +// request an immediate scan via WB_TRACE_REPAIR_STALE_RUNS. +const TRACE_REPAIR_STARTUP_DELAY_MS = 15_000; +setTimeout(() => { void workflowTrace.repairStaleRuns().catch(() => {}); }, TRACE_REPAIR_STARTUP_DELAY_MS); function emergencyDownloadController() { if (!emergencyDownloads) { @@ -1261,6 +1266,16 @@ getContextMenuApi()?.onClicked?.addListener?.((info, tab) => { handleContextMenuAsk(info, tab).catch(() => {}); }); +// Only this instance knows which runs are live in memory, so it owns the +// stale-run repair whenever it is reachable. +browser.runtime.onMessage.addListener((msg, _sender, sendResponse) => { + if (msg?.type !== 'WB_TRACE_REPAIR_STALE_RUNS') return; + workflowTrace.repairStaleRuns() + .then(repaired => sendResponse({ ok: true, repaired })) + .catch(error => sendResponse({ ok: false, error: error?.message || String(error) })); + return true; +}); + browser.runtime.onMessage.addListener((msg, _sender, sendResponse) => { if (msg?.type !== 'WB_SELECTION_SHORTCUT_LOCALIZATION') return; sendResponse({ ok: true, ...getSelectionShortcutLocalization(msg.locale) }); diff --git a/src/firefox/src/trace/error-codes.js b/src/firefox/src/trace/error-codes.js index 82f355c08..ff691f760 100644 --- a/src/firefox/src/trace/error-codes.js +++ b/src/firefox/src/trace/error-codes.js @@ -23,6 +23,7 @@ export const ERROR_CODES = Object.freeze([ 'TRANSPORT', 'TOOL_TIMEOUT', 'COST_LIMIT', + 'SERVICE_WORKER_EVICTED', 'UNKNOWN', ]); @@ -35,4 +36,4 @@ export function isErrorCode(code) { export function normalizeErrorCode(code) { if (isErrorCode(code)) return code; return 'UNKNOWN'; -} \ No newline at end of file +} diff --git a/src/firefox/src/trace/recorder.js b/src/firefox/src/trace/recorder.js index 216e22ab8..98f244ef5 100644 --- a/src/firefox/src/trace/recorder.js +++ b/src/firefox/src/trace/recorder.js @@ -4,6 +4,12 @@ import { formatErrorMessage } from '../error-format.js'; import { TRACE_FORMAT_VERSION, makeEvent } from './event-model.js'; import { normalizeErrorCode } from './error-codes.js'; import { normalizeRunHeader, effectiveDelegationDepth } from './run-header.js'; +import { + buildTraceRepairPlan, + isStaleRunningTrace, + normalizedThreshold, + TRACE_REPAIR_STALE_AFTER_MS, +} from './repair.js'; /** * Trace recorder — writes per-run traces (LLM requests/responses, tool calls, @@ -122,6 +128,50 @@ async function _peekSeq(db, runId) { return result; } +function _repairRunInTransaction(db, runId, { now, staleAfterMs }) { + return new Promise((resolve, reject) => { + const transaction = tx(db, ['runs', 'events'], 'readwrite'); + const runsStore = transaction.objectStore('runs'); + const eventsStore = transaction.objectStore('events'); + let run = null; + let events = null; + let pending = 2; + let repairedRunId = null; + let requestError = null; + + const fail = (error) => { + requestError = error || new Error('trace repair request failed'); + try { transaction.abort(); } catch {} + }; + const apply = () => { + pending -= 1; + if (pending > 0 || requestError) return; + const plan = buildTraceRepairPlan(run, events, { now, staleAfterMs }); + if (!plan) return; + // Last-instant liveness check: a run resumed between the candidate scan + // and this transaction has registered itself in memory again. + if (_runState.has(runId)) return; + for (const event of plan.events) eventsStore.put(event); + runsStore.put(plan.run); + repairedRunId = runId; + }; + + transaction.oncomplete = () => resolve(repairedRunId); + transaction.onerror = () => reject(requestError || transaction.error || new Error('trace repair failed')); + transaction.onabort = () => { + if (requestError) reject(requestError); + else if (!repairedRunId) resolve(null); + }; + + const runRequest = runsStore.get(runId); + runRequest.onsuccess = () => { run = runRequest.result || null; apply(); }; + runRequest.onerror = () => fail(runRequest.error); + const eventsRequest = eventsStore.index('runId').getAll(IDBKeyRange.only(runId)); + eventsRequest.onsuccess = () => { events = eventsRequest.result || []; apply(); }; + eventsRequest.onerror = () => fail(eventsRequest.error); + }); +} + function _newSeq(runId) { const st = _runState.get(runId); if (!st) return 0; @@ -480,6 +530,62 @@ export async function endRun(runId, { status = 'done', finalContent = null } = { } } +/** + * Repair trace records left running after a service-worker eviction. Each run + * is re-read and updated in one transaction so a concurrent normal completion + * wins, and a second repair pass cannot append duplicate terminal events. + */ +async function _listStaleRunCandidates(db, cutoff) { + // Only runs old enough to be stale can qualify (last activity is never + // older than startedAt), so scan just that slice of the startedAt index + // instead of every run on record. + const out = []; + await new Promise((resolve, reject) => { + const idx = tx(db, ['runs'], 'readonly').objectStore('runs').index('startedAt'); + const req = idx.openCursor(IDBKeyRange.upperBound(cutoff)); + req.onsuccess = () => { + const c = req.result; + if (!c) return resolve(); + const row = c.value; + if (row?.status === 'running' && !row.repairedBy) out.push(row); + c.continue(); + }; + req.onerror = () => reject(req.error || new Error('stale-run scan failed')); + }); + return out; +} + +export async function repairStaleRuns({ + now = Date.now(), + staleAfterMs = TRACE_REPAIR_STALE_AFTER_MS, +} = {}) { + if (typeof indexedDB === 'undefined') return []; + if (!Number.isFinite(Number(now))) return []; + try { + const db = await openDB(); + const cutoff = Number(now) - normalizedThreshold(staleAfterMs); + const candidates = await _listStaleRunCandidates(db, cutoff); + const repaired = []; + for (const candidate of candidates) { + if (!isStaleRunningTrace(candidate, { now, staleAfterMs })) continue; + // A live run in this background instance is still owned by the agent. + // The durable marker handles races from another extension page. + if (_runState.has(candidate.runId)) continue; + await _flushRunWrites(candidate.runId); + try { + const repairedRunId = await _repairRunInTransaction(db, candidate.runId, { now, staleAfterMs }); + if (repairedRunId) repaired.push(repairedRunId); + } catch (e) { + console.warn('[trace] stale-run repair failed:', e); + } + } + return repaired; + } catch (e) { + console.warn('[trace] stale-run scan failed:', e); + return []; + } +} + // ----- Reader API (used by traces.html) -------------------------------------- export async function listRuns({ limit = 500, conversationId = null } = {}) { diff --git a/src/firefox/src/trace/repair.js b/src/firefox/src/trace/repair.js new file mode 100644 index 000000000..cde576d1b --- /dev/null +++ b/src/firefox/src/trace/repair.js @@ -0,0 +1,124 @@ +import { makeEvent } from './event-model.js'; +import { normalizeErrorCode } from './error-codes.js'; + +// A run can legitimately take a while, so repair only treats records older +// than this conservative window as abandoned. Callers/tests may provide a +// smaller explicit threshold when they need a deterministic boundary. +export const TRACE_REPAIR_STALE_AFTER_MS = 10 * 60 * 1000; +export const TRACE_REPAIR_MARKER = 'service-worker-eviction'; +export const TRACE_REPAIR_ERROR_CODE = 'SERVICE_WORKER_EVICTED'; +export const TRACE_REPAIR_REASON = 'service_worker_eviction'; +export const TRACE_REPAIR_MESSAGE = 'Trace run interrupted by service-worker eviction.'; + +function repairEvent(runId, seq, kind, data, now) { + const event = makeEvent(runId, seq, kind, data); + return event ? { ...event, ts: now } : null; +} + +function eventStep(event) { + return Number.isInteger(event?.data?.step) ? event.data.step : null; +} + +function openSteps(events) { + const open = new Map(); + for (const event of events) { + if (event?.kind === 'step_start') open.set(eventStep(event), eventStep(event)); + if (event?.kind === 'step_end') open.delete(eventStep(event)); + } + return [...open.values()].sort((a, b) => (a ?? 0) - (b ?? 0)); +} + +function openTurnStep(events) { + let step = null; + for (const event of events) { + if (event?.kind === 'turn_start') step = eventStep(event); + if (event?.kind === 'turn_end') step = null; + } + return step; +} + +export function normalizedThreshold(value) { + return Number.isFinite(Number(value)) ? Math.max(0, Number(value)) : TRACE_REPAIR_STALE_AFTER_MS; +} + +export function isStaleRunningTrace(run, { + now = Date.now(), + staleAfterMs = TRACE_REPAIR_STALE_AFTER_MS, + events = [], +} = {}) { + if (!run || run.status !== 'running' || run.repairedBy) return false; + const currentTime = Number(now); + let lastActivityAt = Number(run.startedAt); + if (!Number.isFinite(lastActivityAt) || !Number.isFinite(currentTime)) return false; + for (const event of Array.isArray(events) ? events : []) { + const eventTime = Number(event?.ts); + if (Number.isFinite(eventTime)) lastActivityAt = Math.max(lastActivityAt, eventTime); + } + return currentTime - lastActivityAt >= normalizedThreshold(staleAfterMs); +} + +/** + * Build the durable repair mutation for one abandoned trace. + * + * This is deliberately pure: the recorder applies the returned event/run + * values in one IndexedDB transaction, while tests can prove the interruption + * semantics without a browser or a real IndexedDB implementation. + */ +export function buildTraceRepairPlan(run, events, { + now = Date.now(), + staleAfterMs = TRACE_REPAIR_STALE_AFTER_MS, +} = {}) { + const orderedEvents = (Array.isArray(events) ? events : []) + .filter(Boolean) + .slice() + .sort((a, b) => (Number(a.seq) || 0) - (Number(b.seq) || 0)); + if (!isStaleRunningTrace(run, { now, staleAfterMs, events: orderedEvents })) return null; + const lastSeq = orderedEvents.reduce((max, event) => Math.max(max, Number(event.seq) || 0), 0); + const code = normalizeErrorCode(TRACE_REPAIR_ERROR_CODE); + const repairedEvents = []; + let nextSeq = lastSeq + 1; + + for (const step of openSteps(orderedEvents)) { + repairedEvents.push(repairEvent(run.runId, nextSeq++, 'step_end', { + step, + ok: false, + reason: TRACE_REPAIR_REASON, + code, + repaired: true, + }, now)); + } + + repairedEvents.push(repairEvent(run.runId, nextSeq++, 'error', { + step: null, + phase: 'repair', + message: TRACE_REPAIR_MESSAGE, + code, + }, now)); + + const turnStep = openTurnStep(orderedEvents); + if (turnStep !== null) { + repairedEvents.push(repairEvent(run.runId, nextSeq, 'turn_end', { + step: turnStep, + status: 'error', + reason: TRACE_REPAIR_REASON, + code, + repaired: true, + }, now)); + } + + const maxStep = orderedEvents.reduce((max, event) => Math.max(max, eventStep(event) ?? 0), 0); + const startedAt = Number(run.startedAt); + return { + events: repairedEvents.filter(Boolean), + run: { + ...run, + endedAt: now, + durationMs: Math.max(0, now - startedAt), + status: 'error', + stepCount: Math.max(Number(run.stepCount) || 0, maxStep), + repairedBy: TRACE_REPAIR_MARKER, + repairedAt: now, + repairReason: TRACE_REPAIR_REASON, + }, + }; +} diff --git a/src/firefox/src/ui/traces.js b/src/firefox/src/ui/traces.js index 7ef31e781..22bde8324 100644 --- a/src/firefox/src/ui/traces.js +++ b/src/firefox/src/ui/traces.js @@ -5,12 +5,14 @@ import { listRuns, getRun, getRunEvents, getScreenshot, - deleteRun, clearAllRuns, + deleteRun, clearAllRuns, repairStaleRuns, } from '../trace/recorder.js'; import { isKnownKind, isIgnorableKind } from '../trace/event-model.js'; import { t } from './i18n.js'; import { escapeHtml, escapeAttr } from './utils.js'; +const runtimeApi = globalThis.browser || globalThis.chrome; + const listEl = document.getElementById('run-list'); const mainPane = document.getElementById('main-pane'); const emptyState = document.getElementById('empty-state'); @@ -645,9 +647,31 @@ document.addEventListener('visibilitychange', () => { else if (hasRunningJob()) scheduleAutoRefresh(); }); +// Prefer letting the background own the stale-run scan: only its recorder +// instance knows which runs are live in memory. If it cannot be reached, +// nothing can be running anywhere, so a local pass is safe. +async function repairStaleRunsForPage({ timeoutMs = 5_000 } = {}) { + let timer; + try { + if (runtimeApi?.runtime?.sendMessage) { + const response = await Promise.race([ + runtimeApi.runtime.sendMessage({ type: 'WB_TRACE_REPAIR_STALE_RUNS' }), + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error('trace repair roundtrip timed out')), timeoutMs); + }), + ]); + if (response?.ok) return Array.isArray(response.repaired) ? response.repaired : []; + } + } catch {} finally { + clearTimeout(timer); + } + return repairStaleRuns().catch(() => []); +} + // Initial load: always do one refresh so the list populates, then only keep // polling if the freshly-loaded data shows a live run. (async () => { + await repairStaleRunsForPage(); await refresh(); if (initialRunId && await ensureRunLoaded(initialRunId)) { selectedRunId = initialRunId; diff --git a/test/run.js b/test/run.js index 2b0b0019a..10b7656b7 100644 --- a/test/run.js +++ b/test/run.js @@ -8106,6 +8106,8 @@ console.log('\ntrace event model'); const EVENT_MODEL_CH = await import('file://' + path.join(ROOT, 'src/chrome/src/trace/event-model.js').replace(/\\/g, '/')); const EVENT_MODEL_FX = await import('file://' + path.join(ROOT, 'src/firefox/src/trace/event-model.js').replace(/\\/g, '/')); +const TRACE_REPAIR_CH = await import('file://' + path.join(ROOT, 'src/chrome/src/trace/repair.js').replace(/\\/g, '/')); +const TRACE_REPAIR_FX = await import('file://' + path.join(ROOT, 'src/firefox/src/trace/repair.js').replace(/\\/g, '/')); const CLOUD_RUNTIME_OUTBOX_CH = await import('file://' + path.join(ROOT, 'src/chrome/src/trace/cloud-runtime-outbox.js').replace(/\\/g, '/')); const CLOUD_RUNTIME_OUTBOX_FX = await import('file://' + path.join(ROOT, 'src/firefox/src/trace/cloud-runtime-outbox.js').replace(/\\/g, '/')); @@ -9144,6 +9146,7 @@ test('trace error codes: catalog is stable and normalizeErrorCode bounds the val assert.equal(normalizeErrorCode('NOT_A_CODE'), 'UNKNOWN', 'unknown code must normalize to UNKNOWN'); assert.equal(normalizeErrorCode(null), 'UNKNOWN', 'null code must normalize to UNKNOWN'); assert.equal(normalizeErrorCode(undefined), 'UNKNOWN', 'missing code must normalize to UNKNOWN'); + assert.equal(isErrorCode('SERVICE_WORKER_EVICTED'), true, 'service-worker interruption needs a stable repair code'); assert.deepEqual(ERROR_CODES_FX.ERROR_CODES, ERROR_CODES, 'Firefox code catalog drifted'); }); @@ -9155,6 +9158,113 @@ test('trace error codes: mirrors are identical and browser-neutral', () => { assert.doesNotMatch(chromeSource, /indexedDB|storage\./, 'error codes must not touch storage'); }); +test('trace repair: stale interrupted runs receive one ordered terminal repair', () => { + const run = { + runId: 'run_interrupted', + startedAt: 1_000, + status: 'running', + stepCount: 1, + finalContent: null, + }; + const events = [ + { runId: run.runId, seq: 1, kind: 'turn_start', data: { step: 0 } }, + { runId: run.runId, seq: 2, kind: 'step_start', data: { step: 1 } }, + { runId: run.runId, seq: 3, kind: 'llm_request', data: { step: 1 } }, + ]; + const plan = TRACE_REPAIR_CH.buildTraceRepairPlan(run, events, { + now: 100_000, + staleAfterMs: 60_000, + }); + + assert.ok(plan, 'an old running run must produce a repair plan'); + assert.deepEqual( + plan.events.map((event) => [event.seq, event.kind]), + [[4, 'step_end'], [5, 'error'], [6, 'turn_end']], + 'repair events must continue the existing sequence and close open lifecycle events', + ); + assert.deepEqual(plan.events[0].data, { + step: 1, + ok: false, + reason: 'service_worker_eviction', + code: 'SERVICE_WORKER_EVICTED', + repaired: true, + }); + assert.deepEqual(plan.events[1].data, { + step: null, + phase: 'repair', + message: 'Trace run interrupted by service-worker eviction.', + code: 'SERVICE_WORKER_EVICTED', + }); + assert.deepEqual(plan.events[2].data, { + step: 0, + status: 'error', + reason: 'service_worker_eviction', + code: 'SERVICE_WORKER_EVICTED', + repaired: true, + }); + assert.equal(plan.run.status, 'error', 'repaired run must no longer be running'); + assert.equal(plan.run.endedAt, 100_000); + assert.equal(plan.run.durationMs, 99_000); + assert.equal(plan.run.repairedBy, 'service-worker-eviction'); + assert.equal(plan.run.repairedAt, 100_000); +}); + +test('trace repair: ignores recent and already repaired runs, with mirrored helpers', () => { + const recent = { runId: 'recent', startedAt: 95_000, status: 'running' }; + const activeLongRun = { runId: 'active_long', startedAt: 1_000, status: 'running' }; + const repaired = { + runId: 'repaired', + startedAt: 1_000, + status: 'error', + repairedBy: 'service-worker-eviction', + }; + for (const [label, repair] of [['chrome', TRACE_REPAIR_CH], ['firefox', TRACE_REPAIR_FX]]) { + assert.equal( + repair.buildTraceRepairPlan(recent, [], { now: 100_000, staleAfterMs: 60_000 }), + null, + `${label}: recent runs must remain eligible for normal completion`, + ); + assert.equal( + repair.buildTraceRepairPlan(activeLongRun, [ + { runId: activeLongRun.runId, seq: 1, ts: 95_000, kind: 'step_start', data: { step: 1 } }, + ], { now: 100_000, staleAfterMs: 60_000 }), + null, + `${label}: recent durable activity must protect a long-running active trace`, + ); + assert.equal( + repair.buildTraceRepairPlan(repaired, [], { now: 100_000, staleAfterMs: 60_000 }), + null, + `${label}: repaired runs must be idempotent`, + ); + } + assert.equal( + fs.readFileSync(path.join(ROOT, 'src/chrome/src/trace/repair.js'), 'utf8'), + fs.readFileSync(path.join(ROOT, 'src/firefox/src/trace/repair.js'), 'utf8'), + 'Chrome and Firefox repair helpers must remain mirrored', + ); +}); + +test('trace repair: recorder and browser entry points apply the repair transaction', () => { + for (const browser of ['chrome', 'firefox']) { + const recorder = fs.readFileSync(path.join(ROOT, `src/${browser}/src/trace/recorder.js`), 'utf8'); + const background = fs.readFileSync(path.join(ROOT, `src/${browser}/src/background.js`), 'utf8'); + const traces = fs.readFileSync(path.join(ROOT, `src/${browser}/src/ui/traces.js`), 'utf8'); + assert.match(recorder, /import \{[\s\S]*?buildTraceRepairPlan[\s\S]*?from '\.\/repair\.js';/, `${browser}: recorder repair import missing`); + assert.match(recorder, /const transaction = tx\(db, \['runs', 'events'\], 'readwrite'\)/, `${browser}: repair must share one atomic transaction`); + assert.match(recorder, /for \(const event of plan\.events\) eventsStore\.put\(event\);/, `${browser}: repair events are not persisted`); + assert.match(recorder, /runsStore\.put\(plan\.run\);/, `${browser}: repaired run is not persisted`); + assert.match(recorder, /export async function repairStaleRuns\(/, `${browser}: stale-run scan is not exported`); + // Candidate scan must be bounded to the possibly-stale slice of history + // and re-check in-memory liveness at the last instant before writing. + assert.match(recorder, /index\('startedAt'\)[\s\S]*?openCursor\(IDBKeyRange\.upperBound\(cutoff\)\)/, `${browser}: stale-run scan must not walk the full runs store`); + assert.match(recorder, /if \(_runState\.has\(runId\)\) return;\n\s*for \(const event of plan\.events\)/, `${browser}: repair must re-check live runs before writing`); + assert.match(background, /setTimeout\(\(\) => \{ void workflowTrace\.repairStaleRuns\(\)\.catch\(\(\) => \{\}\); \}, TRACE_REPAIR_STARTUP_DELAY_MS\)/, `${browser}: background startup must defer the stale-run scan`); + assert.match(background, /WB_TRACE_REPAIR_STALE_RUNS[\s\S]*?workflowTrace\.repairStaleRuns\(\)/, `${browser}: background does not answer the traces-page repair request`); + assert.match(traces, /\(async \(\) => \{\s*await repairStaleRunsForPage\(\);\s*await refresh\(\);/, `${browser}: Traces page does not route its first repair through the background`); + assert.match(traces, /WB_TRACE_REPAIR_STALE_RUNS[\s\S]*?return repairStaleRuns\(\)\.catch\(\(\) => \[\]\);/, `${browser}: Traces page lost its local fallback for when the background is unreachable`); + } +}); + test('agent trace error classification: _traceErrorCodeFor maps failures to stable codes', () => { for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) { const agent = new AgentClass({});