fix(trace): repair runs interrupted by service-worker eviction

src/chrome/src/background.js

@@ -112,6 +112,11 @@ import {
 const providerManager = new ProviderManager();
 const apocalypseController = createApocalypseController(chrome);
 const VISION_OFFSCREEN_URL = chrome.runtime.getURL('src/offscreen/offscreen.html');
+// The stale-run repair scan waits a beat after wake so a run resuming from
+// eviction registers its in-memory state first; the Traces page can also
+// request an immediate scan via WB_TRACE_REPAIR_STALE_RUNS.
+const TRACE_REPAIR_STARTUP_DELAY_MS = 15_000;
+setTimeout(() => { void workflowTrace.repairStaleRuns().catch(() => {}); }, TRACE_REPAIR_STARTUP_DELAY_MS);
 
 function normalizeVisionDownloadState(state) {
   return {
@@ -1486,6 +1491,16 @@ chrome.contextMenus?.onClicked?.addListener?.((info, tab) => {
   handleContextMenuAsk(info, tab).catch(() => {});
 });
 
+// Only this instance knows which runs are live in memory, so it owns the
+// stale-run repair whenever it is reachable.
+chrome.runtime.onMessage.addListener((msg, _sender, sendResponse) => {
+  if (msg?.type !== 'WB_TRACE_REPAIR_STALE_RUNS') return;
+  workflowTrace.repairStaleRuns()
+    .then(repaired => sendResponse({ ok: true, repaired }))
+    .catch(error => sendResponse({ ok: false, error: error?.message || String(error) }));
+  return true;
+});
+
 chrome.runtime.onMessage.addListener((msg, _sender, sendResponse) => {
   if (msg?.type !== 'WB_SELECTION_SHORTCUT_LOCALIZATION') return;
   sendResponse({ ok: true, ...getSelectionShortcutLocalization(msg.locale) });

src/chrome/src/trace/error-codes.js

@@ -23,6 +23,7 @@ export const ERROR_CODES = Object.freeze([
   'TRANSPORT',
   'TOOL_TIMEOUT',
   'COST_LIMIT',
+  'SERVICE_WORKER_EVICTED',
   'UNKNOWN',
 ]);
 
@@ -35,4 +36,4 @@ export function isErrorCode(code) {
 export function normalizeErrorCode(code) {
   if (isErrorCode(code)) return code;
   return 'UNKNOWN';
-}
+}

src/chrome/src/trace/recorder.js

@@ -4,6 +4,12 @@ import { formatErrorMessage } from '../error-format.js';
 import { TRACE_FORMAT_VERSION, makeEvent } from './event-model.js';
 import { normalizeErrorCode } from './error-codes.js';
 import { normalizeRunHeader, effectiveDelegationDepth } from './run-header.js';
+import {
+  buildTraceRepairPlan,
+  isStaleRunningTrace,
+  normalizedThreshold,
+  TRACE_REPAIR_STALE_AFTER_MS,
+} from './repair.js';
 
 /**
  * Trace recorder — writes per-run traces (LLM requests/responses, tool calls,
@@ -137,6 +143,50 @@ async function _peekSeq(db, runId) {
   return result;
 }
 
+function _repairRunInTransaction(db, runId, { now, staleAfterMs }) {
+  return new Promise((resolve, reject) => {
+    const transaction = tx(db, ['runs', 'events'], 'readwrite');
+    const runsStore = transaction.objectStore('runs');
+    const eventsStore = transaction.objectStore('events');
+    let run = null;
+    let events = null;
+    let pending = 2;
+    let repairedRunId = null;
+    let requestError = null;
+
+    const fail = (error) => {
+      requestError = error || new Error('trace repair request failed');
+      try { transaction.abort(); } catch {}
+    };
+    const apply = () => {
+      pending -= 1;
+      if (pending > 0 || requestError) return;
+      const plan = buildTraceRepairPlan(run, events, { now, staleAfterMs });
+      if (!plan) return;
+      // Last-instant liveness check: a run resumed between the candidate scan
+      // and this transaction has registered itself in memory again.
+      if (_runState.has(runId)) return;
+      for (const event of plan.events) eventsStore.put(event);
+      runsStore.put(plan.run);
+      repairedRunId = runId;
+    };
+
+    transaction.oncomplete = () => resolve(repairedRunId);
+    transaction.onerror = () => reject(requestError || transaction.error || new Error('trace repair failed'));
+    transaction.onabort = () => {
+      if (requestError) reject(requestError);
+      else if (!repairedRunId) resolve(null);
+    };
+
+    const runRequest = runsStore.get(runId);
+    runRequest.onsuccess = () => { run = runRequest.result || null; apply(); };
+    runRequest.onerror = () => fail(runRequest.error);
+    const eventsRequest = eventsStore.index('runId').getAll(IDBKeyRange.only(runId));
+    eventsRequest.onsuccess = () => { events = eventsRequest.result || []; apply(); };
+    eventsRequest.onerror = () => fail(eventsRequest.error);
+  });
+}
+
 function _newSeq(runId) {
   const st = _runState.get(runId);
   if (!st) return 0;
@@ -498,6 +548,62 @@ export async function endRun(runId, { status = 'done', finalContent = null } = {
   }
 }
 
+/**
+ * Repair trace records left running after a service-worker eviction. Each run
+ * is re-read and updated in one transaction so a concurrent normal completion
+ * wins, and a second repair pass cannot append duplicate terminal events.
+ */
+async function _listStaleRunCandidates(db, cutoff) {
+  // Only runs old enough to be stale can qualify (last activity is never
+  // older than startedAt), so scan just that slice of the startedAt index
+  // instead of every run on record.
+  const out = [];
+  await new Promise((resolve, reject) => {
+    const idx = tx(db, ['runs'], 'readonly').objectStore('runs').index('startedAt');
+    const req = idx.openCursor(IDBKeyRange.upperBound(cutoff));
+    req.onsuccess = () => {
+      const c = req.result;
+      if (!c) return resolve();
+      const row = c.value;
+      if (row?.status === 'running' && !row.repairedBy) out.push(row);
+      c.continue();
+    };
+    req.onerror = () => reject(req.error || new Error('stale-run scan failed'));
+  });
+  return out;
+}
+
+export async function repairStaleRuns({
+  now = Date.now(),
+  staleAfterMs = TRACE_REPAIR_STALE_AFTER_MS,
+} = {}) {
+  if (typeof indexedDB === 'undefined') return [];
+  if (!Number.isFinite(Number(now))) return [];
+  try {
+    const db = await openDB();
+    const cutoff = Number(now) - normalizedThreshold(staleAfterMs);
+    const candidates = await _listStaleRunCandidates(db, cutoff);
+    const repaired = [];
+    for (const candidate of candidates) {
+      if (!isStaleRunningTrace(candidate, { now, staleAfterMs })) continue;
+      // A live run in this service-worker instance is still owned by the
+      // agent. The durable marker handles races from another extension page.
+      if (_runState.has(candidate.runId)) continue;
+      await _flushRunWrites(candidate.runId);
+      try {
+        const repairedRunId = await _repairRunInTransaction(db, candidate.runId, { now, staleAfterMs });
+        if (repairedRunId) repaired.push(repairedRunId);
+      } catch (e) {
+        console.warn('[trace] stale-run repair failed:', e);
+      }
+    }
+    return repaired;
+  } catch (e) {
+    console.warn('[trace] stale-run scan failed:', e);
+    return [];
+  }
+}
+
 // ----- Reader API (used by traces.html) --------------------------------------
 
 export async function listRuns({ limit = 500, conversationId = null } = {}) {

src/chrome/src/trace/repair.js

@@ -0,0 +1,124 @@
+import { makeEvent } from './event-model.js';
+import { normalizeErrorCode } from './error-codes.js';
+
+// A run can legitimately take a while, so repair only treats records older
+// than this conservative window as abandoned. Callers/tests may provide a
+// smaller explicit threshold when they need a deterministic boundary.
+export const TRACE_REPAIR_STALE_AFTER_MS = 10 * 60 * 1000;
+export const TRACE_REPAIR_MARKER = 'service-worker-eviction';
+export const TRACE_REPAIR_ERROR_CODE = 'SERVICE_WORKER_EVICTED';
+export const TRACE_REPAIR_REASON = 'service_worker_eviction';
+export const TRACE_REPAIR_MESSAGE = 'Trace run interrupted by service-worker eviction.';
+
+function repairEvent(runId, seq, kind, data, now) {
+  const event = makeEvent(runId, seq, kind, data);
+  return event ? { ...event, ts: now } : null;
+}
+
+function eventStep(event) {
+  return Number.isInteger(event?.data?.step) ? event.data.step : null;
+}
+
+function openSteps(events) {
+  const open = new Map();
+  for (const event of events) {
+    if (event?.kind === 'step_start') open.set(eventStep(event), eventStep(event));
+    if (event?.kind === 'step_end') open.delete(eventStep(event));
+  }
+  return [...open.values()].sort((a, b) => (a ?? 0) - (b ?? 0));
+}
+
+function openTurnStep(events) {
+  let step = null;
+  for (const event of events) {
+    if (event?.kind === 'turn_start') step = eventStep(event);
+    if (event?.kind === 'turn_end') step = null;
+  }
+  return step;
+}
+
+export function normalizedThreshold(value) {
+  return Number.isFinite(Number(value)) ? Math.max(0, Number(value)) : TRACE_REPAIR_STALE_AFTER_MS;
+}
+
+export function isStaleRunningTrace(run, {
+  now = Date.now(),
+  staleAfterMs = TRACE_REPAIR_STALE_AFTER_MS,
+  events = [],
+} = {}) {
+  if (!run || run.status !== 'running' || run.repairedBy) return false;
+  const currentTime = Number(now);
+  let lastActivityAt = Number(run.startedAt);
+  if (!Number.isFinite(lastActivityAt) || !Number.isFinite(currentTime)) return false;
+  for (const event of Array.isArray(events) ? events : []) {
+    const eventTime = Number(event?.ts);
+    if (Number.isFinite(eventTime)) lastActivityAt = Math.max(lastActivityAt, eventTime);
+  }
+  return currentTime - lastActivityAt >= normalizedThreshold(staleAfterMs);
+}
+
+/**
+ * Build the durable repair mutation for one abandoned trace.
+ *
+ * This is deliberately pure: the recorder applies the returned event/run
+ * values in one IndexedDB transaction, while tests can prove the interruption
+ * semantics without a browser or a real IndexedDB implementation.
+ */
+export function buildTraceRepairPlan(run, events, {
+  now = Date.now(),
+  staleAfterMs = TRACE_REPAIR_STALE_AFTER_MS,
+} = {}) {
+  const orderedEvents = (Array.isArray(events) ? events : [])
+    .filter(Boolean)
+    .slice()
+    .sort((a, b) => (Number(a.seq) || 0) - (Number(b.seq) || 0));
+  if (!isStaleRunningTrace(run, { now, staleAfterMs, events: orderedEvents })) return null;
+  const lastSeq = orderedEvents.reduce((max, event) => Math.max(max, Number(event.seq) || 0), 0);
+  const code = normalizeErrorCode(TRACE_REPAIR_ERROR_CODE);
+  const repairedEvents = [];
+  let nextSeq = lastSeq + 1;
+
+  for (const step of openSteps(orderedEvents)) {
+    repairedEvents.push(repairEvent(run.runId, nextSeq++, 'step_end', {
+      step,
+      ok: false,
+      reason: TRACE_REPAIR_REASON,
+      code,
+      repaired: true,
+    }, now));
+  }
+
+  repairedEvents.push(repairEvent(run.runId, nextSeq++, 'error', {
+    step: null,
+    phase: 'repair',
+    message: TRACE_REPAIR_MESSAGE,
+    code,
+  }, now));
+
+  const turnStep = openTurnStep(orderedEvents);
+  if (turnStep !== null) {
+    repairedEvents.push(repairEvent(run.runId, nextSeq, 'turn_end', {
+      step: turnStep,
+      status: 'error',
+      reason: TRACE_REPAIR_REASON,
+      code,
+      repaired: true,
+    }, now));
+  }
+
+  const maxStep = orderedEvents.reduce((max, event) => Math.max(max, eventStep(event) ?? 0), 0);
+  const startedAt = Number(run.startedAt);
+  return {
+    events: repairedEvents.filter(Boolean),
+    run: {
+      ...run,
+      endedAt: now,
+      durationMs: Math.max(0, now - startedAt),
+      status: 'error',
+      stepCount: Math.max(Number(run.stepCount) || 0, maxStep),
+      repairedBy: TRACE_REPAIR_MARKER,
+      repairedAt: now,
+      repairReason: TRACE_REPAIR_REASON,
+    },
+  };
+}

src/chrome/src/ui/traces.js

@@ -5,12 +5,14 @@
 
 import {
   listRuns, getRun, getRunEvents, getScreenshot,
-  deleteRun, clearAllRuns,
+  deleteRun, clearAllRuns, repairStaleRuns,
 } from '../trace/recorder.js';
 import { isKnownKind, isIgnorableKind } from '../trace/event-model.js';
 import { t } from './i18n.js';
 import { escapeHtml, escapeAttr } from './utils.js';
 
+const runtimeApi = globalThis.browser || globalThis.chrome;
+
 const listEl = document.getElementById('run-list');
 const mainPane = document.getElementById('main-pane');
 const emptyState = document.getElementById('empty-state');
@@ -645,9 +647,31 @@ document.addEventListener('visibilitychange', () => {
   else if (hasRunningJob()) scheduleAutoRefresh();
 });
 
+// Prefer letting the background own the stale-run scan: only its recorder
+// instance knows which runs are live in memory. If it cannot be reached,
+// nothing can be running anywhere, so a local pass is safe.
+async function repairStaleRunsForPage({ timeoutMs = 5_000 } = {}) {
+  let timer;
+  try {
+    if (runtimeApi?.runtime?.sendMessage) {
+      const response = await Promise.race([
+        runtimeApi.runtime.sendMessage({ type: 'WB_TRACE_REPAIR_STALE_RUNS' }),
+        new Promise((_resolve, reject) => {
+          timer = setTimeout(() => reject(new Error('trace repair roundtrip timed out')), timeoutMs);
+        }),
+      ]);
+      if (response?.ok) return Array.isArray(response.repaired) ? response.repaired : [];
+    }
+  } catch {} finally {
+    clearTimeout(timer);
+  }
+  return repairStaleRuns().catch(() => []);
+}
+
 // Initial load: always do one refresh so the list populates, then only keep
 // polling if the freshly-loaded data shows a live run.
 (async () => {
+  await repairStaleRunsForPage();
   await refresh();
   if (initialRunId && await ensureRunLoaded(initialRunId)) {
     selectedRunId = initialRunId;

src/firefox/src/background.js

@@ -100,6 +100,11 @@ import {
 const providerManager = new ProviderManager();
 const apocalypseController = createApocalypseController(browser);
 let emergencyDownloads = null;
+// The stale-run repair scan waits a beat after wake so a run resuming from
+// eviction registers its in-memory state first; the Traces page can also
+// request an immediate scan via WB_TRACE_REPAIR_STALE_RUNS.
+const TRACE_REPAIR_STARTUP_DELAY_MS = 15_000;
+setTimeout(() => { void workflowTrace.repairStaleRuns().catch(() => {}); }, TRACE_REPAIR_STARTUP_DELAY_MS);
 
 function emergencyDownloadController() {
   if (!emergencyDownloads) {
@@ -1261,6 +1266,16 @@ getContextMenuApi()?.onClicked?.addListener?.((info, tab) => {
   handleContextMenuAsk(info, tab).catch(() => {});
 });
 
+// Only this instance knows which runs are live in memory, so it owns the
+// stale-run repair whenever it is reachable.
+browser.runtime.onMessage.addListener((msg, _sender, sendResponse) => {
+  if (msg?.type !== 'WB_TRACE_REPAIR_STALE_RUNS') return;
+  workflowTrace.repairStaleRuns()
+    .then(repaired => sendResponse({ ok: true, repaired }))
+    .catch(error => sendResponse({ ok: false, error: error?.message || String(error) }));
+  return true;
+});
+
 browser.runtime.onMessage.addListener((msg, _sender, sendResponse) => {
   if (msg?.type !== 'WB_SELECTION_SHORTCUT_LOCALIZATION') return;
   sendResponse({ ok: true, ...getSelectionShortcutLocalization(msg.locale) });

src/firefox/src/trace/error-codes.js

@@ -23,6 +23,7 @@ export const ERROR_CODES = Object.freeze([
   'TRANSPORT',
   'TOOL_TIMEOUT',
   'COST_LIMIT',
+  'SERVICE_WORKER_EVICTED',
   'UNKNOWN',
 ]);
 
@@ -35,4 +36,4 @@ export function isErrorCode(code) {
 export function normalizeErrorCode(code) {
   if (isErrorCode(code)) return code;
   return 'UNKNOWN';
-}
+}