From 3ed005cc164ff494d54ffabdbb88db02b3336760 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:23:06 +0000 Subject: [PATCH 1/7] [Fix] Bound OpenCode session creation so hung self-host tasks fail closed Add request timeouts and session-create diagnostics so tasks no longer sit running forever when OpenCode never returns from the first POST /session. --- apps/worker/src/run-task/create-harness.ts | 3 + .../__tests__/opencode-server-client.test.ts | 118 ++++++++++ .../opencode-server-stall-watchdogs.test.ts | 5 +- .../opencode-server-subagent-watchdog.test.ts | 3 + .../__tests__/opencode-server.test.ts | 77 +++++++ .../lib/harnesses/opencode-server/client.ts | 214 +++++++++++++++--- .../lib/harnesses/opencode-server/harness.ts | 123 +++++++++- .../lib/harnesses/opencode-server/start.ts | 20 ++ 8 files changed, 529 insertions(+), 34 deletions(-) create mode 100644 apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-client.test.ts diff --git a/apps/worker/src/run-task/create-harness.ts b/apps/worker/src/run-task/create-harness.ts index 31bcfca2..bdd2562e 100644 --- a/apps/worker/src/run-task/create-harness.ts +++ b/apps/worker/src/run-task/create-harness.ts @@ -115,6 +115,9 @@ export async function createHarness({ return await startOpenCodeServerHarness({ ...commonOptions, developerInstructionsContent, + onDiagnostic: (input) => { + diagnosticEvents.record(input); + }, onUnexpectedExit: (certificate) => { const summary = `OpenCode server exited unexpectedly (code=${ certificate.exitCode ?? 'none' diff --git a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-client.test.ts b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-client.test.ts new file mode 100644 index 00000000..74f295a5 --- /dev/null +++ b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-client.test.ts @@ -0,0 +1,118 @@ +import { + DEFAULT_OPENCODE_HTTP_TIMEOUT_MS, + DEFAULT_OPENCODE_SESSION_CREATE_TIMEOUT_MS, + OpenCodeServerClient, + formatOpenCodeSessionCreateTimeoutText, + resolveOpenCodeHttpTimeoutMs, + resolveOpenCodeSessionCreateTimeoutMs, +} from '../opencode-server/client'; + +function createLogger() { + return { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + log: vi.fn(), + }; +} + +describe('OpenCodeServerClient timeouts', () => { + afterEach(() => { + vi.unstubAllGlobals(); + vi.useRealTimers(); + }); + + it('resolves timeout overrides from env', () => { + expect( + resolveOpenCodeHttpTimeoutMs(undefined, { + ROOMOTE_OPENCODE_HTTP_TIMEOUT_MS: '45000', + }), + ).toBe(45_000); + expect( + resolveOpenCodeSessionCreateTimeoutMs(undefined, { + ROOMOTE_OPENCODE_SESSION_CREATE_TIMEOUT_MS: '120000', + }), + ).toBe(120_000); + expect(resolveOpenCodeHttpTimeoutMs()).toBe( + DEFAULT_OPENCODE_HTTP_TIMEOUT_MS, + ); + expect(resolveOpenCodeSessionCreateTimeoutMs()).toBe( + DEFAULT_OPENCODE_SESSION_CREATE_TIMEOUT_MS, + ); + }); + + it('times out createSession when OpenCode never responds', async () => { + vi.useFakeTimers(); + const logger = createLogger(); + const fetchMock = vi.fn((_url: URL | string, init?: RequestInit) => { + return new Promise((_resolve, reject) => { + const signal = init?.signal; + signal?.addEventListener('abort', () => { + reject(new DOMException('The operation was aborted.', 'AbortError')); + }); + }); + }); + vi.stubGlobal('fetch', fetchMock); + + const client = new OpenCodeServerClient({ + baseUrl: 'http://127.0.0.1:4096', + workspacePath: '/sandbox/repos', + logger: logger as never, + sessionCreateTimeoutMs: 50, + }); + + const pending = client.createSession({ title: 'hello' }); + const assertion = expect(pending).rejects.toThrow( + formatOpenCodeSessionCreateTimeoutText(50), + ); + + await vi.advanceTimersByTimeAsync(50); + await assertion; + expect(fetchMock).toHaveBeenCalledTimes(1); + const calledUrl = String(fetchMock.mock.calls[0]?.[0]); + expect(calledUrl).toContain('/session'); + expect(calledUrl).toContain('directory=%2Fsandbox%2Frepos'); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining('HTTP request timed out'), + ); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining('label=createSession'), + ); + }); + + it('logs start/completion for successful session create', async () => { + const logger = createLogger(); + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + return new Response(JSON.stringify({ id: 'ses_new', title: 't' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }), + ); + + const client = new OpenCodeServerClient({ + baseUrl: 'http://127.0.0.1:4096', + workspacePath: '/sandbox/repos', + logger: logger as never, + sessionCreateTimeoutMs: 5_000, + }); + + await expect(client.createSession()).resolves.toEqual({ + id: 'ses_new', + title: 't', + }); + + expect(logger.info).toHaveBeenCalledWith( + expect.stringContaining('HTTP request start'), + ); + expect(logger.info).toHaveBeenCalledWith( + expect.stringContaining('HTTP request ok'), + ); + expect(logger.info).toHaveBeenCalledWith( + expect.stringContaining('label=createSession'), + ); + }); +}); diff --git a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-stall-watchdogs.test.ts b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-stall-watchdogs.test.ts index 52923c6a..6869c992 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-stall-watchdogs.test.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-stall-watchdogs.test.ts @@ -28,7 +28,10 @@ class FakeOpenCodeServerClient { promptAsync = vi.fn(async (_options: unknown) => undefined); messages = vi.fn(async () => [] as OpenCodeSessionMessage[]); message = vi.fn<() => Promise>(); - abort = vi.fn(async (_options: { sessionId: string }) => true); + abort = vi.fn(async () => true); + get sessionCreateTimeoutMsValue(): number { + return 90_000; + } streamEvents = vi.fn( async (options: { signal: AbortSignal; diff --git a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-subagent-watchdog.test.ts b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-subagent-watchdog.test.ts index 4412d94d..e4e0170f 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-subagent-watchdog.test.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-subagent-watchdog.test.ts @@ -28,6 +28,9 @@ class FakeOpenCodeServerClient { children = vi.fn( async (_options: { sessionId: string }) => [] as OpenCodeSession[], ); + get sessionCreateTimeoutMsValue(): number { + return 90_000; + } streamEvents = vi.fn( async (options: { signal: AbortSignal; diff --git a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts index b0be55bb..7525378f 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts @@ -41,6 +41,9 @@ class FakeOpenCodeServerClient { messages = vi.fn(async () => [] as OpenCodeSessionMessage[]); message = vi.fn<() => Promise>(); abort = vi.fn(async () => true); + get sessionCreateTimeoutMsValue(): number { + return 90_000; + } streamEvents = vi.fn( async (options: { signal: AbortSignal; @@ -3380,6 +3383,80 @@ describe('OpenCodeServerHarness', () => { harness.dispose(); } }); + + it('aborts the initial task and records a diagnostic when session create fails', async () => { + const client = new FakeOpenCodeServerClient(); + client.createSession.mockRejectedValueOnce( + new Error( + 'OpenCode session creation did not respond within 90s. The OpenCode server is up, but the first session request never finished.', + ), + ); + const onDiagnostic = vi.fn(); + const harness = new OpenCodeServerHarness({ + client: client as unknown as OpenCodeServerClient, + workspacePath: '/sandbox/repos', + logger: createLogger(), + model: TEST_OPENCODE_MODEL, + eventStreamReadyTimeoutMs: 100, + mcpServerNames: ['roomote'], + commandEnv: { HOME: '/sandbox/repos/.roomote-runtime-home' }, + onDiagnostic, + }); + const taskEvents: TaskEvent[] = []; + const persistedEnvelopes: AcpPersistedEnvelope[] = []; + + harness.subscribe((event) => taskEvents.push(event)); + harness.subscribeRuntimePersistedEnvelope((envelope) => + persistedEnvelopes.push(envelope), + ); + + try { + await connectHarness(harness, client); + + harness.sendCommand({ + commandName: TaskCommandName.StartNewTask, + data: { text: 'Start work.', visibleInTranscript: true }, + }); + + await vi.waitFor(() => { + expect( + taskEvents.some( + (event) => event.eventName === TaskEventName.TaskAborted, + ), + ).toBe(true); + }); + + expect( + taskEvents.some( + (event) => + event.eventName === TaskEventName.TaskStarted && + event.payload?.[0] === 'opencode-session-create-failed', + ), + ).toBe(true); + expect(client.promptAsync).not.toHaveBeenCalled(); + expect(onDiagnostic).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'opencode_session_create_failed', + details: expect.objectContaining({ + workspacePath: '/sandbox/repos', + homeDir: '/sandbox/repos/.roomote-runtime-home', + mcpServerNames: ['roomote'], + }), + }), + ); + expect( + persistedEnvelopes.some( + (envelope) => + envelope.eventType === ACP_ENVELOPE_EVENT_TYPES.AssistantMessage && + String(envelope.payload.text ?? '').includes( + 'session creation did not respond', + ), + ), + ).toBe(true); + } finally { + harness.dispose(); + } + }); }); describe('OpenCodeServerHarness cancel marker', () => { diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/client.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/client.ts index 3e91e51a..49df2367 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/client.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/client.ts @@ -8,10 +8,22 @@ import type { OpenCodeSessionMessage, } from './types'; +// Bound every unary OpenCode HTTP call so a wedged server cannot leave the +// worker sitting forever on a bare `fetch` with only the task-wide cancel +// signal. Session create is the first workspace touch (lazy instance bootstrap) +// and historically has wedged self-hosted workers with zero further logs — +// give it its own ceiling and override. Streaming events are unbounded. +export const DEFAULT_OPENCODE_HTTP_TIMEOUT_MS = 60_000; +export const DEFAULT_OPENCODE_SESSION_CREATE_TIMEOUT_MS = 90_000; +export const DEFAULT_OPENCODE_HEALTH_TIMEOUT_MS = 15_000; + interface OpenCodeServerClientOptions { baseUrl: string; workspacePath: string; logger: HarnessLogger; + httpTimeoutMs?: number; + sessionCreateTimeoutMs?: number; + healthTimeoutMs?: number; } interface RequestOptions { @@ -20,6 +32,85 @@ interface RequestOptions { signal?: AbortSignal; expectedStatus?: number; query?: Record; + timeoutMs?: number; + label?: string; +} + +export function parsePositiveTimeoutMs( + value: string | undefined, +): number | undefined { + const trimmed = value?.trim(); + + if (!trimmed) { + return undefined; + } + + const parsed = Number.parseInt(trimmed, 10); + + if (!Number.isFinite(parsed) || parsed <= 0) { + return undefined; + } + + return parsed; +} + +export function resolveOpenCodeHttpTimeoutMs( + override?: number, + env: NodeJS.ProcessEnv = process.env, +): number { + return ( + override ?? + parsePositiveTimeoutMs(env.ROOMOTE_OPENCODE_HTTP_TIMEOUT_MS) ?? + DEFAULT_OPENCODE_HTTP_TIMEOUT_MS + ); +} + +export function resolveOpenCodeSessionCreateTimeoutMs( + override?: number, + env: NodeJS.ProcessEnv = process.env, +): number { + return ( + override ?? + parsePositiveTimeoutMs(env.ROOMOTE_OPENCODE_SESSION_CREATE_TIMEOUT_MS) ?? + DEFAULT_OPENCODE_SESSION_CREATE_TIMEOUT_MS + ); +} + +export function resolveOpenCodeHealthTimeoutMs( + override?: number, + env: NodeJS.ProcessEnv = process.env, +): number { + return ( + override ?? + parsePositiveTimeoutMs(env.ROOMOTE_OPENCODE_HEALTH_TIMEOUT_MS) ?? + DEFAULT_OPENCODE_HEALTH_TIMEOUT_MS + ); +} + +export function formatOpenCodeSessionCreateTimeoutText( + timeoutMs: number, +): string { + const seconds = Math.max(1, Math.round(timeoutMs / 1_000)); + + return `OpenCode session creation did not respond within ${seconds}s. The OpenCode server is up, but the first session request never finished — usually a hang while OpenCode bootstraps the workspace instance (config, plugins, storage, or MCP). Cancel and retry, or inspect the worker OpenCode logs for a stuck \`creating instance\` / \`bootstrapping\` sequence.`; +} + +function composeAbortSignals( + signals: Array, +): AbortSignal | undefined { + const defined = signals.filter( + (signal): signal is AbortSignal => signal !== undefined, + ); + + if (defined.length === 0) { + return undefined; + } + + if (defined.length === 1) { + return defined[0]; + } + + return AbortSignal.any(defined); } function appendDirectorySearchParam( @@ -116,6 +207,9 @@ export class OpenCodeServerClient { private readonly baseUrl: string; private readonly workspacePath: string; private readonly logger: HarnessLogger; + private readonly httpTimeoutMs: number; + private readonly sessionCreateTimeoutMs: number; + private readonly healthTimeoutMs: number; constructor(options: OpenCodeServerClientOptions) { this.baseUrl = options.baseUrl.endsWith('/') @@ -123,6 +217,21 @@ export class OpenCodeServerClient { : `${options.baseUrl}/`; this.workspacePath = options.workspacePath; this.logger = options.logger; + this.httpTimeoutMs = resolveOpenCodeHttpTimeoutMs(options.httpTimeoutMs); + this.sessionCreateTimeoutMs = resolveOpenCodeSessionCreateTimeoutMs( + options.sessionCreateTimeoutMs, + ); + this.healthTimeoutMs = resolveOpenCodeHealthTimeoutMs( + options.healthTimeoutMs, + ); + } + + get sessionCreateTimeoutMsValue(): number { + return this.sessionCreateTimeoutMs; + } + + get workspaceDirectory(): string { + return this.workspacePath; } async health( @@ -131,6 +240,8 @@ export class OpenCodeServerClient { return await this.request('/global/health', { signal, query: undefined, + timeoutMs: this.healthTimeoutMs, + label: 'health', }); } @@ -142,6 +253,8 @@ export class OpenCodeServerClient { method: 'POST', body: options?.title ? { title: options.title } : {}, signal: options?.signal, + timeoutMs: this.sessionCreateTimeoutMs, + label: 'createSession', }); } @@ -157,6 +270,7 @@ export class OpenCodeServerClient { body: options.request, signal: options.signal, expectedStatus: 204, + label: 'promptAsync', }, ); } @@ -171,6 +285,7 @@ export class OpenCodeServerClient { { signal: options.signal, query: { limit: options.limit }, + label: 'messages', }, ); } @@ -186,6 +301,7 @@ export class OpenCodeServerClient { )}`, { signal: options.signal, + label: 'message', }, ); } @@ -198,6 +314,7 @@ export class OpenCodeServerClient { `/session/${encodeURIComponent(options.sessionId)}/children`, { signal: options.signal, + label: 'children', }, ); } @@ -212,6 +329,7 @@ export class OpenCodeServerClient { method: 'POST', body: {}, signal: options.signal, + label: 'abort', }, ); } @@ -276,41 +394,85 @@ export class OpenCodeServerClient { path: string, options: RequestOptions = {}, ): Promise { + const method = options.method ?? 'GET'; + const label = options.label ?? path; + const timeoutMs = options.timeoutMs ?? this.httpTimeoutMs; const url = new URL(path.replace(/^\//u, ''), this.baseUrl); appendDirectorySearchParam(url, this.workspacePath, options.query); - const response = await fetch(url, { - method: options.method ?? 'GET', - signal: options.signal, - headers: { + const timeoutSignal = AbortSignal.timeout(timeoutMs); + const signal = composeAbortSignals([options.signal, timeoutSignal]); + const startedAt = Date.now(); + + this.logger.info( + `[opencode-server] HTTP request start method=${method} path=${path} label=${label} directory=${this.workspacePath} timeoutMs=${timeoutMs}`, + ); + + try { + const response = await fetch(url, { + method, + signal, + headers: { + ...(options.body !== undefined + ? { 'Content-Type': 'application/json' } + : {}), + }, ...(options.body !== undefined - ? { 'Content-Type': 'application/json' } + ? { body: JSON.stringify(options.body) } : {}), - }, - ...(options.body !== undefined - ? { body: JSON.stringify(options.body) } - : {}), - }); - const expectedStatus = options.expectedStatus ?? 200; + }); + const expectedStatus = options.expectedStatus ?? 200; + const elapsedMs = Date.now() - startedAt; + + if (response.status === expectedStatus) { + if (expectedStatus === 204) { + this.logger.info( + `[opencode-server] HTTP request ok method=${method} path=${path} label=${label} status=${response.status} elapsedMs=${elapsedMs}`, + ); + return undefined as T; + } - if (response.status === expectedStatus) { - if (expectedStatus === 204) { - return undefined as T; + const body = (await response.json()) as T; + this.logger.info( + `[opencode-server] HTTP request ok method=${method} path=${path} label=${label} status=${response.status} elapsedMs=${elapsedMs}`, + ); + return body; } - return (await response.json()) as T; - } + const responseText = await response.text().catch(() => ''); + this.logger.warn( + `[opencode-server] HTTP request failed method=${method} path=${path} label=${label} status=${response.status} elapsedMs=${elapsedMs} body=${responseText.slice( + 0, + 500, + )}`, + ); + throw new Error( + `OpenCode request failed method=${method} path=${path} status=${response.status}`, + ); + } catch (error) { + const elapsedMs = Date.now() - startedAt; + + if (timeoutSignal.aborted && !options.signal?.aborted) { + const message = + label === 'createSession' + ? formatOpenCodeSessionCreateTimeoutText(timeoutMs) + : `OpenCode ${label} request timed out after ${timeoutMs}ms method=${method} path=${path} directory=${this.workspacePath}`; + this.logger.error( + `[opencode-server] HTTP request timed out method=${method} path=${path} label=${label} timeoutMs=${timeoutMs} elapsedMs=${elapsedMs} directory=${this.workspacePath}`, + ); + throw new Error(message); + } - const responseText = await response.text().catch(() => ''); - this.logger.warn( - `[opencode-server] HTTP request failed method=${options.method ?? 'GET'} path=${path} status=${response.status} body=${responseText.slice( - 0, - 500, - )}`, - ); - throw new Error( - `OpenCode request failed method=${options.method ?? 'GET'} path=${path} status=${response.status}`, - ); + if (!options.signal?.aborted) { + this.logger.warn( + `[opencode-server] HTTP request error method=${method} path=${path} label=${label} elapsedMs=${elapsedMs} error=${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + + throw error; + } } } diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts index 5f743e35..e1a9771f 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts @@ -48,7 +48,11 @@ import { import { RuntimePromptQueue } from '../runtime-prompt-queue'; import { OpenCodeRuntimeEventEmitter } from './runtime-event-emitter'; -import { OpenCodeServerClient, createOpenCodePromptParts } from './client'; +import { + OpenCodeServerClient, + createOpenCodePromptParts, + formatOpenCodeSessionCreateTimeoutText, +} from './client'; import { PLAN_WORKFLOW_SKILL, resolveWorkflowSkillTransition, @@ -94,6 +98,15 @@ interface OpenCodeServerHarnessOptions { subagentSettlementGraceMs?: number; queuedPromptRetryDelayMs?: number; mcpServerNames?: string[]; + /** + * Observer-only breadcrumb for rare harness failures that need a durable + * post-mortem outside the sandbox (e.g. infinite OpenCode session create). + */ + onDiagnostic?: (input: { + kind: string; + message: string; + details?: Record; + }) => void; beforeQueuedPrompt?: (input: { userId?: string }) => Promise right.length - left.length); this.beforeQueuedPrompt = options.beforeQueuedPrompt; + this.onDiagnostic = options.onDiagnostic; this.runtimeEvents = new OpenCodeRuntimeEventEmitter({ taskEvent: (event) => this.emit('taskEvent', event), runtimeOutput: (event) => this.emit('runtimeOutput', event), @@ -1976,7 +1993,15 @@ export class OpenCodeServerHarness this.currentWorkflowPhase = command.data.workflowPhase ?? null; this.activeWorkflowSkill = null; - const sessionId = await this.ensureSession(command.data.text); + let sessionId: string; + + try { + sessionId = await this.ensureSession(command.data.text); + } catch (error) { + this.failSessionCreateForInitialTask(error); + throw error; + } + this.runtimeEvents.taskStarted(sessionId); this.runtimeEvents.userPrompt({ sessionId, @@ -2948,12 +2973,96 @@ export class OpenCodeServerHarness return this.sessionId; } - const session = await this.client.createSession({ - title: title?.slice(0, 80), - signal: this.eventAbortController.signal, + const startedAt = Date.now(); + const timeoutMs = this.client.sessionCreateTimeoutMsValue; + + this.logger.info( + `Creating OpenCode session workspace=${this.workspacePath} mcpServers=${ + this.knownMcpServerNames.length > 0 + ? this.knownMcpServerNames.join(',') + : 'none' + } timeoutMs=${timeoutMs}`, + ); + + try { + const session = await this.client.createSession({ + title: title?.slice(0, 80), + signal: this.eventAbortController.signal, + }); + this.sessionId = session.id; + this.logger.info( + `Created OpenCode session sessionId=${session.id} elapsedMs=${ + Date.now() - startedAt + }`, + ); + return session.id; + } catch (error) { + const elapsedMs = Date.now() - startedAt; + const message = error instanceof Error ? error.message : String(error); + + this.logger.error( + `OpenCode session create failed elapsedMs=${elapsedMs} timeoutMs=${timeoutMs} workspace=${this.workspacePath} mcpServers=${ + this.knownMcpServerNames.length > 0 + ? this.knownMcpServerNames.join(',') + : 'none' + } error=${message}`, + ); + this.recordSessionCreateFailureDiagnostic({ + message, + elapsedMs, + timeoutMs, + }); + throw error; + } + } + + private recordSessionCreateFailureDiagnostic(input: { + message: string; + elapsedMs: number; + timeoutMs: number; + }): void { + this.onDiagnostic?.({ + kind: 'opencode_session_create_failed', + message: `OpenCode session creation failed after ${input.elapsedMs}ms: ${input.message}`, + details: { + workspacePath: this.workspacePath, + homeDir: this.commandEnv?.HOME ?? null, + mcpServerNames: this.knownMcpServerNames, + model: this.model?.qualifiedModel ?? null, + elapsedMs: input.elapsedMs, + timeoutMs: input.timeoutMs, + error: input.message, + }, + }); + } + + /** + * Leave a terminal, retryable failure when the very first session never + * materializes. Without this the harness manager stays in `running` forever + * (commandError is currently unsubscribed) and `first_assistant_output_at` + * never stamps. + */ + private failSessionCreateForInitialTask(error: unknown): void { + if (this.sessionId) { + return; + } + + const message = error instanceof Error ? error.message : String(error); + const sessionId = 'opencode-session-create-failed'; + const timeoutMs = this.client.sessionCreateTimeoutMsValue; + const userText = message.includes('did not respond within') + ? message + : `OpenCode session creation failed before the agent could start.\n\n${message}\n\n${formatOpenCodeSessionCreateTimeoutText(timeoutMs)}`; + + this.logger.error( + `OpenCode initial session create failed; aborting task so it does not hang in running forever error=${message}`, + ); + this.runtimeEvents.taskStarted(sessionId); + this.runtimeEvents.assistantMessage({ + sessionId, + text: userText, }); - this.sessionId = session.id; - return session.id; + this.runtimeEvents.taskAborted(sessionId); } private async validateResumedSession(sessionId: string): Promise { diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/start.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/start.ts index 16855729..fead610d 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/start.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/start.ts @@ -31,6 +31,15 @@ interface StartOpenCodeServerHarnessOptions { * failures inside the callback must not affect the harness lifecycle. */ onUnexpectedExit?: (certificate: OpenCodeExitCertificate) => void; + /** + * Observer-only diagnostic breadcrumb for rare OpenCode runtime failures + * that need durable post-mortem outside the sandbox log surface. + */ + onDiagnostic?: (input: { + kind: string; + message: string; + details?: Record; + }) => void; beforeQueuedPrompt?: (input: { userId?: string }) => Promise { const log = createPrefixedLogger(logger, '[opencode-server]'); @@ -349,6 +359,15 @@ export async function startOpenCodeServerHarness({ baseUrl, workspacePath, logger, + httpTimeoutMs: parseTimeoutMs( + process.env.ROOMOTE_OPENCODE_HTTP_TIMEOUT_MS, + ), + sessionCreateTimeoutMs: parseTimeoutMs( + process.env.ROOMOTE_OPENCODE_SESSION_CREATE_TIMEOUT_MS, + ), + healthTimeoutMs: parseTimeoutMs( + process.env.ROOMOTE_OPENCODE_HEALTH_TIMEOUT_MS, + ), }); const harness = new OpenCodeServerHarness({ client, @@ -370,6 +389,7 @@ export async function startOpenCodeServerHarness({ process.env.ROOMOTE_SUBAGENT_SETTLEMENT_GRACE_MS, ), mcpServerNames: Object.keys(mcpServers), + onDiagnostic, beforeQueuedPrompt, }); From 1f6eb2509ea1bb354415c58b7832ed2b6a59f516 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:24:16 +0000 Subject: [PATCH 2/7] [Fix] Drop unused OpenCode timeout exports for knip --- .../lib/harnesses/opencode-server/client.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/client.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/client.ts index 49df2367..7aa1ae25 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/client.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/client.ts @@ -15,7 +15,7 @@ import type { // give it its own ceiling and override. Streaming events are unbounded. export const DEFAULT_OPENCODE_HTTP_TIMEOUT_MS = 60_000; export const DEFAULT_OPENCODE_SESSION_CREATE_TIMEOUT_MS = 90_000; -export const DEFAULT_OPENCODE_HEALTH_TIMEOUT_MS = 15_000; +const DEFAULT_OPENCODE_HEALTH_TIMEOUT_MS = 15_000; interface OpenCodeServerClientOptions { baseUrl: string; @@ -36,9 +36,7 @@ interface RequestOptions { label?: string; } -export function parsePositiveTimeoutMs( - value: string | undefined, -): number | undefined { +function parsePositiveTimeoutMs(value: string | undefined): number | undefined { const trimmed = value?.trim(); if (!trimmed) { @@ -76,7 +74,7 @@ export function resolveOpenCodeSessionCreateTimeoutMs( ); } -export function resolveOpenCodeHealthTimeoutMs( +function resolveOpenCodeHealthTimeoutMs( override?: number, env: NodeJS.ProcessEnv = process.env, ): number { From b4719bbd6ecb6e5be36d0e4200f93a7cfb1a90d0 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:35:25 +0000 Subject: [PATCH 3/7] [Fix] Treat session-create timeouts as terminal failures Stop mapping OpenCode session-create failures through TaskAborted (Canceled); emit a transcript error and shut down via commandError so resolveStatus returns Failed. --- .../src/run-task/reconnectable-harness.ts | 17 ++++++ .../lib/__tests__/harness-manager.test.ts | 57 +++++++++++++++++-- .../src/sandbox-server/lib/harness-manager.ts | 36 ++++++++++++ .../__tests__/opencode-server.test.ts | 23 ++++++-- .../lib/harnesses/opencode-server/harness.ts | 27 +++++++-- 5 files changed, 143 insertions(+), 17 deletions(-) diff --git a/apps/worker/src/run-task/reconnectable-harness.ts b/apps/worker/src/run-task/reconnectable-harness.ts index 6735b2d6..4587ec45 100644 --- a/apps/worker/src/run-task/reconnectable-harness.ts +++ b/apps/worker/src/run-task/reconnectable-harness.ts @@ -24,6 +24,7 @@ import type { HarnessPendingUserInputRequest, HarnessRestartRequest, HarnessQueuedMessage, + HarnessCommandError, QueuedPromptMessageSnapshot, TaskCommand, } from '../sandbox-server/lib/harness'; @@ -58,6 +59,8 @@ interface BoundHarnessListeners { runtimePersistedEnvelope: (envelope: AcpPersistedEnvelope) => void; runtimeTurnCompleted: (event: AcpTurnCompletedEvent) => void; runtimeInferenceUsage: (event: HarnessInferenceUsageEvent) => void; + commandError: (error: HarnessCommandError) => void; + commandErrorUnsubscribe?: () => void; } const DEFAULT_MAX_RECONNECT_ATTEMPTS = 7; @@ -174,6 +177,13 @@ export class ReconnectableHarness return () => this.off('runtimeInferenceUsage', listener); } + subscribeCommandError( + listener: (error: HarnessCommandError) => void, + ): () => void { + this.on('commandError', listener); + return () => this.off('commandError', listener); + } + sendCommand(command: TaskCommand): boolean { this.updateSessionTrackingForCommand(command); this.updateReconnectReplayStateForCommand(command); @@ -396,6 +406,9 @@ export class ReconnectableHarness this.updateSessionTracking(event.sessionId); this.emit('runtimeInferenceUsage', event); }, + commandError: (error) => { + this.emit('commandError', error); + }, }; this.currentListeners = listeners; @@ -408,6 +421,9 @@ export class ReconnectableHarness harness.on('runtimePersistedEnvelope', listeners.runtimePersistedEnvelope); harness.on('runtimeTurnCompleted', listeners.runtimeTurnCompleted); harness.on('runtimeInferenceUsage', listeners.runtimeInferenceUsage); + listeners.commandErrorUnsubscribe = harness.subscribeCommandError?.( + listeners.commandError, + ); this.requestQueuedCommandFlush(); } @@ -428,6 +444,7 @@ export class ReconnectableHarness ); harness.off('runtimeTurnCompleted', listeners.runtimeTurnCompleted); harness.off('runtimeInferenceUsage', listeners.runtimeInferenceUsage); + listeners.commandErrorUnsubscribe?.(); } if (harness && options?.dispose !== false) { diff --git a/apps/worker/src/sandbox-server/lib/__tests__/harness-manager.test.ts b/apps/worker/src/sandbox-server/lib/__tests__/harness-manager.test.ts index 4246992d..a510d407 100644 --- a/apps/worker/src/sandbox-server/lib/__tests__/harness-manager.test.ts +++ b/apps/worker/src/sandbox-server/lib/__tests__/harness-manager.test.ts @@ -13,6 +13,7 @@ import { type TaskCommand, TaskCommandName, type Harness, + type HarnessCommandError, type HarnessEvents, } from '../harness'; import { @@ -71,6 +72,13 @@ class FakeHarness extends EventEmitter implements Harness { return () => this.off('runtimeTurnCompleted', listener); } + subscribeCommandError( + listener: (error: HarnessCommandError) => void, + ): () => void { + this.on('commandError', listener); + return () => this.off('commandError', listener); + } + sendCommand(command: TaskCommand): boolean { this.sentCommands.push(command); return true; @@ -103,6 +111,10 @@ class FakeHarness extends EventEmitter implements Harness { emitRuntimeOutput(event: AcpMessage): void { this.emit('runtimeOutput', event); } + + emitCommandError(error: HarnessCommandError): void { + this.emit('commandError', error); + } } function createManager( @@ -2726,31 +2738,66 @@ describe('HarnessManager error status', () => { harness.emitTaskEvent({ eventName: TaskEventName.TaskStarted, - payload: ['task-error'], + payload: ['task-error-clear'], } as TaskEvent); harness.emitTaskEvent({ eventName: TaskEventName.Message, payload: [ { - taskId: 'task-error', + taskId: 'task-error-clear', + action: 'created', message: { ts: Date.now(), type: 'say', say: 'error', - text: 'unexpected status 401 Unauthorized: proxy token rejected', - partial: false, + text: 'temporary glitch', }, }, ], } as TaskEvent); expect(manager.getStatus().lastErrorMessage).toBeDefined(); - expect(manager.sendFollowUpPrompt({ prompt: 'retry' })).toBe(true); + manager.sendFollowUpPrompt({ prompt: 'retry' }); expect(manager.getStatus().lastErrorMessage).toBeUndefined(); } finally { manager.dispose(); harness.dispose(); } }); + + it('shuts down terminally when StartNewTask fails asynchronously', async () => { + const { harness, manager } = createManager(); + + try { + manager.initializeWithoutPrompt(); + manager.startNewTaskFromPrompt({ prompt: 'hello' }); + expect(manager.getStatus().phase).toBe('running'); + + harness.emitCommandError({ + command: { + commandName: TaskCommandName.StartNewTask, + data: { text: 'hello' }, + }, + error: new Error( + 'OpenCode session creation did not respond within 90s', + ), + }); + + await expect(manager.waitForShutdown()).resolves.toMatchObject({ + lastErrorMessage: + 'OpenCode session creation did not respond within 90s', + taskAbortedAt: undefined, + taskFinishedAt: undefined, + }); + expect(manager.getStatus()).toMatchObject({ + phase: 'shutting_down', + lastErrorMessage: + 'OpenCode session creation did not respond within 90s', + }); + } finally { + manager.dispose(); + harness.dispose(); + } + }); }); diff --git a/apps/worker/src/sandbox-server/lib/harness-manager.ts b/apps/worker/src/sandbox-server/lib/harness-manager.ts index 9069919c..304eea75 100644 --- a/apps/worker/src/sandbox-server/lib/harness-manager.ts +++ b/apps/worker/src/sandbox-server/lib/harness-manager.ts @@ -183,6 +183,7 @@ export class HarnessManager extends EventEmitter { private sleepHeartbeatTimer: NodeJS.Timeout | null = null; private unsubscribeEvents: (() => void) | null = null; private unsubscribeRuntimeOutput: (() => void) | null = null; + private unsubscribeCommandError: (() => void) | null = null; private shutdownResolve: ((state: TaskState) => void) | null = null; private shutdownPromise: Promise; private runtimeQueuedMessagesCount = 0; @@ -765,6 +766,8 @@ export class HarnessManager extends EventEmitter { this.unsubscribeEvents = null; this.unsubscribeRuntimeOutput?.(); this.unsubscribeRuntimeOutput = null; + this.unsubscribeCommandError?.(); + this.unsubscribeCommandError = null; } // --------------------------------------------------------------------------- @@ -953,6 +956,12 @@ export class HarnessManager extends EventEmitter { }, ); + this.unsubscribeCommandError = this.harness.subscribeCommandError?.( + ({ command, error }) => { + this.handleCommandError(command.commandName, error); + }, + ); + this.harness.on('connected', () => { if (this.state.clientDisconnectedAt) { this.logger.info( @@ -1216,6 +1225,33 @@ export class HarnessManager extends EventEmitter { this.state.lastMessageAt = Date.now(); } + /** + * Fail closed on async StartNewTask failures (e.g. OpenCode session create + * timeout). TaskAborted is intentionally not used: it is resumable and + * resolveStatus maps it to Canceled. Terminal Failed is lastErrorMessage + * without abort/finish stamps, then immediate shutdown. + */ + private handleCommandError(commandName: string, error: unknown): void { + if (commandName !== HarnessCommand.StartNewTask) { + this.logger.error( + `[HarnessManager] Background command failed command=${commandName} error=${formatCallbackError(error)}`, + ); + return; + } + + if (this.phase === 'shutting_down' || this.state.cancelTriggeredAt) { + return; + } + + const message = formatCallbackError(error); + this.state.lastErrorMessage = message; + this.logger.error( + `[HarnessManager] StartNewTask failed; shutting down terminally error=${message}`, + ); + this.emit('stateChange', this.phase, this.state); + this.triggerShutdown(); + } + private onTaskStarted(payload: TaskEventStartedPayload): void { const taskId = payload[0]; diff --git a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts index 7525378f..aed58a3a 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts @@ -3384,7 +3384,7 @@ describe('OpenCodeServerHarness', () => { } }); - it('aborts the initial task and records a diagnostic when session create fails', async () => { + it('fails the initial task and records a diagnostic when session create fails', async () => { const client = new FakeOpenCodeServerClient(); client.createSession.mockRejectedValueOnce( new Error( @@ -3404,11 +3404,13 @@ describe('OpenCodeServerHarness', () => { }); const taskEvents: TaskEvent[] = []; const persistedEnvelopes: AcpPersistedEnvelope[] = []; + const commandErrors: unknown[] = []; harness.subscribe((event) => taskEvents.push(event)); harness.subscribeRuntimePersistedEnvelope((envelope) => persistedEnvelopes.push(envelope), ); + harness.subscribeCommandError?.((error) => commandErrors.push(error)); try { await connectHarness(harness, client); @@ -3419,11 +3421,7 @@ describe('OpenCodeServerHarness', () => { }); await vi.waitFor(() => { - expect( - taskEvents.some( - (event) => event.eventName === TaskEventName.TaskAborted, - ), - ).toBe(true); + expect(commandErrors.length).toBeGreaterThan(0); }); expect( @@ -3433,6 +3431,19 @@ describe('OpenCodeServerHarness', () => { event.payload?.[0] === 'opencode-session-create-failed', ), ).toBe(true); + expect( + taskEvents.some( + (event) => event.eventName === TaskEventName.TaskAborted, + ), + ).toBe(false); + expect( + taskEvents.some( + (event) => + event.eventName === TaskEventName.Message && + (event.payload?.[0] as { message?: { say?: string } })?.message + ?.say === 'error', + ), + ).toBe(true); expect(client.promptAsync).not.toHaveBeenCalled(); expect(onDiagnostic).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts index e1a9771f..a63a4ddb 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts @@ -14,6 +14,7 @@ import { parseAcpFlattenedMcpToolName, OPENCODE_ARCHITECT_AGENT, OPENCODE_BUILD_AGENT, + TaskEventName, } from '@roomote/types'; import type { AcpMessage, @@ -3037,10 +3038,10 @@ export class OpenCodeServerHarness } /** - * Leave a terminal, retryable failure when the very first session never - * materializes. Without this the harness manager stays in `running` forever - * (commandError is currently unsubscribed) and `first_assistant_output_at` - * never stamps. + * Leave a transcript-visible failure for the initial prompt when the first + * OpenCode session never materializes. Do not emit TaskAborted — that path is + * resumable and resolveStatus maps it to Canceled. The surrounding + * `commandError` + HarnessManager handler force a terminal Failed shutdown. */ private failSessionCreateForInitialTask(error: unknown): void { if (this.sessionId) { @@ -3055,14 +3056,28 @@ export class OpenCodeServerHarness : `OpenCode session creation failed before the agent could start.\n\n${message}\n\n${formatOpenCodeSessionCreateTimeoutText(timeoutMs)}`; this.logger.error( - `OpenCode initial session create failed; aborting task so it does not hang in running forever error=${message}`, + `OpenCode initial session create failed; failing the task terminally error=${message}`, ); this.runtimeEvents.taskStarted(sessionId); this.runtimeEvents.assistantMessage({ sessionId, text: userText, }); - this.runtimeEvents.taskAborted(sessionId); + this.emit('taskEvent', { + eventName: TaskEventName.Message, + payload: [ + { + taskId: sessionId, + action: 'created', + message: { + ts: Date.now(), + type: 'say', + say: 'error', + text: userText, + }, + }, + ], + }); } private async validateResumedSession(sessionId: string): Promise { From 5689e9ed4eb54a8f2fa49b26788a39e4bc7103e9 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:35:46 +0000 Subject: [PATCH 4/7] [Fix] Assign null when commandError subscription is absent --- apps/worker/src/sandbox-server/lib/harness-manager.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/worker/src/sandbox-server/lib/harness-manager.ts b/apps/worker/src/sandbox-server/lib/harness-manager.ts index 304eea75..c8094d7d 100644 --- a/apps/worker/src/sandbox-server/lib/harness-manager.ts +++ b/apps/worker/src/sandbox-server/lib/harness-manager.ts @@ -956,11 +956,10 @@ export class HarnessManager extends EventEmitter { }, ); - this.unsubscribeCommandError = this.harness.subscribeCommandError?.( - ({ command, error }) => { + this.unsubscribeCommandError = + this.harness.subscribeCommandError?.(({ command, error }) => { this.handleCommandError(command.commandName, error); - }, - ); + }) ?? null; this.harness.on('connected', () => { if (this.state.clientDisconnectedAt) { From e651208bac283eff99e2c5a086b8bed39a0d1241 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:41:01 +0000 Subject: [PATCH 5/7] [Fix] Shut down after cancel races with session-create failure If the user cancels before OpenCode invents a session, CancelTask is a no-op and a later StartNewTask timeout must still force sandbox shutdown. --- .../lib/__tests__/harness-manager.test.ts | 38 +++++++++++++++++++ .../src/sandbox-server/lib/harness-manager.ts | 29 ++++++++++++-- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/apps/worker/src/sandbox-server/lib/__tests__/harness-manager.test.ts b/apps/worker/src/sandbox-server/lib/__tests__/harness-manager.test.ts index a510d407..14eafdb1 100644 --- a/apps/worker/src/sandbox-server/lib/__tests__/harness-manager.test.ts +++ b/apps/worker/src/sandbox-server/lib/__tests__/harness-manager.test.ts @@ -2800,4 +2800,42 @@ describe('HarnessManager error status', () => { harness.dispose(); } }); + + it('still shuts down when StartNewTask fails after cancel before a session exists', async () => { + const { harness, manager } = createManager(); + + try { + manager.initializeWithoutPrompt(); + manager.startNewTaskFromPrompt({ prompt: 'hello' }); + expect(manager.getStatus().phase).toBe('running'); + expect(manager.getStatus().sessionId).toBeUndefined(); + + // Cancel before OpenCode invents a session id — CancelTask is a no-op on + // the runtime side because there is nothing to abort yet. + manager.cancelTask(); + expect(manager.getStatus().phase).toBe('stopped'); + expect(manager.getSleepAt()).toBeNull(); + expect(manager.getState().cancelTriggeredAt).toBeDefined(); + + harness.emitCommandError({ + command: { + commandName: TaskCommandName.StartNewTask, + data: { text: 'hello' }, + }, + error: new Error( + 'OpenCode session creation did not respond within 90s', + ), + }); + + const finalState = await manager.waitForShutdown(); + expect(finalState.taskAbortedAt).toBe(finalState.cancelTriggeredAt); + expect(finalState.taskFinishedAt).toBeUndefined(); + expect(manager.getStatus().phase).toBe('shutting_down'); + // User cancel wins for report status; do not rewrite it as Failed. + expect(manager.getStatus().lastErrorMessage).toBeUndefined(); + } finally { + manager.dispose(); + harness.dispose(); + } + }); }); diff --git a/apps/worker/src/sandbox-server/lib/harness-manager.ts b/apps/worker/src/sandbox-server/lib/harness-manager.ts index c8094d7d..7f57da89 100644 --- a/apps/worker/src/sandbox-server/lib/harness-manager.ts +++ b/apps/worker/src/sandbox-server/lib/harness-manager.ts @@ -1226,9 +1226,15 @@ export class HarnessManager extends EventEmitter { /** * Fail closed on async StartNewTask failures (e.g. OpenCode session create - * timeout). TaskAborted is intentionally not used: it is resumable and - * resolveStatus maps it to Canceled. Terminal Failed is lastErrorMessage - * without abort/finish stamps, then immediate shutdown. + * timeout). TaskAborted is intentionally not used for the plain failure + * path: it is resumable and resolveStatus maps it to Canceled. Terminal + * Failed is lastErrorMessage without abort/finish stamps, then immediate + * shutdown. + * + * If the user already canceled before a session existed, CancelTask is + * effectively a no-op and leaves phase `stopped` with getSleepAt() null — + * do not silent-ignore the later StartNewTask settle; still shut down so + * the sandbox cannot linger forever. */ private handleCommandError(commandName: string, error: unknown): void { if (commandName !== HarnessCommand.StartNewTask) { @@ -1238,11 +1244,26 @@ export class HarnessManager extends EventEmitter { return; } - if (this.phase === 'shutting_down' || this.state.cancelTriggeredAt) { + if (this.phase === 'shutting_down') { return; } const message = formatCallbackError(error); + + if (this.state.cancelTriggeredAt) { + if (!this.state.taskAbortedAt) { + // Preserve cancel semantics for resolveStatus while closing the + // session-create race where CancelTask had nothing to abort yet. + this.state.taskAbortedAt = this.state.cancelTriggeredAt; + } + this.logger.info( + `[HarnessManager] StartNewTask failed after cancel; completing canceled shutdown error=${message}`, + ); + this.emit('stateChange', this.phase, this.state); + this.triggerShutdown(); + return; + } + this.state.lastErrorMessage = message; this.logger.error( `[HarnessManager] StartNewTask failed; shutting down terminally error=${message}`, From cc06e11bf24108f160412fd3fd4f386f9279f373 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:28:09 +0000 Subject: [PATCH 6/7] [Fix] Abort late sessions when cancel races session create If CancelTask arrives before ensureSession finishes, mark the race and abort any session that still gets created so a stopped manager cannot keep an orphan live turn. --- .../__tests__/opencode-server.test.ts | 68 +++++++++++++ .../lib/harnesses/opencode-server/harness.ts | 97 ++++++++++++++++++- 2 files changed, 164 insertions(+), 1 deletion(-) diff --git a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts index aed58a3a..68f7f72a 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts @@ -3468,6 +3468,74 @@ describe('OpenCodeServerHarness', () => { harness.dispose(); } }); + + it('aborts a late-created session when cancel races ahead of ensureSession', async () => { + let resolveCreate: + | ((value: { id: string; title: string }) => void) + | undefined; + const client = new FakeOpenCodeServerClient(); + client.createSession.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveCreate = resolve; + }), + ); + + const harness = new OpenCodeServerHarness({ + client: client as unknown as OpenCodeServerClient, + workspacePath: '/sandbox/repos', + logger: createLogger(), + model: TEST_OPENCODE_MODEL, + eventStreamReadyTimeoutMs: 100, + }); + const taskEvents: TaskEvent[] = []; + harness.subscribe((event) => taskEvents.push(event)); + + try { + await connectHarness(harness, client); + + harness.sendCommand({ + commandName: TaskCommandName.StartNewTask, + data: { text: 'Start work.', visibleInTranscript: true }, + }); + + await vi.waitFor(() => { + expect(client.createSession).toHaveBeenCalledTimes(1); + }); + + // Cancel while createSession is still pending — no session id yet. + harness.sendCommand({ + commandName: TaskCommandName.CancelTask, + data: { cancelledBy: { name: 'Matt', source: 'web' } }, + }); + + resolveCreate?.({ id: 'ses_late', title: 'late' }); + + await vi.waitFor(() => { + expect(client.abort).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: 'ses_late' }), + ); + }); + + expect(client.promptAsync).not.toHaveBeenCalled(); + expect( + taskEvents.some( + (event) => + event.eventName === TaskEventName.TaskStarted && + event.payload?.[0] === 'ses_late', + ), + ).toBe(true); + expect( + taskEvents.some( + (event) => + event.eventName === TaskEventName.TaskAborted && + event.payload?.[0] === 'ses_late', + ), + ).toBe(true); + } finally { + harness.dispose(); + } + }); }); describe('OpenCodeServerHarness cancel marker', () => { diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts index a63a4ddb..dcf87457 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts @@ -1499,6 +1499,13 @@ export class OpenCodeServerHarness private sessionId: string | undefined; private resumedSessionPendingValidation = false; private inFlight = false; + /** + * Set when CancelTask arrives before any OpenCode session id exists. A + * later-successful createSession must abort that session instead of starting + * a prompt while the HarnessManager has already moved to `stopped`. + */ + private cancelRequestedBeforeSession = false; + private sessionCreateAbortController = new AbortController(); private queuedPromptRetryTimer: ReturnType | null = null; private currentWorkflowPhase: string | null = null; // Most recent packaged workflow skill loaded by the primary session's agent @@ -1993,16 +2000,32 @@ export class OpenCodeServerHarness this.stopHookReminderCount = 0; this.currentWorkflowPhase = command.data.workflowPhase ?? null; this.activeWorkflowSkill = null; + this.cancelRequestedBeforeSession = false; + this.resetSessionCreateAbortController(); let sessionId: string; try { sessionId = await this.ensureSession(command.data.text); } catch (error) { + if (this.cancelRequestedBeforeSession || this.isAbortError(error)) { + this.logger.info( + `OpenCode initial session create aborted because cancel was requested before a session existed error=${ + error instanceof Error ? error.message : String(error) + }`, + ); + return; + } + this.failSessionCreateForInitialTask(error); throw error; } + if (this.cancelRequestedBeforeSession) { + await this.terminateLateCreatedSessionAfterCancel(sessionId); + return; + } + this.runtimeEvents.taskStarted(sessionId); this.runtimeEvents.userPrompt({ sessionId, @@ -2101,6 +2124,11 @@ export class OpenCodeServerHarness const sessionId = this.sessionId; if (!sessionId) { + // Cancel raced ahead of ensureSession. Remember the request so a late + // successful create is aborted instead of submitting the initial prompt + // while the manager has already moved to stopped. + this.cancelRequestedBeforeSession = true; + this.sessionCreateAbortController.abort(); this.inFlight = false; this.prompts.clear(); this.clearQueuedPromptRetryTimer(); @@ -2988,7 +3016,7 @@ export class OpenCodeServerHarness try { const session = await this.client.createSession({ title: title?.slice(0, 80), - signal: this.eventAbortController.signal, + signal: this.composeSessionCreateSignal(), }); this.sessionId = session.id; this.logger.info( @@ -3001,6 +3029,13 @@ export class OpenCodeServerHarness const elapsedMs = Date.now() - startedAt; const message = error instanceof Error ? error.message : String(error); + if (this.cancelRequestedBeforeSession || this.isAbortError(error)) { + this.logger.info( + `OpenCode session create canceled/aborted elapsedMs=${elapsedMs} error=${message}`, + ); + throw error; + } + this.logger.error( `OpenCode session create failed elapsedMs=${elapsedMs} timeoutMs=${timeoutMs} workspace=${this.workspacePath} mcpServers=${ this.knownMcpServerNames.length > 0 @@ -3017,6 +3052,66 @@ export class OpenCodeServerHarness } } + private composeSessionCreateSignal(): AbortSignal { + return AbortSignal.any([ + this.eventAbortController.signal, + this.sessionCreateAbortController.signal, + ]); + } + + private resetSessionCreateAbortController(): void { + if (!this.sessionCreateAbortController.signal.aborted) { + return; + } + + this.sessionCreateAbortController = new AbortController(); + } + + private isAbortError(error: unknown): boolean { + if (!(error instanceof Error)) { + return false; + } + + return error.name === 'AbortError' || /aborted/i.test(error.message); + } + + /** + * Cancel raced with ensureSession and arrived first. The create still + * succeeded — stop the late session before it becomes an orphan turn while + * the manager is already stopped. + */ + private async terminateLateCreatedSessionAfterCancel( + sessionId: string, + ): Promise { + this.logger.warn( + `OpenCode session ${sessionId} was created after cancel was requested with no prior session; aborting the late session instead of starting the turn`, + ); + this.sessionId = sessionId; + this.cancelRequestedBeforeSession = false; + this.armReplayAbortErrorSuppression(); + + try { + await this.client.abort({ + sessionId, + signal: this.eventAbortController.signal, + }); + } catch (error) { + this.logger.warn( + `Failed to abort late OpenCode session ${sessionId} after racey cancel: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + + this.inFlight = false; + this.prompts.clear(); + this.clearQueuedPromptRetryTimer(); + this.pendingUserInputRequests.clear(); + this.clearAllExecuteToolProgress(); + this.runtimeEvents.taskStarted(sessionId); + this.runtimeEvents.taskAborted(sessionId); + } + private recordSessionCreateFailureDiagnostic(input: { message: string; elapsedMs: number; From 3d85cca4540f32d35f63f3a012d3202fca29adcd Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:35:33 +0000 Subject: [PATCH 7/7] [Improve] Point session-create hang errors at harness.log in Logs Make the OpenCode session-create timeout copy tell users to open the Logs sidebar and inspect harness.log instead of a vague worker logs reference. --- .../lib/harnesses/__tests__/opencode-server-client.test.ts | 7 +++++++ .../sandbox-server/lib/harnesses/opencode-server/client.ts | 2 +- .../lib/harnesses/opencode-server/harness.ts | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-client.test.ts b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-client.test.ts index 74f295a5..c06b065c 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-client.test.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server-client.test.ts @@ -42,6 +42,13 @@ describe('OpenCodeServerClient timeouts', () => { ); }); + it('points session-create timeout copy at the Logs sidebar harness.log', () => { + const text = formatOpenCodeSessionCreateTimeoutText(90_000); + expect(text).toContain('Logs sidebar'); + expect(text).toContain('harness.log'); + expect(text).toContain('[opencode-server]'); + }); + it('times out createSession when OpenCode never responds', async () => { vi.useFakeTimers(); const logger = createLogger(); diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/client.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/client.ts index 7aa1ae25..87115178 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/client.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/client.ts @@ -90,7 +90,7 @@ export function formatOpenCodeSessionCreateTimeoutText( ): string { const seconds = Math.max(1, Math.round(timeoutMs / 1_000)); - return `OpenCode session creation did not respond within ${seconds}s. The OpenCode server is up, but the first session request never finished — usually a hang while OpenCode bootstraps the workspace instance (config, plugins, storage, or MCP). Cancel and retry, or inspect the worker OpenCode logs for a stuck \`creating instance\` / \`bootstrapping\` sequence.`; + return `OpenCode session creation did not respond within ${seconds}s. The OpenCode server is up, but the first session request never finished — usually a hang while OpenCode bootstraps the workspace instance (config, plugins, storage, or MCP). Open the Logs sidebar and inspect harness.log for OpenCode lines (prefixed [opencode-server], including stuck \`creating instance\` / \`bootstrapping\` output), then cancel and retry.`; } function composeAbortSignals( diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts index dcf87457..47a51ca2 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts @@ -3148,7 +3148,7 @@ export class OpenCodeServerHarness const timeoutMs = this.client.sessionCreateTimeoutMsValue; const userText = message.includes('did not respond within') ? message - : `OpenCode session creation failed before the agent could start.\n\n${message}\n\n${formatOpenCodeSessionCreateTimeoutText(timeoutMs)}`; + : `OpenCode session creation failed before the agent could start.\n\n${message}\n\nOpen the Logs sidebar and inspect harness.log for OpenCode lines (prefixed [opencode-server]).\n\n${formatOpenCodeSessionCreateTimeoutText(timeoutMs)}`; this.logger.error( `OpenCode initial session create failed; failing the task terminally error=${message}`,