Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions apps/worker/src/run-task/create-harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,9 @@ export async function createHarness({
return await startOpenCodeServerHarness({
...commonOptions,
developerInstructionsContent,
onDiagnostic: (input) => {
diagnosticEvents.record(input);
},
onUnexpectedExit: (certificate) => {
const summary = `OpenCode server exited unexpectedly (code=${
certificate.exitCode ?? 'none'
Expand Down
17 changes: 17 additions & 0 deletions apps/worker/src/run-task/reconnectable-harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import type {
HarnessPendingUserInputRequest,
HarnessRestartRequest,
HarnessQueuedMessage,
HarnessCommandError,
QueuedPromptMessageSnapshot,
TaskCommand,
} from '../sandbox-server/lib/harness';
Expand Down Expand Up @@ -58,6 +59,8 @@ interface BoundHarnessListeners {
runtimePersistedEnvelope: (envelope: AcpPersistedEnvelope) => void;
runtimeTurnCompleted: (event: AcpTurnCompletedEvent) => void;
runtimeInferenceUsage: (event: HarnessInferenceUsageEvent) => void;
commandError: (error: HarnessCommandError) => void;
commandErrorUnsubscribe?: () => void;
}

const DEFAULT_MAX_RECONNECT_ATTEMPTS = 7;
Expand Down Expand Up @@ -174,6 +177,13 @@ export class ReconnectableHarness
return () => this.off('runtimeInferenceUsage', listener);
}

subscribeCommandError(
listener: (error: HarnessCommandError) => void,
): () => void {
this.on('commandError', listener);
return () => this.off('commandError', listener);
}

sendCommand(command: TaskCommand): boolean {
this.updateSessionTrackingForCommand(command);
this.updateReconnectReplayStateForCommand(command);
Expand Down Expand Up @@ -396,6 +406,9 @@ export class ReconnectableHarness
this.updateSessionTracking(event.sessionId);
this.emit('runtimeInferenceUsage', event);
},
commandError: (error) => {
this.emit('commandError', error);
},
};

this.currentListeners = listeners;
Expand All @@ -408,6 +421,9 @@ export class ReconnectableHarness
harness.on('runtimePersistedEnvelope', listeners.runtimePersistedEnvelope);
harness.on('runtimeTurnCompleted', listeners.runtimeTurnCompleted);
harness.on('runtimeInferenceUsage', listeners.runtimeInferenceUsage);
listeners.commandErrorUnsubscribe = harness.subscribeCommandError?.(
listeners.commandError,
);

this.requestQueuedCommandFlush();
}
Expand All @@ -428,6 +444,7 @@ export class ReconnectableHarness
);
harness.off('runtimeTurnCompleted', listeners.runtimeTurnCompleted);
harness.off('runtimeInferenceUsage', listeners.runtimeInferenceUsage);
listeners.commandErrorUnsubscribe?.();
}

if (harness && options?.dispose !== false) {
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

56 changes: 56 additions & 0 deletions apps/worker/src/sandbox-server/lib/harness-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,7 @@ export class HarnessManager extends EventEmitter<HarnessManagerEvents> {
private sleepHeartbeatTimer: NodeJS.Timeout | null = null;
private unsubscribeEvents: (() => void) | null = null;
private unsubscribeRuntimeOutput: (() => void) | null = null;
private unsubscribeCommandError: (() => void) | null = null;
private shutdownResolve: ((state: TaskState) => void) | null = null;
private shutdownPromise: Promise<TaskState>;
private runtimeQueuedMessagesCount = 0;
Expand Down Expand Up @@ -765,6 +766,8 @@ export class HarnessManager extends EventEmitter<HarnessManagerEvents> {
this.unsubscribeEvents = null;
this.unsubscribeRuntimeOutput?.();
this.unsubscribeRuntimeOutput = null;
this.unsubscribeCommandError?.();
this.unsubscribeCommandError = null;
}

// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -953,6 +956,11 @@ export class HarnessManager extends EventEmitter<HarnessManagerEvents> {
},
);

this.unsubscribeCommandError =
this.harness.subscribeCommandError?.(({ command, error }) => {
this.handleCommandError(command.commandName, error);
}) ?? null;

this.harness.on('connected', () => {
if (this.state.clientDisconnectedAt) {
this.logger.info(
Expand Down Expand Up @@ -1216,6 +1224,54 @@ export class HarnessManager extends EventEmitter<HarnessManagerEvents> {
this.state.lastMessageAt = Date.now();
}

/**
* Fail closed on async StartNewTask failures (e.g. OpenCode session create
* timeout). TaskAborted is intentionally not used for the plain failure
* path: it is resumable and resolveStatus maps it to Canceled. Terminal
* Failed is lastErrorMessage without abort/finish stamps, then immediate
* shutdown.
*
* If the user already canceled before a session existed, CancelTask is
* effectively a no-op and leaves phase `stopped` with getSleepAt() null —
* do not silent-ignore the later StartNewTask settle; still shut down so
* the sandbox cannot linger forever.
*/
private handleCommandError(commandName: string, error: unknown): void {
if (commandName !== HarnessCommand.StartNewTask) {
this.logger.error(
`[HarnessManager] Background command failed command=${commandName} error=${formatCallbackError(error)}`,
);
return;
}

if (this.phase === 'shutting_down') {
return;
}

const message = formatCallbackError(error);

if (this.state.cancelTriggeredAt) {
if (!this.state.taskAbortedAt) {
// Preserve cancel semantics for resolveStatus while closing the
// session-create race where CancelTask had nothing to abort yet.
this.state.taskAbortedAt = this.state.cancelTriggeredAt;
}
this.logger.info(
`[HarnessManager] StartNewTask failed after cancel; completing canceled shutdown error=${message}`,
);
this.emit('stateChange', this.phase, this.state);
this.triggerShutdown();
return;
}

this.state.lastErrorMessage = message;
this.logger.error(
`[HarnessManager] StartNewTask failed; shutting down terminally error=${message}`,
);
this.emit('stateChange', this.phase, this.state);
this.triggerShutdown();
}

private onTaskStarted(payload: TaskEventStartedPayload): void {
const taskId = payload[0];

Expand Down
Loading
Loading