From 9a5a0787537e74ee114e62d4439f29d6194ce0dc Mon Sep 17 00:00:00 2001 From: BingZ Date: Sat, 11 Jul 2026 21:29:06 +0800 Subject: [PATCH 01/36] fix(windows): serialize worktree terminal teardown --- src/main/runtime/orca-runtime.test.ts | 37 ++++++++++++++++++++++++--- src/main/runtime/orca-runtime.ts | 9 ++++++- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index b8d9eee175f..74a2d7a30ef 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -30,10 +30,7 @@ import { removeWorktree } from '../git/worktree' import * as gitRunner from '../git/runner' -import { - clearSubmodulePathsCacheForTests, - listSubmodulePaths -} from '../git/status' +import { clearSubmodulePathsCacheForTests, listSubmodulePaths } from '../git/status' import { createSetupRunnerScript, getEffectiveHooks, @@ -21845,6 +21842,38 @@ describe('OrcaRuntimeService', () => { expect(killed).toBe(false) }) + it('waits for verified terminal stops before completing a worktree stop', async () => { + const runtime = new OrcaRuntimeService(store) + let releaseStop = () => {} + const stopAndWait = vi.fn( + () => + new Promise((resolve) => { + releaseStop = () => resolve(true) + }) + ) + const kill = vi.fn(() => true) + runtime.setPtyController({ + write: () => true, + kill, + stopAndWait, + getForegroundProcess: async () => null + }) + syncSinglePty(runtime, 'pty-1') + + let completed = false + const stopping = runtime.stopTerminalsForWorktree(TEST_WORKTREE_ID).then((result) => { + completed = true + return result + }) + await vi.waitFor(() => expect(stopAndWait).toHaveBeenCalledWith('pty-1')) + + expect(completed).toBe(false) + expect(kill).not.toHaveBeenCalled() + releaseStop() + + await expect(stopping).resolves.toEqual({ stopped: 1 }) + }) + it('stops exactly the expected live PTYs for a worktree', async () => { const runtime = new OrcaRuntimeService(store) const stopped: string[] = [] diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 84f1954d194..520ccf167b3 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -18676,7 +18676,14 @@ export class OrcaRuntimeService { let stopped = 0 for (const ptyId of ptyIds) { - if (this.ptyController?.kill(ptyId)) { + // Why: worktree removal immediately follows this with provider/registry + // sweeps. Await the verified stop so those sweeps cannot overlap a + // graceful kill with a second ConPTY teardown on Windows (#8275). + if (this.ptyController?.stopAndWait) { + if (await this.ptyController.stopAndWait(ptyId)) { + stopped += 1 + } + } else if (this.ptyController?.kill(ptyId)) { stopped += 1 } } From ee5b79356419fbb9d2e4a5a0cb975bb631175120 Mon Sep 17 00:00:00 2001 From: Brennan Benson Date: Sun, 12 Jul 2026 11:46:00 -0700 Subject: [PATCH 02/36] test(runtime): cover sequential worktree teardown isolation --- src/main/runtime/orca-runtime.ts | 2 +- src/main/runtime/worktree-teardown.test.ts | 49 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 51d1ae91098..fb41f053da8 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -18768,7 +18768,7 @@ export class OrcaRuntimeService { for (const ptyId of ptyIds) { // Why: worktree removal immediately follows this with provider/registry // sweeps. Await the verified stop so those sweeps cannot overlap a - // graceful kill with a second ConPTY teardown on Windows (#8275). + // provider shutdown with a second ConPTY teardown on Windows (#8275). if (this.ptyController?.stopAndWait) { if (await this.ptyController.stopAndWait(ptyId)) { stopped += 1 diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index 63e207cd8da..e4caf2d6b3a 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -161,6 +161,55 @@ describe('killAllProcessesForWorktree', () => { expect(result.runtimeStopped).toBe(3) }) + it('awaits each runtime stop before fallback sweeps and preserves unrelated sessions', async () => { + const releases = new Map void>() + const liveSessions = new Set(['w1@@target', 'w2@@target', 'witness@@unrelated']) + const stopTerminalsForWorktree = vi.fn( + (worktreeId: string) => + new Promise<{ stopped: number }>((resolve) => { + releases.set(worktreeId, () => { + liveSessions.delete(`${worktreeId}@@target`) + // Why: a spawn that finishes after the runtime snapshot still has + // to be caught by the later provider sweep before deletion. + liveSessions.add(`${worktreeId}@@late`) + resolve({ stopped: 1 }) + }) + }) + ) + const runtime = { + stopTerminalsForWorktree + } as unknown as Parameters[1]['runtime'] + const localProvider = createProviderStub(async () => + [...liveSessions].map((id) => ({ id, cwd: '/tmp', title: 'shell' })) + ) + listRegisteredPtysMock.mockReturnValue([]) + + for (const worktreeId of ['w1', 'w2']) { + const listCountBeforeStop = vi.mocked(localProvider.listProcesses).mock.calls.length + const shutdownCountBeforeStop = vi.mocked(localProvider.shutdown).mock.calls.length + const removal = killAllProcessesForWorktree(worktreeId, { runtime, localProvider }) + await vi.waitFor(() => expect(releases.has(worktreeId)).toBe(true)) + + expect(localProvider.listProcesses).toHaveBeenCalledTimes(listCountBeforeStop) + expect(localProvider.shutdown).toHaveBeenCalledTimes(shutdownCountBeforeStop) + releases.get(worktreeId)?.() + + await expect(removal).resolves.toEqual({ + runtimeStopped: 1, + providerStopped: 1, + registryStopped: 0 + }) + expect(localProvider.shutdown).toHaveBeenCalledWith(`${worktreeId}@@late`, { + immediate: true + }) + } + + expect(localProvider.shutdown).not.toHaveBeenCalledWith('witness@@unrelated', { + immediate: true + }) + expect(liveSessions.has('witness@@unrelated')).toBe(true) + }) + it('tolerates runtime.stopTerminalsForWorktree throwing (headless assertGraphReady reject)', async () => { const stopTerminalsForWorktree = vi.fn().mockRejectedValue(new Error('graph not ready')) const runtime = { From 525c190441e5282a7b2255191d3ce17d280ba541 Mon Sep 17 00:00:00 2001 From: Brennan Benson Date: Sun, 12 Jul 2026 11:49:23 -0700 Subject: [PATCH 03/36] perf(pty): use targeted liveness for verified stops --- src/main/ipc/pty.test.ts | 7 ++++++- src/main/ipc/pty.ts | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index 35d386eb473..db12915f32f 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -2700,6 +2700,8 @@ describe('registerPtyHandlers', () => { it('controller stopAndWait skips the synthetic exit when the provider emitted one', async () => { vi.useFakeTimers() const exitListeners = new Set<(payload: { id: string; code: number }) => void>() + const hasPty = vi.fn(() => false) + const listProcesses = vi.fn(async () => []) const shutdown = vi.fn(async (id: string) => { for (const listener of exitListeners) { listener({ id, code: 0 }) @@ -2711,6 +2713,7 @@ describe('registerPtyHandlers', () => { } setLocalPtyProvider({ spawn: vi.fn(), + hasPty, write: vi.fn(), resize: vi.fn(), shutdown, @@ -2729,7 +2732,7 @@ describe('registerPtyHandlers', () => { exitListeners.add(listener) return () => exitListeners.delete(listener) }), - listProcesses: vi.fn(async () => []), + listProcesses, attach: vi.fn(), getDefaultShell: vi.fn(), getProfiles: vi.fn() @@ -2744,6 +2747,8 @@ describe('registerPtyHandlers', () => { await vi.advanceTimersByTimeAsync(1_200) await expect(stopPromise).resolves.toBe(true) + expect(hasPty).toHaveBeenCalledWith('local-pty') + expect(listProcesses).not.toHaveBeenCalled() expect(runtime.onPtyExit).toHaveBeenCalledTimes(1) expect(runtime.onPtyExit).toHaveBeenCalledWith('local-pty', 0) expect( diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 305ebabfc14..7ec7afcd90c 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -490,6 +490,11 @@ function delay(ms: number): Promise { } async function isProviderPtyLive(provider: IPtyProvider, ptyId: string): Promise { + // Why: worktree teardown verifies each stopped PTY. Local/daemon providers + // can answer by ID, avoiding one full session inventory per split pane. + if (provider.hasPty) { + return provider.hasPty(ptyId) + } return (await provider.listProcesses()).some((session) => session.id === ptyId) } From 70da22d102d71a7c3788beddd756ce8f8f1fe3dd Mon Sep 17 00:00:00 2001 From: Brennan Benson Date: Sun, 12 Jul 2026 12:14:21 -0700 Subject: [PATCH 04/36] perf(ssh): target PTY liveness checks --- src/main/ipc/pty.test.ts | 51 ++++++++ src/main/ipc/pty.ts | 3 + src/main/providers/ssh-pty-liveness.ts | 123 ++++++++++++++++++++ src/main/providers/ssh-pty-provider.test.ts | 104 +++++++++++++++++ src/main/providers/ssh-pty-provider.ts | 27 ++++- src/main/providers/types.ts | 2 + src/relay/pty-handler.test.ts | 8 ++ src/relay/pty-handler.ts | 6 + 8 files changed, 321 insertions(+), 3 deletions(-) create mode 100644 src/main/providers/ssh-pty-liveness.ts diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index db12915f32f..dc3c9be4553 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -2756,6 +2756,57 @@ describe('registerPtyHandlers', () => { ).toEqual([['pty:exit', { id: 'local-pty', code: 0 }]]) }) + it('controller stopAndWait uses targeted SSH liveness for multiple PTYs', async () => { + const hasPtyAsync = vi.fn(async (_ptyId: string) => false) + const listProcesses = vi.fn(async () => []) + const shutdown = vi.fn(async () => undefined) + const runtime = { + setPtyController: vi.fn(), + onPtyExit: vi.fn() + } + registerSshPtyProvider('ssh-1', { + spawn: vi.fn(), + write: vi.fn(), + resize: vi.fn(), + shutdown, + sendSignal: vi.fn(), + getCwd: vi.fn(), + getInitialCwd: vi.fn(), + clearBuffer: vi.fn(), + acknowledgeDataEvent: vi.fn(), + hasChildProcesses: vi.fn(), + hasPtyAsync, + getForegroundProcess: vi.fn(), + serialize: vi.fn(), + revive: vi.fn(), + onData: vi.fn(() => () => {}), + onReplay: vi.fn(() => () => {}), + onExit: vi.fn(() => () => {}), + listProcesses, + attach: vi.fn(), + getDefaultShell: vi.fn(), + getProfiles: vi.fn() + } as never) + const ptyIds = ['ssh:ssh-1@@relay-1', 'ssh:ssh-1@@relay-2', 'ssh:ssh-1@@relay-3'] + for (const ptyId of ptyIds) { + setPtyOwnership(ptyId, 'ssh-1') + } + handlers.clear() + registerPtyHandlers(mainWindow as never, runtime as never) + const controller = runtime.setPtyController.mock.calls[0]?.[0] as { + stopAndWait: (ptyId: string) => Promise + } + + await expect( + Promise.all(ptyIds.map((ptyId) => controller.stopAndWait(ptyId))) + ).resolves.toEqual([true, true, true]) + + expect(shutdown).toHaveBeenCalledTimes(ptyIds.length) + expect(hasPtyAsync).toHaveBeenCalledTimes(ptyIds.length) + expect(hasPtyAsync.mock.calls.map(([ptyId]) => ptyId)).toEqual(ptyIds) + expect(listProcesses).not.toHaveBeenCalled() + }) + it('passes keepHistory through runtime controller stopAndWait', async () => { vi.useFakeTimers() const shutdown = vi.fn(async () => undefined) diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 7ec7afcd90c..e64ab5de63a 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -495,6 +495,9 @@ async function isProviderPtyLive(provider: IPtyProvider, ptyId: string): Promise if (provider.hasPty) { return provider.hasPty(ptyId) } + if (provider.hasPtyAsync) { + return provider.hasPtyAsync(ptyId) + } return (await provider.listProcesses()).some((session) => session.id === ptyId) } diff --git a/src/main/providers/ssh-pty-liveness.ts b/src/main/providers/ssh-pty-liveness.ts new file mode 100644 index 00000000000..0d40f63e4fa --- /dev/null +++ b/src/main/providers/ssh-pty-liveness.ts @@ -0,0 +1,123 @@ +import { JsonRpcErrorCode } from '../ssh/relay-protocol' + +type SshPtyLivenessOptions = { + probe: (id: string) => Promise + listIds: () => Promise +} + +export class SshPtyLiveness { + private supportsTargetedProbe: boolean | undefined + private targetedProbeInFlight: Promise | undefined + private legacyIds: Set | undefined + private legacyInventoryInFlight: Promise> | undefined + private legacyInventoryGeneration = 0 + private legacyMembershipOverrides = new Map() + + constructor(private readonly options: SshPtyLivenessOptions) {} + + dispose(): void { + this.targetedProbeInFlight = undefined + this.legacyInventoryGeneration += 1 + this.legacyIds = undefined + this.legacyInventoryInFlight = undefined + this.legacyMembershipOverrides.clear() + } + + markLive(id: string): void { + this.updateLegacyMembership(id, true) + } + + markStopped(id: string): void { + this.updateLegacyMembership(id, false) + } + + async hasPty(id: string): Promise { + if (this.supportsTargetedProbe === false) { + return (await this.getLegacyIds()).has(id) + } + if (this.supportsTargetedProbe === true) { + return this.options.probe(id) + } + if (this.targetedProbeInFlight) { + await this.targetedProbeInFlight + return this.hasPty(id) + } + const probe = this.probeCapability(id) + this.targetedProbeInFlight = probe + try { + return await probe + } finally { + if (this.targetedProbeInFlight === probe) { + this.targetedProbeInFlight = undefined + } + } + } + + private async probeCapability(id: string): Promise { + try { + const live = await this.options.probe(id) + this.supportsTargetedProbe = true + this.legacyIds = undefined + return live + } catch (error) { + if ( + !error || + typeof error !== 'object' || + (error as { code?: unknown }).code !== JsonRpcErrorCode.MethodNotFound + ) { + throw error + } + // Why: a new desktop may reconnect to a preserved older relay. Cache the + // narrow protocol miss and one inventory for the multi-PTY stop sequence. + this.supportsTargetedProbe = false + return (await this.getLegacyIds()).has(id) + } + } + + private updateLegacyMembership(id: string, live: boolean): void { + if (this.legacyIds) { + if (live) { + this.legacyIds.add(id) + } else { + this.legacyIds.delete(id) + } + } else if (this.supportsTargetedProbe === false) { + // Why: lifecycle changes can settle while the legacy inventory RPC is + // in flight; replay them onto its result before any liveness read. + this.legacyMembershipOverrides.set(id, live) + } + } + + private async getLegacyIds(): Promise> { + if (this.legacyIds) { + return this.legacyIds + } + if (this.legacyInventoryInFlight) { + return this.legacyInventoryInFlight + } + const generation = this.legacyInventoryGeneration + const inventory = this.options.listIds().then((ids) => { + const liveIds = new Set(ids) + if (this.legacyInventoryGeneration === generation) { + for (const [id, live] of this.legacyMembershipOverrides) { + if (live) { + liveIds.add(id) + } else { + liveIds.delete(id) + } + } + this.legacyMembershipOverrides.clear() + this.legacyIds = liveIds + } + return liveIds + }) + this.legacyInventoryInFlight = inventory + try { + return await inventory + } finally { + if (this.legacyInventoryInFlight === inventory) { + this.legacyInventoryInFlight = undefined + } + } + } +} diff --git a/src/main/providers/ssh-pty-provider.test.ts b/src/main/providers/ssh-pty-provider.test.ts index ca0a9fd8261..c754c68ab85 100644 --- a/src/main/providers/ssh-pty-provider.test.ts +++ b/src/main/providers/ssh-pty-provider.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi, beforeEach } from 'vitest' import { SshPtyProvider } from './ssh-pty-provider' import { POWERLEVEL10K_WIZARD_DISABLE_ENV } from '../pty/powerlevel10k-wizard-env' +import { JsonRpcErrorCode } from '../ssh/relay-protocol' type MockMultiplexer = { request: ReturnType @@ -492,6 +493,109 @@ describe('SshPtyProvider', () => { expect(result).toEqual([{ id: scopedPty1, cwd: '/home', title: 'zsh' }]) }) + it('uses targeted relay liveness without listing sessions', async () => { + mux.request.mockImplementation(async (method: string, params?: { id?: string }) => { + if (method === 'pty.hasPty') { + return params?.id === 'pty-1' + } + throw new Error(`Unexpected request: ${method}`) + }) + + await expect(provider.hasPtyAsync(scopedPty1)).resolves.toBe(true) + await expect(provider.hasPtyAsync('ssh:conn-1@@pty-2')).resolves.toBe(false) + + expect(mux.request.mock.calls.filter(([method]) => method === 'pty.hasPty')).toHaveLength(2) + expect( + mux.request.mock.calls.filter(([method]) => method === 'pty.listProcesses') + ).toHaveLength(0) + }) + + it('caches an older relay method miss and falls back to inventories', async () => { + mux.request.mockImplementation(async (method: string) => { + if (method === 'pty.hasPty') { + throw Object.assign(new Error('Method not found'), { + code: JsonRpcErrorCode.MethodNotFound + }) + } + if (method === 'pty.listProcesses') { + return [ + { id: 'pty-1', cwd: '/home', title: 'zsh' }, + { id: 'pty-2', cwd: '/home', title: 'zsh' } + ] + } + if (method === 'pty.spawn') { + return { id: 'pty-3' } + } + if (method === 'pty.shutdown' || method === 'pty.attach') { + return undefined + } + throw new Error(`Unexpected request: ${method}`) + }) + + await expect( + Promise.all([provider.hasPtyAsync(scopedPty1), provider.hasPtyAsync('ssh:conn-1@@pty-2')]) + ).resolves.toEqual([true, true]) + + expect(mux.request.mock.calls.filter(([method]) => method === 'pty.hasPty')).toHaveLength(1) + expect( + mux.request.mock.calls.filter(([method]) => method === 'pty.listProcesses') + ).toHaveLength(1) + + await provider.shutdown(scopedPty1, { immediate: true }) + await expect(provider.hasPtyAsync(scopedPty1)).resolves.toBe(false) + + const spawned = await provider.spawn({ cols: 80, rows: 24 }) + await expect(provider.hasPtyAsync(spawned.id)).resolves.toBe(true) + + const attached = 'ssh:conn-1@@pty-4' + await provider.attach(attached) + await expect(provider.hasPtyAsync(attached)).resolves.toBe(true) + const notificationHandler = mux.onNotification.mock.calls[0]?.[0] + notificationHandler?.('pty.exit', { id: 'pty-4', code: 0 }) + await expect(provider.hasPtyAsync(attached)).resolves.toBe(false) + + expect(mux.request.mock.calls.filter(([method]) => method === 'pty.hasPty')).toHaveLength(1) + expect( + mux.request.mock.calls.filter(([method]) => method === 'pty.listProcesses') + ).toHaveLength(1) + }) + + it('applies shutdowns that complete while the legacy inventory is in flight', async () => { + let resolveInventory = (_sessions: { id: string; cwd: string; title: string }[]): void => { + throw new Error('inventory resolver not initialized') + } + const inventory = new Promise<{ id: string; cwd: string; title: string }[]>((resolve) => { + resolveInventory = resolve + }) + mux.request.mockImplementation(async (method: string) => { + if (method === 'pty.hasPty') { + throw Object.assign(new Error('Method not found'), { + code: JsonRpcErrorCode.MethodNotFound + }) + } + if (method === 'pty.listProcesses') { + return inventory + } + if (method === 'pty.shutdown') { + return undefined + } + throw new Error(`Unexpected request: ${method}`) + }) + + const liveness = provider.hasPtyAsync(scopedPty1) + await vi.waitFor(() => { + expect(mux.request.mock.calls.some(([method]) => method === 'pty.listProcesses')).toBe(true) + }) + await provider.shutdown(scopedPty1, { immediate: true }) + resolveInventory([{ id: 'pty-1', cwd: '/home', title: 'zsh' }]) + + await expect(liveness).resolves.toBe(false) + expect(mux.request.mock.calls.filter(([method]) => method === 'pty.hasPty')).toHaveLength(1) + expect( + mux.request.mock.calls.filter(([method]) => method === 'pty.listProcesses') + ).toHaveLength(1) + }) + it('getDefaultShell returns shell path', async () => { mux.request.mockResolvedValue('/bin/bash') const result = await provider.getDefaultShell() diff --git a/src/main/providers/ssh-pty-provider.ts b/src/main/providers/ssh-pty-provider.ts index e386e89dbba..bc271574cc8 100644 --- a/src/main/providers/ssh-pty-provider.ts +++ b/src/main/providers/ssh-pty-provider.ts @@ -1,6 +1,7 @@ import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' import type { IPtyProvider, PtyProcessInfo, PtySpawnOptions, PtySpawnResult } from './types' import { toAppSshPtyId, toRelaySshPtyId } from './ssh-pty-id' +import { SshPtyLiveness } from './ssh-pty-liveness' import { seedPowerlevel10kWizardEnv } from '../pty/powerlevel10k-wizard-env' type DataCallback = (payload: { id: string; data: string }) => void @@ -38,6 +39,7 @@ export class SshPtyProvider implements IPtyProvider { private dataListeners = new Set() private replayListeners = new Set() private exitListeners = new Set() + private ptyLiveness: SshPtyLiveness // Why: store the unsubscribe handle so dispose() can detach from the // multiplexer. Without this, notification callbacks keep firing after // the provider is torn down on disconnect, routing events to stale state. @@ -50,6 +52,11 @@ export class SshPtyProvider implements IPtyProvider { ) { this.connectionId = connectionId this.mux = mux + this.ptyLiveness = new SshPtyLiveness({ + probe: async (id) => + (await this.mux.request('pty.hasPty', { id: this.toRelayPtyId(id) })) === true, + listIds: async () => (await this.listProcesses()).map((session) => session.id) + }) // Subscribe to relay notifications for PTY events this.unsubscribeNotifications = mux.onNotification((method, params) => { @@ -67,6 +74,7 @@ export class SshPtyProvider implements IPtyProvider { break case 'pty.exit': + this.ptyLiveness.markStopped(this.toAppPtyId(params.id as string)) for (const cb of this.exitListeners) { cb({ id: this.toAppPtyId(params.id as string), code: params.code as number }) } @@ -83,6 +91,7 @@ export class SshPtyProvider implements IPtyProvider { this.dataListeners.clear() this.replayListeners.clear() this.exitListeners.clear() + this.ptyLiveness.dispose() } getConnectionId(): string { @@ -124,8 +133,10 @@ export class SshPtyProvider implements IPtyProvider { console.warn( `[ssh-pty] pty.attach succeeded for ${opts.sessionId}, replay=${!!attachResult.replay}` ) + const appPtyId = this.toAppPtyId(relaySessionId) + this.ptyLiveness.markLive(appPtyId) return { - id: this.toAppPtyId(relaySessionId), + id: appPtyId, isReattach: true, ...(attachResult.replay ? { replay: attachResult.replay } : {}) } @@ -168,9 +179,11 @@ export class SshPtyProvider implements IPtyProvider { ...(opts.paneKey ? { paneKey: opts.paneKey } : {}), ...(opts.tabId ? { tabId: opts.tabId } : {}) }) + const appPtyId = this.toAppPtyId((result as PtySpawnResult).id) + this.ptyLiveness.markLive(appPtyId) return { ...(result as PtySpawnResult), - id: this.toAppPtyId((result as PtySpawnResult).id), + id: appPtyId, ...(opts.sessionId ? { sessionExpired: true } : {}) } } @@ -207,6 +220,7 @@ export class SshPtyProvider implements IPtyProvider { async attach(id: string): Promise { await this.mux.request('pty.attach', { id: this.toRelayPtyId(id) }) + this.ptyLiveness.markLive(id) } async attachForReconnect( @@ -223,6 +237,7 @@ export class SshPtyProvider implements IPtyProvider { ...(expected?.paneKey ? { expectedPaneKey: expected.paneKey } : {}), ...(expected?.tabId ? { expectedTabId: expected.tabId } : {}) })) as { replay?: string } | undefined + this.ptyLiveness.markLive(id) return result ?? {} } @@ -240,6 +255,7 @@ export class SshPtyProvider implements IPtyProvider { immediate: opts.immediate ?? false, keepHistory: opts.keepHistory ?? false }) + this.ptyLiveness.markStopped(id) } async sendSignal(id: string, signal: string): Promise { @@ -287,10 +303,15 @@ export class SshPtyProvider implements IPtyProvider { async listProcesses(): Promise { const result = await this.mux.request('pty.listProcesses') - return (result as PtyProcessInfo[]).map((session) => ({ + const sessions = (result as PtyProcessInfo[]).map((session) => ({ ...session, id: this.toAppPtyId(session.id) })) + return sessions + } + + async hasPtyAsync(id: string): Promise { + return this.ptyLiveness.hasPty(id) } async getDefaultShell(): Promise { diff --git a/src/main/providers/types.ts b/src/main/providers/types.ts index d5cf5c60985..4ff4530d944 100644 --- a/src/main/providers/types.ts +++ b/src/main/providers/types.ts @@ -156,6 +156,8 @@ export type IPtyProvider = { spawn(opts: PtySpawnOptions): Promise attach(id: string): Promise hasPty?: (id: string) => boolean + /** Why: remote providers need an ID-only probe without building a full session inventory. */ + hasPtyAsync?: (id: string) => Promise write(id: string, data: string): void resize(id: string, cols: number, rows: number): void /** diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index 332828408bd..3ce8702cd6b 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -123,6 +123,7 @@ describe('PtyHandler', () => { expect(methods).toContain('pty.clearBuffer') expect(methods).toContain('pty.hasChildProcesses') expect(methods).toContain('pty.getForegroundProcess') + expect(methods).toContain('pty.hasPty') expect(methods).toContain('pty.listProcesses') expect(methods).toContain('pty.getDefaultShell') @@ -174,6 +175,13 @@ describe('PtyHandler', () => { expect(handler.activePtyCount).toBe(1) }) + it('answers targeted PTY liveness without building a process inventory', async () => { + await dispatcher.callRequest('pty.spawn', { cols: 80, rows: 24 }) + + await expect(dispatcher.callRequest('pty.hasPty', { id: 'pty-1' })).resolves.toBe(true) + await expect(dispatcher.callRequest('pty.hasPty', { id: 'missing' })).resolves.toBe(false) + }) + it('uses an explicit shell override and falls back to the default shell otherwise', async () => { const originalPlatform = process.platform Object.defineProperty(process, 'platform', { diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index 965d8cbdc30..f3d5c9e8d65 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -476,6 +476,7 @@ export class PtyHandler { this.dispatcher.onRequest('pty.clearBuffer', (p) => this.clearBuffer(p)) this.dispatcher.onRequest('pty.hasChildProcesses', (p) => this.hasChildProcesses(p)) this.dispatcher.onRequest('pty.getForegroundProcess', (p) => this.getForegroundProcess(p)) + this.dispatcher.onRequest('pty.hasPty', (p) => this.hasPty(p)) this.dispatcher.onRequest('pty.listProcesses', () => this.listProcesses()) this.dispatcher.onRequest('pty.getDefaultShell', async () => resolveDefaultShell()) this.dispatcher.onRequest('pty.serialize', (p) => this.serialize(p)) @@ -954,6 +955,11 @@ export class PtyHandler { return await getForegroundProcessName(managed.pty.pid, managed.pty.process || null) } + private async hasPty(params: Record): Promise { + const managed = this.ptys.get(params.id as string) + return managed !== undefined && !managed.disposed + } + private async listProcesses(): Promise { const results: PtyProcessSummary[] = [] for (const [id, managed] of this.ptys) { From 3d72336e2da2537799d8e62aad20626145a86a83 Mon Sep 17 00:00:00 2001 From: Brennan Benson Date: Sun, 12 Jul 2026 13:05:46 -0700 Subject: [PATCH 05/36] fix(runtime): fence worktree terminal lifecycle --- src/main/ipc/pty.test.ts | 38 + src/main/ipc/pty.ts | 36 +- src/main/ipc/worktrees.test.ts | 12 +- src/main/ipc/worktrees.ts | 251 +++--- src/main/runtime/orca-runtime.test.ts | 240 ++++- src/main/runtime/orca-runtime.ts | 817 ++++++++++-------- src/main/runtime/rpc/terminal-stop.test.ts | 27 + .../runtime/worktree-pty-admission.test.ts | 64 ++ src/main/runtime/worktree-pty-admission.ts | 92 ++ src/main/runtime/worktree-teardown.test.ts | 4 +- src/main/runtime/worktree-teardown.ts | 4 +- 11 files changed, 1091 insertions(+), 494 deletions(-) create mode 100644 src/main/runtime/rpc/terminal-stop.test.ts create mode 100644 src/main/runtime/worktree-pty-admission.test.ts create mode 100644 src/main/runtime/worktree-pty-admission.ts diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index dc3c9be4553..424f9a26815 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -822,6 +822,40 @@ describe('registerPtyHandlers', () => { } describe('spawn environment', () => { + it('holds renderer spawns in worktree admission even when IPC input claims a lease', async () => { + const releaseWorktreeSpawn = vi.fn() + const runtime = { + setPtyController: vi.fn(), + beginWorktreePtySpawn: vi.fn(() => releaseWorktreeSpawn), + preAllocateHandleForPty: vi.fn(() => undefined), + registerPty: vi.fn(), + noteTerminalSpawnCommand: vi.fn(), + onPtySpawned: vi.fn(), + onPtyExit: vi.fn(), + onPtyData: vi.fn() + } + registerPtyHandlers(mainWindow as never, runtime as never) + + const result = await handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24, + worktreeId: 'wt-renderer', + // Renderer payloads are structurally open at runtime; an injected field + // must never reach the controller-internal admission bypass. + worktreePtyAdmissionHeld: true + }) + + expect(result).toMatchObject({ id: expect.any(String) }) + expect(runtime.beginWorktreePtySpawn).toHaveBeenCalledWith('wt-renderer') + expect(runtime.registerPty).toHaveBeenCalledWith( + expect.any(String), + 'wt-renderer', + null, + undefined + ) + expect(releaseWorktreeSpawn).toHaveBeenCalledTimes(1) + }) + it('marks local Claude launches live until the PTY is killed', async () => { const prepareClaudeAuth = vi.fn(async () => ({ configDir: '/tmp/claude', @@ -1741,8 +1775,10 @@ describe('registerPtyHandlers', () => { }): Promise<{ id: string }> } const daemonSpawn = setupDaemonAdapter() + const releaseWorktreeSpawn = vi.fn() const runtime = { setPtyController: vi.fn(), + beginWorktreePtySpawn: vi.fn(() => releaseWorktreeSpawn), registerPty: vi.fn(), noteTerminalSpawnCommand: vi.fn(), onPtySpawned: vi.fn(), @@ -1764,6 +1800,8 @@ describe('registerPtyHandlers', () => { ) expect(spawnOptions.env.ORCA_AGENT_HOOK_PORT).toBe('5678') expect(spawnOptions.env.ORCA_AGENT_HOOK_TOKEN).toBe('agent-token') + expect(runtime.beginWorktreePtySpawn).toHaveBeenCalledWith('wt-runtime') + expect(releaseWorktreeSpawn).toHaveBeenCalledTimes(1) }) it('threads the validated pane identity into registerPty for a runtime-created daemon PTY (#7587)', async () => { diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index e64ab5de63a..961313462bd 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -16,7 +16,11 @@ import { powerMonitor } from 'electron' export { getBashShellReadyRcfileContent } from '../providers/local-pty-shell-ready' -import type { OrcaRuntimeService } from '../runtime/orca-runtime' +import type { + OrcaRuntimeService, + RuntimePtyController, + RuntimePtySpawnOptions +} from '../runtime/orca-runtime' import type { Store } from '../persistence' import type { GlobalSettings, TuiAgent } from '../../shared/types' import { terminalOutputBacklogCapChars } from '../../shared/terminal-scrollback-policy' @@ -2924,7 +2928,8 @@ export function registerPtyHandlers( // Why: the runtime controller must route through getProviderForPty() so that // CLI commands (terminal.send, terminal.stop) work for both local and remote PTYs. // Hardcoding localProvider.getPtyProcess() would silently fail for remote PTYs. - runtime?.setPtyController({ + const admissionHeldRuntimeSpawnArgs = new WeakSet() + const runtimePtyController: RuntimePtyController = { spawn: async (args) => { const startupPromise = getLocalPtyStartupPromise(args.connectionId) if (startupPromise) { @@ -3118,6 +3123,12 @@ export function registerPtyHandlers( if (existingPaneSpawn) { return await existingPaneSpawn.promise } + const releaseWorktreeSpawn = + !admissionHeldRuntimeSpawnArgs.has(args) && + args.worktreeId && + runtime?.beginWorktreePtySpawn + ? runtime.beginWorktreePtySpawn(args.worktreeId) + : null const paneSpawnReservation = materializedPaneKey ? reservePaneSpawn(materializedPaneKey) : null @@ -3292,6 +3303,18 @@ export function registerPtyHandlers( // no-op once the reservation has already resolved. rejectPaneSpawnReservation(materializedPaneKey, paneSpawnReservation, err) throw err + } finally { + releaseWorktreeSpawn?.() + } + }, + // Why: runtime create/split already hold admission across launch preparation; + // this controller-only entry point avoids a nested lease without trusting IPC input. + spawnWithWorktreePtyAdmissionHeld: async (args) => { + admissionHeldRuntimeSpawnArgs.add(args) + try { + return await runtimePtyController.spawn!(args) + } finally { + admissionHeldRuntimeSpawnArgs.delete(args) } }, write: (ptyId, data) => { @@ -3483,7 +3506,8 @@ export function registerPtyHandlers( return false } } - }) + } + runtime?.setPtyController(runtimePtyController) // ─── IPC Handlers (thin dispatch layer) ───────────────────────── @@ -4003,6 +4027,10 @@ export function registerPtyHandlers( if (existingPaneSpawn) { return await existingPaneSpawn.promise } + const releaseWorktreeSpawn = + typeof args.worktreeId === 'string' && runtime?.beginWorktreePtySpawn + ? runtime.beginWorktreePtySpawn(args.worktreeId) + : null const paneSpawnReservation = reservationPaneKey ? reservePaneSpawn(reservationPaneKey) : null const initiallyHidden = args.initiallyHidden === true // Why pre-spawn for daemon-host sessions (id minted up front): daemon @@ -4405,6 +4433,8 @@ export function registerPtyHandlers( // no-op once the reservation has already resolved. rejectPaneSpawnReservation(reservationPaneKey, paneSpawnReservation, err) throw err + } finally { + releaseWorktreeSpawn?.() } } ) diff --git a/src/main/ipc/worktrees.test.ts b/src/main/ipc/worktrees.test.ts index 9f4dea40f92..30a73ee3522 100644 --- a/src/main/ipc/worktrees.test.ts +++ b/src/main/ipc/worktrees.test.ts @@ -290,6 +290,8 @@ describe('registerWorktreeHandlers', () => { createTerminal: ReturnType splitTerminal: ReturnType notifyWorktreesChangedForRemoteClients: ReturnType + runWithWorktreePtyTeardown: ReturnType + beginWorktreePtyTeardown: ReturnType } beforeEach(() => { @@ -501,7 +503,11 @@ describe('registerWorktreeHandlers', () => { tabId: 'tab-startup', paneRuntimeId: -1 }), - notifyWorktreesChangedForRemoteClients: vi.fn() + notifyWorktreesChangedForRemoteClients: vi.fn(), + runWithWorktreePtyTeardown: vi.fn( + async (_worktreeId: string, operation: () => Promise) => await operation() + ), + beginWorktreePtyTeardown: vi.fn(async () => () => {}) } registerWorktreeHandlers(mainWindow as never, store as never, runtimeStub as never) }) @@ -8526,6 +8532,10 @@ describe('registerWorktreeHandlers', () => { const result = await handlers['worktrees:forgetLocal'](null, { worktreeId }) expect(result).toEqual({}) + expect(runtimeStub.runWithWorktreePtyTeardown).toHaveBeenCalledWith( + worktreeId, + expect.any(Function) + ) expect(killAllProcessesForWorktreeMock).toHaveBeenCalledWith(worktreeId, { runtime: runtimeStub, localProvider: ptyProvider, diff --git a/src/main/ipc/worktrees.ts b/src/main/ipc/worktrees.ts index 06397b341db..f2cd53ccbe8 100644 --- a/src/main/ipc/worktrees.ts +++ b/src/main/ipc/worktrees.ts @@ -1393,6 +1393,17 @@ export function registerWorktreeHandlers( ) const worktreeRemovalsInFlight = new Map() + const runWithWorktreePtyTeardown = ( + worktreeId: string, + operation: () => Promise + ): Promise => + typeof runtime.runWithWorktreePtyTeardown === 'function' + ? runtime.runWithWorktreePtyTeardown(worktreeId, operation) + : operation() + const beginWorktreePtyTeardown = (worktreeId: string): Promise<() => void> => + typeof runtime.beginWorktreePtyTeardown === 'function' + ? runtime.beginWorktreePtyTeardown(worktreeId) + : Promise.resolve(() => {}) ipcMain.handle( 'worktrees:remove', @@ -1425,19 +1436,21 @@ export function registerWorktreeHandlers( 'Cannot delete the project root workspace. Remove the folder project instead.' ) } - // Why: folder workspaces share one filesystem root, so there is no Git - // remove step to close shells; sweep PTYs before dropping metadata. - await killAllProcessesForWorktree(args.worktreeId, { - runtime, - localProvider: getLocalPtyProvider(), - onPtyStopped: clearProviderPtyState - }).catch((err) => { - console.warn(`[worktree-teardown] failed for ${args.worktreeId}:`, err) + return runWithWorktreePtyTeardown(args.worktreeId, async () => { + // Why: folder workspaces share one filesystem root, so there is no Git + // remove step to close shells; sweep PTYs before dropping metadata. + await killAllProcessesForWorktree(args.worktreeId, { + runtime, + localProvider: getLocalPtyProvider(), + onPtyStopped: clearProviderPtyState + }).catch((err) => { + console.warn(`[worktree-teardown] failed for ${args.worktreeId}:`, err) + }) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + preservedBranchCleanupByWorktreeId.delete(args.worktreeId) + notifyWorktreesChanged(mainWindow, repoId) + return {} }) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - preservedBranchCleanupByWorktreeId.delete(args.worktreeId) - notifyWorktreesChanged(mainWindow, repoId) - return {} } // Why: the renderer-supplied worktreeId contains a filesystem path. @@ -1683,9 +1696,16 @@ export function registerWorktreeHandlers( } const remoteRemoveOptions = !deleteBranch ? { deleteBranch } : {} - const rawRemovalResult = await (Object.keys(remoteRemoveOptions).length > 0 - ? provider!.removeWorktree(canonicalWorktreePath, args.force, remoteRemoveOptions) - : provider!.removeWorktree(canonicalWorktreePath, args.force)) + const rawRemovalResult = await runWithWorktreePtyTeardown(args.worktreeId, async () => { + await killAllProcessesForWorktree(args.worktreeId, { + runtime, + localProvider: getLocalPtyProvider(), + onPtyStopped: clearProviderPtyState + }) + return Object.keys(remoteRemoveOptions).length > 0 + ? provider!.removeWorktree(canonicalWorktreePath, args.force, remoteRemoveOptions) + : provider!.removeWorktree(canonicalWorktreePath, args.force) + }) const removalResult = preserveBranchHeadFallback( rawRemovalResult, registeredWorktree.head @@ -1759,112 +1779,117 @@ export function registerWorktreeHandlers( shouldTearDownPtys = false } - await closeLocalWatcherForRemoval(canonicalWorktreePath) - - if (shouldTearDownPtys) { - // Why: once preflight proves normal deletion is clean, kill PTYs before - // git-level removal so Windows handles cannot keep the directory busy. - await killAllProcessesForWorktree(args.worktreeId, { - runtime, - localProvider: getLocalPtyProvider(), - onPtyStopped: clearProviderPtyState - }) - .then((r) => { - const total = r.runtimeStopped + r.providerStopped + r.registryStopped - if (total > 0) { - console.info( - `[worktree-teardown] ${args.worktreeId} killed runtime=${r.runtimeStopped} provider=${r.providerStopped} registry=${r.registryStopped}` - ) - } - }) - .catch((err) => { - console.warn(`[worktree-teardown] failed for ${args.worktreeId}:`, err) - }) - } - + const releaseWorktreePtyTeardown = await beginWorktreePtyTeardown(args.worktreeId) let removalResult: RemoveWorktreeResult | undefined try { - const removeOptions = { - ...(!deleteBranch ? { deleteBranch } : {}), - // Why: this handler already paid for an authoritative worktree - // list to validate the target; reuse it instead of rescanning - // every sibling worktree during the hot delete path. - knownRemovedWorktree: refreshedRegisteredWorktree, - ...(hasLocalWorktreeGitOptions ? localWorktreeGitOptions : {}) + await closeLocalWatcherForRemoval(canonicalWorktreePath) + + if (shouldTearDownPtys) { + // Why: once preflight proves normal deletion is clean, kill PTYs before + // git-level removal so Windows handles cannot keep the directory busy. + await killAllProcessesForWorktree(args.worktreeId, { + runtime, + localProvider: getLocalPtyProvider(), + onPtyStopped: clearProviderPtyState + }) + .then((r) => { + const total = r.runtimeStopped + r.providerStopped + r.registryStopped + if (total > 0) { + console.info( + `[worktree-teardown] ${args.worktreeId} killed runtime=${r.runtimeStopped} provider=${r.providerStopped} registry=${r.registryStopped}` + ) + } + }) + .catch((err) => { + console.warn(`[worktree-teardown] failed for ${args.worktreeId}:`, err) + }) } - removalResult = preserveBranchHeadFallback( - await removeWorktree( - repo.path, - canonicalWorktreePath, - args.force ?? false, - removeOptions - ), - refreshedRegisteredWorktree.head - ) - } catch (error) { - // Why: Git for Windows can deregister a clean worktree before its - // recursive filesystem deletion fails transiently. - const recoveredRemovalResult = await recoverLocalWindowsWorktreeRemoval({ - error, - force: args.force ?? false, - canonicalWorktreePath, - repoPath: repo.path, - localWorktreeGitOptions, - registeredWorktree: refreshedRegisteredWorktree, - deleteBranch, - closeWatcher: closeLocalWatcherForRemoval - }) - if (recoveredRemovalResult) { - removalResult = recoveredRemovalResult - } else if (isOrphanedWorktreeError(error)) { - // If git no longer tracks this worktree, clean up the directory and metadata - console.warn( - `[worktrees] Orphaned worktree detected at ${canonicalWorktreePath}, cleaning up` + + try { + const removeOptions = { + ...(!deleteBranch ? { deleteBranch } : {}), + // Why: this handler already paid for an authoritative worktree + // list to validate the target; reuse it instead of rescanning + // every sibling worktree during the hot delete path. + knownRemovedWorktree: refreshedRegisteredWorktree, + ...(hasLocalWorktreeGitOptions ? localWorktreeGitOptions : {}) + } + removalResult = preserveBranchHeadFallback( + await removeWorktree( + repo.path, + canonicalWorktreePath, + args.force ?? false, + removeOptions + ), + refreshedRegisteredWorktree.head ) - const access = getLocalWorktreePathAccess(localWorktreeGitOptions) - if ( - await canSafelyRemoveOrphanedWorktreeDirectory( - toLocalWorktreeRuntimePath(canonicalWorktreePath, localWorktreeGitOptions), - toLocalWorktreeRuntimePath(repo.path, localWorktreeGitOptions), - access.statPath, - access.readPath + } catch (error) { + // Why: Git for Windows can deregister a clean worktree before its + // recursive filesystem deletion fails transiently. + const recoveredRemovalResult = await recoverLocalWindowsWorktreeRemoval({ + error, + force: args.force ?? false, + canonicalWorktreePath, + repoPath: repo.path, + localWorktreeGitOptions, + registeredWorktree: refreshedRegisteredWorktree, + deleteBranch, + closeWatcher: closeLocalWatcherForRemoval + }) + if (recoveredRemovalResult) { + removalResult = recoveredRemovalResult + } else if (isOrphanedWorktreeError(error)) { + // If git no longer tracks this worktree, clean up the directory and metadata + console.warn( + `[worktrees] Orphaned worktree detected at ${canonicalWorktreePath}, cleaning up` ) - ) { - await closeLocalWatcherForRemoval(canonicalWorktreePath) - await removeLocalWorktreePath(canonicalWorktreePath, localWorktreeGitOptions).catch( - () => {} + const access = getLocalWorktreePathAccess(localWorktreeGitOptions) + if ( + await canSafelyRemoveOrphanedWorktreeDirectory( + toLocalWorktreeRuntimePath(canonicalWorktreePath, localWorktreeGitOptions), + toLocalWorktreeRuntimePath(repo.path, localWorktreeGitOptions), + access.statPath, + access.readPath + ) + ) { + await closeLocalWatcherForRemoval(canonicalWorktreePath) + await removeLocalWorktreePath(canonicalWorktreePath, localWorktreeGitOptions).catch( + () => {} + ) + } else { + console.warn( + `[worktrees] Refusing recursive cleanup for unproven worktree directory: ${canonicalWorktreePath}` + ) + } + // Why: `git worktree remove` failed, so git's internal worktree tracking + // (`.git/worktrees/`) is still intact. Without pruning, `git worktree + // list` continues to show the stale entry and the branch it had checked out + // remains locked — other worktrees cannot check it out. + await gitExecFileAsync(['worktree', 'prune'], { + cwd: repo.path, + ...localWorktreeGitOptions + }).catch(() => {}) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + args.worktreeId, + removedPushTarget, + store, + localWorktreeGitOptions ) + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + preservedBranchCleanupByWorktreeId.delete(args.worktreeId) + invalidateAuthorizedRootsCache() + notifyWorktreesChanged(mainWindow, repoId) + return {} } else { - console.warn( - `[worktrees] Refusing recursive cleanup for unproven worktree directory: ${canonicalWorktreePath}` + throw new Error( + formatWorktreeRemovalError(error, canonicalWorktreePath, args.force ?? false) ) } - // Why: `git worktree remove` failed, so git's internal worktree tracking - // (`.git/worktrees/`) is still intact. Without pruning, `git worktree - // list` continues to show the stale entry and the branch it had checked out - // remains locked — other worktrees cannot check it out. - await gitExecFileAsync(['worktree', 'prune'], { - cwd: repo.path, - ...localWorktreeGitOptions - }).catch(() => {}) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - args.worktreeId, - removedPushTarget, - store, - localWorktreeGitOptions - ) - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - preservedBranchCleanupByWorktreeId.delete(args.worktreeId) - invalidateAuthorizedRootsCache() - notifyWorktreesChanged(mainWindow, repoId) - return {} - } else { - throw new Error( - formatWorktreeRemovalError(error, canonicalWorktreePath, args.force ?? false) - ) } + } finally { + releaseWorktreePtyTeardown() } await cleanupUnusedWorktreePushTargetRemote( repo.path, @@ -1930,7 +1955,7 @@ export function registerWorktreeHandlers( throw new Error(`Worktree deletion already in progress: ${args.worktreeId}`) } - const forget = (async (): Promise => { + const forget = runWithWorktreePtyTeardown(args.worktreeId, async () => { if (isFolderRepo(repo) && args.worktreeId === getFolderWorkspaceRootId(repo)) { throw new Error( 'Cannot delete the project root workspace. Remove the folder project instead.' @@ -1953,7 +1978,7 @@ export function registerWorktreeHandlers( preservedBranchCleanupByWorktreeId.delete(args.worktreeId) notifyWorktreesChanged(mainWindow, repoId) return {} - })() + }) worktreeRemovalsInFlight.set(inFlightKey, { optionsKey, promise: forget }) try { return await forget diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 65ce4681d53..f08b1e280ea 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -17306,6 +17306,50 @@ describe('OrcaRuntimeService', () => { }) }) + it('holds runtime terminal admission from resolved workspace through launch preparation', async () => { + const launchPreparation = deferred>() + const spawn = vi.fn().mockResolvedValue({ id: 'runtime-prepared-pty' }) + const spawnWithWorktreePtyAdmissionHeld = vi + .fn() + .mockResolvedValue({ id: 'runtime-prepared-pty' }) + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + spawn, + spawnWithWorktreePtyAdmissionHeld, + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + const resolveLaunchOptions = vi + .spyOn( + runtime as unknown as { + resolveAgentTerminalCreateOptions: () => Promise> + }, + 'resolveAgentTerminalCreateOptions' + ) + .mockReturnValue(launchPreparation.promise) + + const creating = runtime.createTerminal(`id:${TEST_WORKTREE_ID}`) + await vi.waitFor(() => expect(resolveLaunchOptions).toHaveBeenCalled()) + + let teardownEntered = false + const teardown = runtime.runWithWorktreePtyTeardown(TEST_WORKTREE_ID, async () => { + teardownEntered = true + }) + await new Promise((resolve) => setImmediate(resolve)) + + expect(teardownEntered).toBe(false) + expect(spawnWithWorktreePtyAdmissionHeld).not.toHaveBeenCalled() + + launchPreparation.resolve({}) + await creating + await teardown + + expect(spawn).not.toHaveBeenCalled() + expect(spawnWithWorktreePtyAdmissionHeld).toHaveBeenCalledTimes(1) + expect(teardownEntered).toBe(true) + }) + it('keeps background-presentation PTY-backed mobile session tabs inactive', async () => { const spawn = vi.fn().mockResolvedValue({ id: 'laptop-created-pty' }) const runtime = new OrcaRuntimeService(store) @@ -17429,6 +17473,64 @@ describe('OrcaRuntimeService', () => { ) }) + it('holds runtime split admission through registration and reveal', async () => { + const splitSpawn = deferred<{ id: string }>() + const splitReveal = deferred<{ tabId: string }>() + const spawnWithWorktreePtyAdmissionHeld = vi + .fn() + .mockResolvedValueOnce({ id: 'runtime-parent-pty' }) + .mockReturnValueOnce(splitSpawn.promise) + const revealTerminalSession = vi + .fn() + .mockResolvedValueOnce({ tabId: 'runtime-parent-tab' }) + .mockReturnValueOnce(splitReveal.promise) + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + spawn: vi.fn(), + spawnWithWorktreePtyAdmissionHeld, + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + runtime.setNotifier({ + worktreesChanged: vi.fn(), + reposChanged: vi.fn(), + activateWorktree: vi.fn(), + createTerminal: vi.fn(), + revealTerminalSession, + splitTerminal: vi.fn(), + renameTerminal: vi.fn(), + focusTerminal: vi.fn(), + closeTerminal: vi.fn(), + sleepWorktree: vi.fn(), + terminalFitOverrideChanged: vi.fn(), + terminalDriverChanged: vi.fn() + }) + const parent = await runtime.createTerminal(`id:${TEST_WORKTREE_ID}`, { + tabId: 'runtime-parent-tab', + leafId: HEADLESS_LEAF_ID, + activate: true + }) + + const splitting = runtime.splitTerminal(parent.handle) + await vi.waitFor(() => expect(spawnWithWorktreePtyAdmissionHeld).toHaveBeenCalledTimes(2)) + let teardownEntered = false + const teardown = runtime.runWithWorktreePtyTeardown(TEST_WORKTREE_ID, async () => { + teardownEntered = true + }) + + splitSpawn.resolve({ id: 'runtime-split-pty' }) + await vi.waitFor(() => expect(revealTerminalSession).toHaveBeenCalledTimes(2)) + await new Promise((resolve) => setImmediate(resolve)) + expect(teardownEntered).toBe(false) + + splitReveal.resolve({ tabId: 'runtime-parent-tab' }) + await splitting + await teardown + + expect(teardownEntered).toBe(true) + }) + it('pushes PTY-backed mobile session tab title and agent status changes to subscribers', async () => { const spawn = vi.fn().mockResolvedValue({ id: 'laptop-created-pty' }) const runtime = new OrcaRuntimeService(store) @@ -22221,10 +22323,12 @@ describe('OrcaRuntimeService', () => { syncSinglePty(runtime, 'pty-1') let completed = false - const stopping = runtime.stopTerminalsForWorktree(TEST_WORKTREE_ID).then((result) => { - completed = true - return result - }) + const stopping = runtime + .stopTerminalsForWorktree(TEST_WORKTREE_ID, { waitForProviderShutdown: true }) + .then((result) => { + completed = true + return result + }) await vi.waitFor(() => expect(stopAndWait).toHaveBeenCalledWith('pty-1')) expect(completed).toBe(false) @@ -22234,6 +22338,26 @@ describe('OrcaRuntimeService', () => { await expect(stopping).resolves.toEqual({ stopped: 1 }) }) + it('keeps generic worktree terminal stops graceful', async () => { + const runtime = new OrcaRuntimeService(store) + const kill = vi.fn(() => true) + const stopAndWait = vi.fn(async () => true) + runtime.setPtyController({ + write: () => true, + kill, + stopAndWait, + getForegroundProcess: async () => null + }) + syncSinglePty(runtime, 'pty-1') + + await expect(runtime.stopTerminalsForWorktree(TEST_WORKTREE_ID)).resolves.toEqual({ + stopped: 1 + }) + + expect(kill).toHaveBeenCalledWith('pty-1') + expect(stopAndWait).not.toHaveBeenCalled() + }) + it('stops exactly the expected live PTYs for a worktree', async () => { const runtime = new OrcaRuntimeService(store) const stopped: string[] = [] @@ -29333,6 +29457,114 @@ describe('OrcaRuntimeService', () => { // The pre-daemon provider must not have been consulted for the kill. expect(preDaemonProvider.shutdown).not.toHaveBeenCalled() }) + + it('drains a late SSH spawn and stops it before remote worktree removal', async () => { + const remoteRepo = { + ...store.getRepo(TEST_REPO_ID)!, + path: '/remote/repo', + connectionId: 'ssh-1' + } + const remoteWorktree = { + path: '/remote/feature-wt', + head: 'def456', + branch: 'feature/test', + isBare: false, + isMainWorktree: false + } + const remoteWorktreeId = `${remoteRepo.id}::${remoteWorktree.path}` + const runtimeStore = { + ...store, + getRepos: () => [remoteRepo], + getRepo: (id: string) => (id === remoteRepo.id ? remoteRepo : undefined), + getAllWorktreeMeta: () => ({ [remoteWorktreeId]: makeWorktreeMeta() }), + getWorktreeMeta: (id: string) => (id === remoteWorktreeId ? makeWorktreeMeta() : undefined) + } + const callOrder: string[] = [] + const gitProvider = { + listWorktrees: vi.fn().mockResolvedValue([ + { + path: remoteRepo.path, + head: 'main', + branch: 'main', + isBare: false, + isMainWorktree: true + }, + remoteWorktree + ]), + removeWorktree: vi.fn(async () => { + callOrder.push('git-remove') + return {} + }) + } + const localProvider = createProviderStub(async () => []) + registerSshGitProvider('ssh-1', gitProvider as never) + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { + getLocalProvider: () => localProvider as never + }) + let releaseFirstStop = () => {} + runtime.setPtyController({ + write: () => true, + kill: () => false, + stopAndWait: vi.fn( + (ptyId: string) => + new Promise((resolve) => { + callOrder.push(`stop:${ptyId}`) + if (ptyId === 'ssh:ssh-1@@existing') { + releaseFirstStop = () => resolve(true) + } else { + resolve(true) + } + }) + ), + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: 'tab-remote', + worktreeId: remoteWorktreeId, + title: 'Terminal', + activeLeafId: 'pane:remote', + layout: null + } + ], + leaves: [ + { + tabId: 'tab-remote', + worktreeId: remoteWorktreeId, + leafId: 'pane:remote', + paneRuntimeId: 1, + ptyId: 'ssh:ssh-1@@existing' + } + ] + }) + runtime.registerPty('ssh:ssh-1@@existing', remoteWorktreeId, 'ssh-1') + const releaseLateSpawn = runtime.beginWorktreePtySpawn(remoteWorktreeId) + + try { + const removal = runtime.removeManagedWorktree(remoteWorktreeId) + await vi.waitFor(() => expect(gitProvider.listWorktrees).toHaveBeenCalled()) + expect(callOrder).toEqual([]) + + runtime.registerPty('ssh:ssh-1@@late', remoteWorktreeId, 'ssh-1') + releaseLateSpawn() + await vi.waitFor(() => expect(callOrder).toContain('stop:ssh:ssh-1@@existing')) + expect(callOrder).not.toContain('stop:ssh:ssh-1@@late') + expect(callOrder).not.toContain('git-remove') + + releaseFirstStop() + await removal + + expect(callOrder).toEqual([ + 'stop:ssh:ssh-1@@existing', + 'stop:ssh:ssh-1@@late', + 'git-remove' + ]) + } finally { + unregisterSshGitProvider('ssh-1') + } + }) }) describe('stale terminal handle resolution (#7718)', () => { function syncSingleTerminalGraph(runtime: OrcaRuntimeService, ptyId: string): void { diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index fb41f053da8..4be376db10c 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -714,6 +714,7 @@ import { registerTerminalViewAttributesApplier } from './terminal-view-attribute-store' import { killAllProcessesForWorktree } from './worktree-teardown' +import { WorktreePtyAdmission } from './worktree-pty-admission' import { MOBILE_SUBSCRIBE_SCROLLBACK_ROWS } from './scrollback-limits' import { createMobileSessionTabsNotifyCoalescer, @@ -1184,25 +1185,28 @@ type HeadlessSeedMetadata = { kittyKeyboardFlags?: number } -type RuntimePtyController = { - spawn?(opts: { - cols: number - rows: number - cwd?: string - command?: string - commandDelivery?: 'renderer' | 'provider' - startupCommandDelivery?: WorktreeStartupLaunch['startupCommandDelivery'] - env?: Record - envToDelete?: string[] - telemetry?: WorktreeStartupLaunch['telemetry'] - connectionId?: string | null - worktreeId?: string - preAllocatedHandle?: string - tabId?: string - leafId?: string - sessionId?: string - persistHostSessionBinding?: boolean - }): Promise<{ id: string }> +export type RuntimePtySpawnOptions = { + cols: number + rows: number + cwd?: string + command?: string + commandDelivery?: 'renderer' | 'provider' + startupCommandDelivery?: WorktreeStartupLaunch['startupCommandDelivery'] + env?: Record + envToDelete?: string[] + telemetry?: WorktreeStartupLaunch['telemetry'] + connectionId?: string | null + worktreeId?: string + preAllocatedHandle?: string + tabId?: string + leafId?: string + sessionId?: string + persistHostSessionBinding?: boolean +} + +export type RuntimePtyController = { + spawn?(opts: RuntimePtySpawnOptions): Promise<{ id: string }> + spawnWithWorktreePtyAdmissionHeld?(opts: RuntimePtySpawnOptions): Promise<{ id: string }> write(ptyId: string, data: string): boolean kill(ptyId: string): boolean stopAndWait?(ptyId: string, opts?: { keepHistory?: boolean }): Promise @@ -2495,6 +2499,7 @@ export class OrcaRuntimeService { private canonicalFetchKeyCache = new Map() private optimisticReconcileTokens = new Map() private removeManagedWorktreeInFlight = new Map() + private readonly worktreePtyAdmission = new WorktreePtyAdmission() private preservedBranchCleanupByWorktreeId = new Map() private readonly getLocalProviderFn: (() => IPtyProvider) | null private readonly onPtyStopped: ((ptyId: string) => void) | null @@ -16907,23 +16912,25 @@ export class OrcaRuntimeService { 'Cannot delete the project root workspace. Remove the folder project instead.' ) } - const localProvider = this.getLocalProvider() - if (localProvider) { - // Why: folder workspace deletion has no Git removal phase where PTYs - // would otherwise be swept; tear them down before hiding the workspace. - await killAllProcessesForWorktree(removalTarget.id, { - runtime: this, - localProvider, - onPtyStopped: this.onPtyStopped ?? undefined - }).catch((err) => { - console.warn(`[worktree-teardown] failed for ${removalTarget.id}:`, err) - }) - } - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) - this.invalidateResolvedWorktreeCache() - this.notifyWorktreesChanged(repo.id) - return {} + return this.runWithWorktreePtyTeardown(removalTarget.id, async () => { + const localProvider = this.getLocalProvider() + if (localProvider) { + // Why: folder workspace deletion has no Git removal phase where PTYs + // would otherwise be swept; tear them down before hiding the workspace. + await killAllProcessesForWorktree(removalTarget.id, { + runtime: this, + localProvider, + onPtyStopped: this.onPtyStopped ?? undefined + }).catch((err) => { + console.warn(`[worktree-teardown] failed for ${removalTarget.id}:`, err) + }) + } + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) + this.invalidateResolvedWorktreeCache() + this.notifyWorktreesChanged(repo.id) + return {} + }) } const provider = repo.connectionId ? requireSshGitProvider(repo.connectionId) : null const fsProvider = repo.connectionId ? getSshFilesystemProvider(repo.connectionId) : null @@ -17132,9 +17139,22 @@ export class OrcaRuntimeService { } if (repo.connectionId) { const remoteRemoveOptions = !deleteBranch ? { deleteBranch } : {} - const rawRemovalResult = await (Object.keys(remoteRemoveOptions).length > 0 - ? provider!.removeWorktree(canonicalWorktreePath, force, remoteRemoveOptions) - : provider!.removeWorktree(canonicalWorktreePath, force)) + const rawRemovalResult = await this.runWithWorktreePtyTeardown( + removalTarget.id, + async () => { + const localProvider = this.getLocalProvider() + if (localProvider) { + await killAllProcessesForWorktree(removalTarget.id, { + runtime: this, + localProvider, + onPtyStopped: this.onPtyStopped ?? undefined + }) + } + return Object.keys(remoteRemoveOptions).length > 0 + ? provider!.removeWorktree(canonicalWorktreePath, force, remoteRemoveOptions) + : provider!.removeWorktree(canonicalWorktreePath, force) + } + ) const removalResult = this.preserveBranchHeadFallback( rawRemovalResult, registeredWorktree.head @@ -17217,116 +17237,121 @@ export class OrcaRuntimeService { shouldTearDownPtys = false } - const localProvider = this.getLocalProvider() - await closeLocalWatcherForWorktreePath(canonicalWorktreePath).catch((err) => { - console.warn(`[filesystem-watcher] failed to close ${canonicalWorktreePath}:`, err) - }) - if (localProvider && shouldTearDownPtys) { - // Why: once preflight proves normal deletion is clean, kill PTYs before - // git-level removal so Windows handles cannot keep the directory busy. This also - // closes the headless-CLI leak for confirmed-removable worktrees. - await killAllProcessesForWorktree(removalTarget.id, { - runtime: this, - localProvider, - onPtyStopped: this.onPtyStopped ?? undefined - }) - .then((r) => { - const total = r.runtimeStopped + r.providerStopped + r.registryStopped - if (total > 0) { - // Why (design §4.4 observability): breadcrumb lets ops - // distinguish a renderer-state-induced leak (diff-path purge - // non-empty) from a backend-induced one (nothing to kill but - // memory still pinned). Emit only when the sweep actually did - // work so steady-state logs stay quiet. - console.info( - `[worktree-teardown] ${removalTarget.id} killed runtime=${r.runtimeStopped} provider=${r.providerStopped} registry=${r.registryStopped}` - ) - } - }) - .catch((err) => { - console.warn(`[worktree-teardown] failed for ${removalTarget.id}:`, err) - }) - } - + const releaseWorktreePtyTeardown = await this.beginWorktreePtyTeardown(removalTarget.id) let removalResult: RemoveWorktreeResult | undefined try { - const removeOptions = { - ...(!deleteBranch ? { deleteBranch } : {}), - // Why: removal already validated the Git row under the selected - // project runtime; keep branch cleanup on that same canonical row. - knownRemovedWorktree: refreshedRegisteredWorktree, - ...localWorktreeGitOptions - } - removalResult = this.preserveBranchHeadFallback( - await removeWorktree(repo.path, canonicalWorktreePath, force, removeOptions), - refreshedRegisteredWorktree.head - ) - } catch (error) { - // Why: Git for Windows can deregister a clean worktree before its - // recursive filesystem deletion fails transiently. - const recoveredRemovalResult = await recoverLocalWindowsWorktreeRemoval({ - error, - force, - canonicalWorktreePath, - repoPath: repo.path, - localWorktreeGitOptions, - registeredWorktree: refreshedRegisteredWorktree, - deleteBranch, - closeWatcher: (worktreePath) => - closeLocalWatcherForWorktreePath(worktreePath).catch((err) => { - console.warn(`[filesystem-watcher] failed to close ${worktreePath}:`, err) - }) + const localProvider = this.getLocalProvider() + await closeLocalWatcherForWorktreePath(canonicalWorktreePath).catch((err) => { + console.warn(`[filesystem-watcher] failed to close ${canonicalWorktreePath}:`, err) }) - if (recoveredRemovalResult) { - removalResult = recoveredRemovalResult - } else if (isOrphanedWorktreeError(error)) { - const access = getLocalWorktreePathAccess(localWorktreeGitOptions) - if ( - await canSafelyRemoveOrphanedWorktreeDirectory( - toLocalWorktreeRuntimePath(canonicalWorktreePath, localWorktreeGitOptions), - toLocalWorktreeRuntimePath(repo.path, localWorktreeGitOptions), - access.statPath, - access.readPath - ) - ) { - await closeLocalWatcherForWorktreePath(canonicalWorktreePath).catch((err) => { - console.warn(`[filesystem-watcher] failed to close ${canonicalWorktreePath}:`, err) + if (localProvider && shouldTearDownPtys) { + // Why: once preflight proves normal deletion is clean, kill PTYs before + // git-level removal so Windows handles cannot keep the directory busy. This also + // closes the headless-CLI leak for confirmed-removable worktrees. + await killAllProcessesForWorktree(removalTarget.id, { + runtime: this, + localProvider, + onPtyStopped: this.onPtyStopped ?? undefined + }) + .then((r) => { + const total = r.runtimeStopped + r.providerStopped + r.registryStopped + if (total > 0) { + // Why (design §4.4 observability): breadcrumb lets ops + // distinguish a renderer-state-induced leak (diff-path purge + // non-empty) from a backend-induced one (nothing to kill but + // memory still pinned). Emit only when the sweep actually did + // work so steady-state logs stay quiet. + console.info( + `[worktree-teardown] ${removalTarget.id} killed runtime=${r.runtimeStopped} provider=${r.providerStopped} registry=${r.registryStopped}` + ) + } }) - await removeLocalWorktreePath(canonicalWorktreePath, localWorktreeGitOptions).catch( - () => {} - ) - } else { - console.warn( - `[worktrees] Refusing recursive cleanup for unproven worktree directory: ${canonicalWorktreePath}` - ) - } - // Why: `git worktree remove` failed, so git's internal worktree tracking - // (`.git/worktrees/`) is still intact. Without pruning, `git worktree - // list` continues to show the stale entry and the branch it had checked out - // remains locked — other worktrees cannot check it out. - await gitExecFileAsync(['worktree', 'prune'], { - cwd: repo.path, + .catch((err) => { + console.warn(`[worktree-teardown] failed for ${removalTarget.id}:`, err) + }) + } + + try { + const removeOptions = { + ...(!deleteBranch ? { deleteBranch } : {}), + // Why: removal already validated the Git row under the selected + // project runtime; keep branch cleanup on that same canonical row. + knownRemovedWorktree: refreshedRegisteredWorktree, ...localWorktreeGitOptions - }).catch(() => {}) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - removalTarget.id, - removedPushTarget, - store, - localWorktreeGitOptions + } + removalResult = this.preserveBranchHeadFallback( + await removeWorktree(repo.path, canonicalWorktreePath, force, removeOptions), + refreshedRegisteredWorktree.head ) - this.clearOptimisticReconcileToken(removalTarget.id) - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) - this.invalidateResolvedWorktreeCache() - invalidateAuthorizedRootsCache() - this.notifyWorktreesChanged(repo.id) - return { - ...(warning ? { warning } : {}) + } catch (error) { + // Why: Git for Windows can deregister a clean worktree before its + // recursive filesystem deletion fails transiently. + const recoveredRemovalResult = await recoverLocalWindowsWorktreeRemoval({ + error, + force, + canonicalWorktreePath, + repoPath: repo.path, + localWorktreeGitOptions, + registeredWorktree: refreshedRegisteredWorktree, + deleteBranch, + closeWatcher: (worktreePath) => + closeLocalWatcherForWorktreePath(worktreePath).catch((err) => { + console.warn(`[filesystem-watcher] failed to close ${worktreePath}:`, err) + }) + }) + if (recoveredRemovalResult) { + removalResult = recoveredRemovalResult + } else if (isOrphanedWorktreeError(error)) { + const access = getLocalWorktreePathAccess(localWorktreeGitOptions) + if ( + await canSafelyRemoveOrphanedWorktreeDirectory( + toLocalWorktreeRuntimePath(canonicalWorktreePath, localWorktreeGitOptions), + toLocalWorktreeRuntimePath(repo.path, localWorktreeGitOptions), + access.statPath, + access.readPath + ) + ) { + await closeLocalWatcherForWorktreePath(canonicalWorktreePath).catch((err) => { + console.warn(`[filesystem-watcher] failed to close ${canonicalWorktreePath}:`, err) + }) + await removeLocalWorktreePath(canonicalWorktreePath, localWorktreeGitOptions).catch( + () => {} + ) + } else { + console.warn( + `[worktrees] Refusing recursive cleanup for unproven worktree directory: ${canonicalWorktreePath}` + ) + } + // Why: `git worktree remove` failed, so git's internal worktree tracking + // (`.git/worktrees/`) is still intact. Without pruning, `git worktree + // list` continues to show the stale entry and the branch it had checked out + // remains locked — other worktrees cannot check it out. + await gitExecFileAsync(['worktree', 'prune'], { + cwd: repo.path, + ...localWorktreeGitOptions + }).catch(() => {}) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + removalTarget.id, + removedPushTarget, + store, + localWorktreeGitOptions + ) + this.clearOptimisticReconcileToken(removalTarget.id) + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) + this.invalidateResolvedWorktreeCache() + invalidateAuthorizedRootsCache() + this.notifyWorktreesChanged(repo.id) + return { + ...(warning ? { warning } : {}) + } + } else { + throw new Error(formatWorktreeRemovalError(error, canonicalWorktreePath, force)) } - } else { - throw new Error(formatWorktreeRemovalError(error, canonicalWorktreePath, force)) } + } finally { + releaseWorktreePtyTeardown() } await cleanupUnusedWorktreePushTargetRemote( @@ -17482,187 +17507,191 @@ export class OrcaRuntimeService { throw new Error('runtime_unavailable') } const workspace = await this.resolveTerminalWorkspaceLaunchScope(worktreeSelector) - const launchOpts = await this.resolveAgentTerminalCreateOptions(workspace, opts) - const cwd = - this.resolveWorkspaceTerminalStartupCwd(workspace, launchOpts.cwd) ?? workspace.path - const preAllocatedHandle = this.createPreAllocatedTerminalHandle() - // Why: mint tabId in main before spawn so paneKey is known at PTY env - // build time. Hook-based agent status (Claude/Codex/Cursor/Gemini) keys - // off `${tabId}:${leafId}` — without these vars set on the PTY, the - // hook payload arrives with an empty paneKey and the renderer cannot - // attribute the event. Use a stable UUID leaf because hooks reject the - // legacy numeric pane keys after the pane-id migration. - const hintedTabId = launchOpts.tabId?.trim() - const canAdoptPaneIdentity = - hintedTabId !== undefined && - isValidHostTerminalTabId(hintedTabId) && - launchOpts.leafId !== undefined && - isTerminalLeafId(launchOpts.leafId) - const tabId = canAdoptPaneIdentity ? (hintedTabId as string) : randomUUID() - const leafId = canAdoptPaneIdentity ? (launchOpts.leafId as string) : randomUUID() - const paneKey = makePaneKey(tabId, leafId) - const launchToken = launchOpts.launchConfig - ? (launchOpts.launchToken ?? randomUUID()) - : undefined - const baseEnv = { - ...launchOpts.env, - ...(launchToken ? { ORCA_AGENT_LAUNCH_TOKEN: launchToken } : {}) - } - const claudeAgentTeamsSourceCommand = - launchOpts.claudeAgentTeamsSourceCommand?.trim() || launchOpts.command?.trim() || undefined - const claudeAgentTeamsMode = this.store?.getSettings?.().claudeAgentTeamsMode - const effectiveClaudeAgentTeamsMode = inferCapturedClaudeAgentTeamsMode( - launchOpts.launchConfig, - claudeAgentTeamsSourceCommand, - claudeAgentTeamsMode - ) - const agentTeamsPlan = await buildClaudeAgentTeamsLaunchPlan({ - command: claudeAgentTeamsSourceCommand, - mode: effectiveClaudeAgentTeamsMode, - baseEnv: { - ...process.env, - ...baseEnv - }, - createTeamEnv: (shimDir, shimBin) => - this.claudeAgentTeams.createLaunchEnv({ - leaderHandle: preAllocatedHandle, - baseEnv: { - ...process.env, - ...baseEnv - }, - shimDir, - shimBin - }).env - }) - const sequencedStartupCommand = - agentTeamsPlan && - claudeAgentTeamsSourceCommand && - launchOpts.command && - claudeAgentTeamsSourceCommand !== launchOpts.command - ? agentTeamsPlan.command + return await this.runWithWorktreePtySpawn(workspace.id, async () => { + const launchOpts = await this.resolveAgentTerminalCreateOptions(workspace, opts) + const cwd = + this.resolveWorkspaceTerminalStartupCwd(workspace, launchOpts.cwd) ?? workspace.path + const preAllocatedHandle = this.createPreAllocatedTerminalHandle() + // Why: mint tabId in main before spawn so paneKey is known at PTY env + // build time. Hook-based agent status (Claude/Codex/Cursor/Gemini) keys + // off `${tabId}:${leafId}` — without these vars set on the PTY, the + // hook payload arrives with an empty paneKey and the renderer cannot + // attribute the event. Use a stable UUID leaf because hooks reject the + // legacy numeric pane keys after the pane-id migration. + const hintedTabId = launchOpts.tabId?.trim() + const canAdoptPaneIdentity = + hintedTabId !== undefined && + isValidHostTerminalTabId(hintedTabId) && + launchOpts.leafId !== undefined && + isTerminalLeafId(launchOpts.leafId) + const tabId = canAdoptPaneIdentity ? (hintedTabId as string) : randomUUID() + const leafId = canAdoptPaneIdentity ? (launchOpts.leafId as string) : randomUUID() + const paneKey = makePaneKey(tabId, leafId) + const launchToken = launchOpts.launchConfig + ? (launchOpts.launchToken ?? randomUUID()) : undefined - const effectiveLaunchConfig = - launchOpts.launchConfig && agentTeamsPlan - ? { - ...launchOpts.launchConfig, - agentCommand: launchOpts.launchConfig.agentCommand - ? effectiveClaudeAgentTeamsMode === 'in-process' || process.platform === 'win32' - ? addClaudeTeammateModeInProcess(launchOpts.launchConfig.agentCommand) - : addClaudeTeammateModeAuto(launchOpts.launchConfig.agentCommand) - : agentTeamsPlan.command, - agentEnv: { - ...launchOpts.launchConfig.agentEnv, - ...agentTeamsPlan.env - } - } - : launchOpts.launchConfig - // Why: setup/agent sequencing wraps the PTY launch in a wait shell before - // Claude Agent Teams runs. Preserve the direct Claude command separately - // so the wrapper can exec the teammate-mode variant after setup completes. - const env = this.buildTerminalWorkspaceEnv( - workspace, - { - ...baseEnv, - ...(sequencedStartupCommand - ? { [SETUP_AGENT_SEQUENCE_STARTUP_COMMAND_ENV]: sequencedStartupCommand } - : {}) - }, - paneKey, - tabId, - agentTeamsPlan?.env - ) - const result = await this.ptyController.spawn({ - cols: 120, - rows: 40, - cwd, - command: sequencedStartupCommand - ? launchOpts.command - : (agentTeamsPlan?.command ?? launchOpts.command), - commandDelivery: 'provider', - startupCommandDelivery: launchOpts.startupCommandDelivery, - env, - envToDelete: agentTeamsPlan?.envToDelete, - telemetry: launchOpts.telemetry, - connectionId: workspace.connectionId, - worktreeId: workspace.id, - preAllocatedHandle, - tabId, - leafId, - ...(launchOpts.sessionId ? { sessionId: launchOpts.sessionId } : {}), - ...(launchOpts.persistHostSessionBinding ? { persistHostSessionBinding: true } : {}) - }) - this.registerPreAllocatedHandleForPty(result.id, preAllocatedHandle) - this.registerPty(result.id, workspace.id, workspace.connectionId) - const pty = this.getOrCreatePtyWorktreeRecord(result.id) - if (pty) { - if (launchOpts.title) { - const observedAt = this.nextTitleObservationSequence() - pty.title = launchOpts.title - pty.titleUpdatedAt = observedAt - this.setPtyManagementTitleFromObservedTitle(pty, launchOpts.title, observedAt) - } else { - pty.title = null - pty.titleUpdatedAt = null + const baseEnv = { + ...launchOpts.env, + ...(launchToken ? { ORCA_AGENT_LAUNCH_TOKEN: launchToken } : {}) } - pty.tabId = tabId - pty.paneKey = paneKey - pty.launchConfig = effectiveLaunchConfig - ? copySleepingAgentLaunchConfig(effectiveLaunchConfig) - : null - pty.launchToken = launchToken ?? null - pty.launchAgent = launchOpts.launchAgent ?? null - } - const handle = pty ? this.issuePtyHandle(pty) : preAllocatedHandle - if (pty && launchOpts.deferMobileSessionPublish !== true) { - this.publishPtyBackedMobileSessionTerminal(workspace.id, pty, { + const claudeAgentTeamsSourceCommand = + launchOpts.claudeAgentTeamsSourceCommand?.trim() || + launchOpts.command?.trim() || + undefined + const claudeAgentTeamsMode = this.store?.getSettings?.().claudeAgentTeamsMode + const effectiveClaudeAgentTeamsMode = inferCapturedClaudeAgentTeamsMode( + launchOpts.launchConfig, + claudeAgentTeamsSourceCommand, + claudeAgentTeamsMode + ) + const agentTeamsPlan = await buildClaudeAgentTeamsLaunchPlan({ + command: claudeAgentTeamsSourceCommand, + mode: effectiveClaudeAgentTeamsMode, + baseEnv: { + ...process.env, + ...baseEnv + }, + createTeamEnv: (shimDir, shimBin) => + this.claudeAgentTeams.createLaunchEnv({ + leaderHandle: preAllocatedHandle, + baseEnv: { + ...process.env, + ...baseEnv + }, + shimDir, + shimBin + }).env + }) + const sequencedStartupCommand = + agentTeamsPlan && + claudeAgentTeamsSourceCommand && + launchOpts.command && + claudeAgentTeamsSourceCommand !== launchOpts.command + ? agentTeamsPlan.command + : undefined + const effectiveLaunchConfig = + launchOpts.launchConfig && agentTeamsPlan + ? { + ...launchOpts.launchConfig, + agentCommand: launchOpts.launchConfig.agentCommand + ? effectiveClaudeAgentTeamsMode === 'in-process' || process.platform === 'win32' + ? addClaudeTeammateModeInProcess(launchOpts.launchConfig.agentCommand) + : addClaudeTeammateModeAuto(launchOpts.launchConfig.agentCommand) + : agentTeamsPlan.command, + agentEnv: { + ...launchOpts.launchConfig.agentEnv, + ...agentTeamsPlan.env + } + } + : launchOpts.launchConfig + // Why: setup/agent sequencing wraps the PTY launch in a wait shell before + // Claude Agent Teams runs. Preserve the direct Claude command separately + // so the wrapper can exec the teammate-mode variant after setup completes. + const env = this.buildTerminalWorkspaceEnv( + workspace, + { + ...baseEnv, + ...(sequencedStartupCommand + ? { [SETUP_AGENT_SEQUENCE_STARTUP_COMMAND_ENV]: sequencedStartupCommand } + : {}) + }, + paneKey, + tabId, + agentTeamsPlan?.env + ) + const result = await this.spawnPtyWithAdmissionHeld({ + cols: 120, + rows: 40, + cwd, + command: sequencedStartupCommand + ? launchOpts.command + : (agentTeamsPlan?.command ?? launchOpts.command), + commandDelivery: 'provider', + startupCommandDelivery: launchOpts.startupCommandDelivery, + env, + envToDelete: agentTeamsPlan?.envToDelete, + telemetry: launchOpts.telemetry, + connectionId: workspace.connectionId, + worktreeId: workspace.id, + preAllocatedHandle, tabId, leafId, - title: launchOpts.title ?? null, - activate: presentation === 'focused', - // Why: explicit background presentation may carry legacy activate - // metadata from an already-owned renderer pane; don't select it on mobile. - selectIfNoActiveTab: presentation !== 'background', - ...(cwd !== workspace.path ? { startupCwd: cwd } : {}) + ...(launchOpts.sessionId ? { sessionId: launchOpts.sessionId } : {}), + ...(launchOpts.persistHostSessionBinding ? { persistHostSessionBinding: true } : {}) }) - } - let surface: RuntimeTerminalCreate['surface'] = 'background' - let warning: string | undefined - if (presentation !== 'background' && this.notifier?.revealTerminalSession) { - try { - // Why: after the PTY is spawned, renderer tab adoption is best-effort; - // failing here must not strand a live process without returning a handle. - // Pass the pre-minted tabId so the renderer adopts under the same id - // already baked into the PTY env — keeps paneKey hook attribution intact. - await this.notifier.revealTerminalSession(workspace.id, { - ptyId: result.id, + this.registerPreAllocatedHandleForPty(result.id, preAllocatedHandle) + this.registerPty(result.id, workspace.id, workspace.connectionId) + const pty = this.getOrCreatePtyWorktreeRecord(result.id) + if (pty) { + if (launchOpts.title) { + const observedAt = this.nextTitleObservationSequence() + pty.title = launchOpts.title + pty.titleUpdatedAt = observedAt + this.setPtyManagementTitleFromObservedTitle(pty, launchOpts.title, observedAt) + } else { + pty.title = null + pty.titleUpdatedAt = null + } + pty.tabId = tabId + pty.paneKey = paneKey + pty.launchConfig = effectiveLaunchConfig + ? copySleepingAgentLaunchConfig(effectiveLaunchConfig) + : null + pty.launchToken = launchToken ?? null + pty.launchAgent = launchOpts.launchAgent ?? null + } + const handle = pty ? this.issuePtyHandle(pty) : preAllocatedHandle + if (pty && launchOpts.deferMobileSessionPublish !== true) { + this.publishPtyBackedMobileSessionTerminal(workspace.id, pty, { + tabId, + leafId, title: launchOpts.title ?? null, - ...(cwd !== workspace.path ? { cwd } : {}), - ...(effectiveLaunchConfig ? { launchConfig: effectiveLaunchConfig } : {}), - ...(launchToken ? { launchToken } : {}), - ...(launchOpts.launchAgent ? { launchAgent: launchOpts.launchAgent } : {}), activate: presentation === 'focused', - ...(presentation ? { presentation } : {}), - tabId, - leafId + // Why: explicit background presentation may carry legacy activate + // metadata from an already-owned renderer pane; don't select it on mobile. + selectIfNoActiveTab: presentation !== 'background', + ...(cwd !== workspace.path ? { startupCwd: cwd } : {}) }) - surface = 'visible' - } catch (err) { - console.warn(`[terminal-create] failed to create inactive tab for ${result.id}:`, err) - warning = createTerminalRevealWarning(handle, err) } - } else if (presentation !== 'background') { - warning = createTerminalRevealWarning(handle) - } - return { - handle, - tabId, - paneKey, - ptyId: result.id, - worktreeId: workspace.id, - title: launchOpts.title ?? null, - surface, - ...(warning ? { warning } : {}) - } + let surface: RuntimeTerminalCreate['surface'] = 'background' + let warning: string | undefined + if (presentation !== 'background' && this.notifier?.revealTerminalSession) { + try { + // Why: after the PTY is spawned, renderer tab adoption is best-effort; + // failing here must not strand a live process without returning a handle. + // Pass the pre-minted tabId so the renderer adopts under the same id + // already baked into the PTY env — keeps paneKey hook attribution intact. + await this.notifier.revealTerminalSession(workspace.id, { + ptyId: result.id, + title: launchOpts.title ?? null, + ...(cwd !== workspace.path ? { cwd } : {}), + ...(effectiveLaunchConfig ? { launchConfig: effectiveLaunchConfig } : {}), + ...(launchToken ? { launchToken } : {}), + ...(launchOpts.launchAgent ? { launchAgent: launchOpts.launchAgent } : {}), + activate: presentation === 'focused', + ...(presentation ? { presentation } : {}), + tabId, + leafId + }) + surface = 'visible' + } catch (err) { + console.warn(`[terminal-create] failed to create inactive tab for ${result.id}:`, err) + warning = createTerminalRevealWarning(handle, err) + } + } else if (presentation !== 'background') { + warning = createTerminalRevealWarning(handle) + } + return { + handle, + tabId, + paneKey, + ptyId: result.id, + worktreeId: workspace.id, + title: launchOpts.title ?? null, + surface, + ...(warning ? { warning } : {}) + } + }) } this.assertGraphReady() @@ -18597,64 +18626,70 @@ export class OrcaRuntimeService { } const direction = opts.direction ?? 'horizontal' const workspace = await this.resolveTerminalWorkspaceLaunchScope(`id:${pty.worktreeId}`) - const leafId = randomUUID() - const preAllocatedHandle = this.createPreAllocatedTerminalHandle() - const paneKey = makePaneKey(parentTabId, leafId) - const result = await this.ptyController.spawn({ - cols: 120, - rows: 40, - cwd: workspace.path, - command: opts.command, - commandDelivery: 'provider', - env: this.buildTerminalWorkspaceEnv(workspace, opts.env ?? {}, paneKey, parentTabId), - envToDelete: opts.envToDelete, - connectionId: workspace.connectionId, - worktreeId: workspace.id, - preAllocatedHandle - }) - this.registerPreAllocatedHandleForPty(result.id, preAllocatedHandle) - this.registerPty(result.id, workspace.id, workspace.connectionId) - const createdPty = this.getOrCreatePtyWorktreeRecord(result.id) - if (createdPty) { - createdPty.tabId = parentTabId - createdPty.paneKey = paneKey - } - - try { - await this.notifier?.revealTerminalSession?.(workspace.id, { - ptyId: result.id, - title: null, - activate: opts.activate !== false, - tabId: parentTabId, - leafId, - splitFromLeafId: parsedPaneKey.leafId, - splitDirection: direction, - splitTelemetrySource: opts.telemetrySource - }) - } catch (error) { - this.ptyController.kill?.(result.id) - throw error - } - if (createdPty) { - this.publishPtyBackedMobileSessionTerminal(workspace.id, createdPty, { - tabId: parentTabId, - leafId, - title: null, - activate: opts.activate !== false, - split: { splitFromLeafId: parsedPaneKey.leafId, direction } - }) - // Why: persist the split into the workspace session so a later snapshot - // rebuild keeps it instead of collapsing back to a single pane. - this.persistHeadlessTerminalSplit({ - tabId: parentTabId, - leafId, - ptyId: createdPty.ptyId, - splitFromLeafId: parsedPaneKey.leafId, - direction + return await this.runWithWorktreePtySpawn(workspace.id, async () => { + const leafId = randomUUID() + const preAllocatedHandle = this.createPreAllocatedTerminalHandle() + const paneKey = makePaneKey(parentTabId, leafId) + const result = await this.spawnPtyWithAdmissionHeld({ + cols: 120, + rows: 40, + cwd: workspace.path, + command: opts.command, + commandDelivery: 'provider', + env: this.buildTerminalWorkspaceEnv(workspace, opts.env ?? {}, paneKey, parentTabId), + envToDelete: opts.envToDelete, + connectionId: workspace.connectionId, + worktreeId: workspace.id, + preAllocatedHandle }) - } + this.registerPreAllocatedHandleForPty(result.id, preAllocatedHandle) + this.registerPty(result.id, workspace.id, workspace.connectionId) + const createdPty = this.getOrCreatePtyWorktreeRecord(result.id) + if (createdPty) { + createdPty.tabId = parentTabId + createdPty.paneKey = paneKey + } - return { handle: this.issuePtyHandle(createdPty ?? pty), tabId: parentTabId, paneRuntimeId: -1 } + try { + await this.notifier?.revealTerminalSession?.(workspace.id, { + ptyId: result.id, + title: null, + activate: opts.activate !== false, + tabId: parentTabId, + leafId, + splitFromLeafId: parsedPaneKey.leafId, + splitDirection: direction, + splitTelemetrySource: opts.telemetrySource + }) + } catch (error) { + this.ptyController?.kill(result.id) + throw error + } + if (createdPty) { + this.publishPtyBackedMobileSessionTerminal(workspace.id, createdPty, { + tabId: parentTabId, + leafId, + title: null, + activate: opts.activate !== false, + split: { splitFromLeafId: parsedPaneKey.leafId, direction } + }) + // Why: persist the split into the workspace session so a later snapshot + // rebuild keeps it instead of collapsing back to a single pane. + this.persistHeadlessTerminalSplit({ + tabId: parentTabId, + leafId, + ptyId: createdPty.ptyId, + splitFromLeafId: parsedPaneKey.leafId, + direction + }) + } + + return { + handle: this.issuePtyHandle(createdPty ?? pty), + tabId: parentTabId, + paneRuntimeId: -1 + } + }) } async handleAgentTeamsTmuxCompat( @@ -18746,7 +18781,45 @@ export class OrcaRuntimeService { }) } - async stopTerminalsForWorktree(worktreeSelector: string): Promise<{ stopped: number }> { + beginWorktreePtySpawn(worktreeId: string): () => void { + return this.worktreePtyAdmission.beginSpawn(worktreeId) + } + + private async runWithWorktreePtySpawn( + worktreeId: string, + operation: () => Promise + ): Promise { + const release = this.beginWorktreePtySpawn(worktreeId) + try { + return await operation() + } finally { + release() + } + } + + private async spawnPtyWithAdmissionHeld(opts: RuntimePtySpawnOptions): Promise<{ id: string }> { + const controller = this.ptyController + const spawn = controller?.spawnWithWorktreePtyAdmissionHeld ?? controller?.spawn + if (!controller || !spawn) { + throw new Error('runtime_unavailable') + } + return await spawn.call(controller, opts) + } + + async beginWorktreePtyTeardown(worktreeId: string): Promise<() => void> { + return this.worktreePtyAdmission.closeForTeardown(worktreeId) + } + + async runWithWorktreePtyTeardown(worktreeId: string, operation: () => Promise): Promise { + // Why: closing admission before snapshots makes deletion own every spawn + // already in flight and keeps new desktop/mobile/SSH spawns out until Git settles. + return this.worktreePtyAdmission.runTeardown(worktreeId, operation) + } + + async stopTerminalsForWorktree( + worktreeSelector: string, + opts: { waitForProviderShutdown?: boolean } = {} + ): Promise<{ stopped: number }> { // Why: this mutates live PTYs, so the runtime must reject it while the // renderer graph is reloading instead of acting on cached leaf ownership. const graphEpoch = this.captureReadyGraphEpoch() @@ -18769,7 +18842,9 @@ export class OrcaRuntimeService { // Why: worktree removal immediately follows this with provider/registry // sweeps. Await the verified stop so those sweeps cannot overlap a // provider shutdown with a second ConPTY teardown on Windows (#8275). - if (this.ptyController?.stopAndWait) { + // Why: deletion needs provider acknowledgement before its fallback sweeps, + // while the shared terminal.stop RPC must preserve graceful shell teardown. + if (opts.waitForProviderShutdown && this.ptyController?.stopAndWait) { if (await this.ptyController.stopAndWait(ptyId)) { stopped += 1 } diff --git a/src/main/runtime/rpc/terminal-stop.test.ts b/src/main/runtime/rpc/terminal-stop.test.ts new file mode 100644 index 00000000000..29ce3a7f78e --- /dev/null +++ b/src/main/runtime/rpc/terminal-stop.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it, vi } from 'vitest' +import type { OrcaRuntimeService } from '../orca-runtime' +import type { RpcRequest } from './core' +import { RpcDispatcher } from './dispatcher' +import { TERMINAL_METHODS } from './methods/terminal' + +describe('terminal stop RPC', () => { + it('uses the generic graceful worktree-stop mode', async () => { + const stopTerminalsForWorktree = vi.fn().mockResolvedValue({ stopped: 2 }) + const runtime = { + getRuntimeId: () => 'test-runtime', + stopTerminalsForWorktree + } as unknown as OrcaRuntimeService + const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS }) + const request: RpcRequest = { + id: 'req-1', + authToken: 'tok', + method: 'terminal.stop', + params: { worktree: 'repo-1::/worktree' } + } + + const response = await dispatcher.dispatch(request) + + expect(response.ok).toBe(true) + expect(stopTerminalsForWorktree).toHaveBeenCalledWith('repo-1::/worktree') + }) +}) diff --git a/src/main/runtime/worktree-pty-admission.test.ts b/src/main/runtime/worktree-pty-admission.test.ts new file mode 100644 index 00000000000..35bf926a162 --- /dev/null +++ b/src/main/runtime/worktree-pty-admission.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it, vi } from 'vitest' +import { WorktreePtyAdmission } from './worktree-pty-admission' + +describe('WorktreePtyAdmission', () => { + it('drains admitted spawns and rejects new ones until teardown settles', async () => { + const admission = new WorktreePtyAdmission() + const releaseSpawn = admission.beginSpawn('w1') + let releaseTeardown = () => {} + const operation = vi.fn( + () => + new Promise((resolve) => { + releaseTeardown = () => resolve('removed') + }) + ) + + const teardown = admission.runTeardown('w1', operation) + expect(operation).not.toHaveBeenCalled() + expect(() => admission.beginSpawn('w1')).toThrow('Worktree teardown is in progress') + + releaseSpawn() + await vi.waitFor(() => expect(operation).toHaveBeenCalledTimes(1)) + expect(() => admission.beginSpawn('w1')).toThrow('Worktree teardown is in progress') + releaseTeardown() + + await expect(teardown).resolves.toBe('removed') + const releaseNextSpawn = admission.beginSpawn('w1') + releaseNextSpawn() + }) + + it('keeps different worktrees independent', async () => { + const admission = new WorktreePtyAdmission() + const releaseSpawn = admission.beginSpawn('w1') + const teardown = admission.runTeardown('w1', async () => undefined) + + const releaseOtherSpawn = admission.beginSpawn('w2') + releaseOtherSpawn() + releaseSpawn() + + await expect(teardown).resolves.toBeUndefined() + }) + + it('serializes teardown owners from different removal entry points', async () => { + const admission = new WorktreePtyAdmission() + let releaseFirst = () => {} + const order: string[] = [] + const first = admission.runTeardown( + 'w1', + () => + new Promise((resolve) => { + order.push('first') + releaseFirst = resolve + }) + ) + const second = admission.runTeardown('w1', async () => { + order.push('second') + }) + + await vi.waitFor(() => expect(order).toEqual(['first'])) + releaseFirst() + + await Promise.all([first, second]) + expect(order).toEqual(['first', 'second']) + }) +}) diff --git a/src/main/runtime/worktree-pty-admission.ts b/src/main/runtime/worktree-pty-admission.ts new file mode 100644 index 00000000000..7dadd76c8b2 --- /dev/null +++ b/src/main/runtime/worktree-pty-admission.ts @@ -0,0 +1,92 @@ +type WorktreePtyAdmissionState = { + activeSpawns: number + teardownOwners: number + drainWaiters: Set<() => void> + teardownTail: Promise +} + +export class WorktreePtyAdmission { + private readonly states = new Map() + + beginSpawn(worktreeId: string): () => void { + // Why: the lease spans provider creation through runtime registration, so + // deletion can drain every PTY that could become visible after its snapshot. + const state = this.getOrCreate(worktreeId) + if (state.teardownOwners > 0) { + throw new Error(`Worktree teardown is in progress: ${worktreeId}`) + } + state.activeSpawns += 1 + let released = false + return () => { + if (released) { + return + } + released = true + state.activeSpawns -= 1 + if (state.activeSpawns === 0) { + for (const resolve of state.drainWaiters) { + resolve() + } + state.drainWaiters.clear() + } + this.deleteIfIdle(worktreeId, state) + } + } + + async runTeardown(worktreeId: string, operation: () => Promise): Promise { + const release = await this.closeForTeardown(worktreeId) + try { + return await operation() + } finally { + release() + } + } + + async closeForTeardown(worktreeId: string): Promise<() => void> { + const state = this.getOrCreate(worktreeId) + state.teardownOwners += 1 + const previousTeardown = state.teardownTail + let finishTeardown = () => {} + const thisTeardown = new Promise((resolve) => { + finishTeardown = resolve + }) + // Why: runtime RPC and renderer IPC have distinct dedupe maps; serialize + // them here so two removal owners cannot overlap provider or Git teardown. + state.teardownTail = previousTeardown.then(() => thisTeardown) + await previousTeardown + if (state.activeSpawns > 0) { + await new Promise((resolve) => state.drainWaiters.add(resolve)) + } + let released = false + return () => { + if (released) { + return + } + released = true + state.teardownOwners -= 1 + finishTeardown() + this.deleteIfIdle(worktreeId, state) + } + } + + private getOrCreate(worktreeId: string): WorktreePtyAdmissionState { + const existing = this.states.get(worktreeId) + if (existing) { + return existing + } + const state: WorktreePtyAdmissionState = { + activeSpawns: 0, + teardownOwners: 0, + drainWaiters: new Set(), + teardownTail: Promise.resolve() + } + this.states.set(worktreeId, state) + return state + } + + private deleteIfIdle(worktreeId: string, state: WorktreePtyAdmissionState): void { + if (state.activeSpawns === 0 && state.teardownOwners === 0) { + this.states.delete(worktreeId) + } + } +} diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index e4caf2d6b3a..a5a1af380be 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -157,7 +157,9 @@ describe('killAllProcessesForWorktree', () => { const result = await killAllProcessesForWorktree('w1', { runtime, localProvider }) - expect(stopTerminalsForWorktree).toHaveBeenCalledWith('w1') + expect(stopTerminalsForWorktree).toHaveBeenCalledWith('w1', { + waitForProviderShutdown: true + }) expect(result.runtimeStopped).toBe(3) }) diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index 7572b9bbd77..3289a56d1de 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -47,7 +47,9 @@ export async function killAllProcessesForWorktree( } if (deps.runtime) { - const r = await deps.runtime.stopTerminalsForWorktree(worktreeId).catch(() => ({ stopped: 0 })) + const r = await deps.runtime + .stopTerminalsForWorktree(worktreeId, { waitForProviderShutdown: true }) + .catch(() => ({ stopped: 0 })) result.runtimeStopped = r.stopped } From 9ea9f2eab90dccf78b3f1e2d78682b7410a53fe2 Mon Sep 17 00:00:00 2001 From: Brennan Benson Date: Sun, 12 Jul 2026 13:30:15 -0700 Subject: [PATCH 06/36] fix(runtime): cover headless worktree PTY teardown --- src/main/ipc/pty.test.ts | 155 ++++++++++++++++++++- src/main/ipc/pty.ts | 27 +++- src/main/runtime/orca-runtime.test.ts | 44 +++--- src/main/runtime/orca-runtime.ts | 22 +-- src/main/runtime/worktree-teardown.test.ts | 4 +- src/main/runtime/worktree-teardown.ts | 2 +- 6 files changed, 211 insertions(+), 43 deletions(-) diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index 424f9a26815..280d727e904 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -209,6 +209,7 @@ import { isHiddenRendererPty } from './pty-hidden-delivery-gate' import { OrcaRuntimeService } from '../runtime/orca-runtime' +import { WorktreePtyAdmission } from '../runtime/worktree-pty-admission' import { hasLiveClaudePtys, markClaudePtySpawned } from '../claude-accounts/live-pty-gate' import * as livePtyGate from '../claude-accounts/live-pty-gate' import { @@ -217,8 +218,10 @@ import { } from '../powershell-osc133-bootstrap' import { SSH_PTY_IDENTITY_MISMATCH_ERROR, - SSH_SESSION_EXPIRED_ERROR + SSH_SESSION_EXPIRED_ERROR, + SshPtyProvider } from '../providers/ssh-pty-provider' +import { JsonRpcErrorCode } from '../ssh/relay-protocol' import { _resetWslCachesForTests, _setWslCachesForTests } from '../wsl' const POWERSHELL_OSC133_ARGS = [ @@ -856,6 +859,49 @@ describe('registerPtyHandlers', () => { expect(releaseWorktreeSpawn).toHaveBeenCalledTimes(1) }) + it('blocks deletion while renderer launch preparation is still in flight', async () => { + const startup = makeDeferred() + const admission = new WorktreePtyAdmission() + const order: string[] = [] + const runtime = { + setPtyController: vi.fn(), + beginWorktreePtySpawn: (worktreeId: string) => admission.beginSpawn(worktreeId), + preAllocateHandleForPty: vi.fn(() => undefined), + registerPty: vi.fn(() => order.push('registered')), + noteTerminalSpawnCommand: vi.fn(), + onPtySpawned: vi.fn(), + onPtyExit: vi.fn(), + onPtyData: vi.fn() + } + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + undefined, + { awaitLocalPtyStartup: () => startup.promise } + ) + + const spawning = handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24, + cwd: '/outside/worktree', + worktreeId: 'wt-renderer-prep' + }) as Promise<{ id: string }> + const deletion = admission.runTeardown('wt-renderer-prep', async () => { + order.push('deleted') + }) + await Promise.resolve() + + expect(order).toEqual([]) + startup.resolve() + + await spawning + await deletion + expect(order).toEqual(['registered', 'deleted']) + }) + it('marks local Claude launches live until the PTY is killed', async () => { const prepareClaudeAuth = vi.fn(async () => ({ configDir: '/tmp/claude', @@ -2845,6 +2891,113 @@ describe('registerPtyHandlers', () => { expect(listProcesses).not.toHaveBeenCalled() }) + it('finishes headless current-relay SSH shutdown before remote removal without inventory', async () => { + const order: string[] = [] + const request = vi.fn(async (method: string) => { + if (method === 'pty.shutdown') { + order.push('shutdown') + return undefined + } + if (method === 'pty.hasPty') { + order.push('hasPty') + return false + } + throw new Error(`Unexpected request: ${method}`) + }) + const provider = new SshPtyProvider('ssh-1', { + request, + notify: vi.fn(), + onNotification: vi.fn(() => () => {}), + dispose: vi.fn(), + isDisposed: vi.fn(() => false) + } as never) + const runtime = new OrcaRuntimeService() + Object.assign(runtime, { + resolveWorktreeSelector: vi.fn(async () => ({ id: 'wt-headless-current' })) + }) + registerSshPtyProvider('ssh-1', provider) + setPtyOwnership('ssh:ssh-1@@headless-current', 'ssh-1') + handlers.clear() + registerPtyHandlers(mainWindow as never, runtime as never) + runtime.registerPty('ssh:ssh-1@@headless-current', 'wt-headless-current', 'ssh-1') + + await runtime.runWithWorktreePtyTeardown('wt-headless-current', async () => { + await expect( + runtime.stopTerminalsForWorktree('id:wt-headless-current', { + worktreeTeardown: true + }) + ).resolves.toEqual({ stopped: 1 }) + order.push('git-remove') + }) + + expect(order).toEqual(['shutdown', 'hasPty', 'git-remove']) + expect( + request.mock.calls.filter(([method]) => method === 'pty.listProcesses') + ).toHaveLength(0) + deletePtyOwnership('ssh:ssh-1@@headless-current') + }) + + it('bounds headless legacy-relay SSH verification to one inventory before removal', async () => { + const order: string[] = [] + const request = vi.fn(async (method: string) => { + if (method === 'pty.shutdown') { + order.push('shutdown') + return undefined + } + if (method === 'pty.hasPty') { + order.push('hasPty-miss') + throw Object.assign(new Error('Method not found'), { + code: JsonRpcErrorCode.MethodNotFound + }) + } + if (method === 'pty.listProcesses') { + order.push('inventory') + return [] + } + throw new Error(`Unexpected request: ${method}`) + }) + const provider = new SshPtyProvider('ssh-1', { + request, + notify: vi.fn(), + onNotification: vi.fn(() => () => {}), + dispose: vi.fn(), + isDisposed: vi.fn(() => false) + } as never) + const runtime = new OrcaRuntimeService() + Object.assign(runtime, { + resolveWorktreeSelector: vi.fn(async () => ({ id: 'wt-headless-legacy' })) + }) + registerSshPtyProvider('ssh-1', provider) + const ptyIds = ['ssh:ssh-1@@headless-legacy-1', 'ssh:ssh-1@@headless-legacy-2'] + for (const ptyId of ptyIds) { + setPtyOwnership(ptyId, 'ssh-1') + } + handlers.clear() + registerPtyHandlers(mainWindow as never, runtime as never) + for (const ptyId of ptyIds) { + runtime.registerPty(ptyId, 'wt-headless-legacy', 'ssh-1') + } + + await runtime.runWithWorktreePtyTeardown('wt-headless-legacy', async () => { + await expect( + runtime.stopTerminalsForWorktree('id:wt-headless-legacy', { + worktreeTeardown: true + }) + ).resolves.toEqual({ stopped: 2 }) + order.push('git-remove') + }) + + expect(order.at(-1)).toBe('git-remove') + expect(order.filter((step) => step === 'shutdown')).toHaveLength(2) + expect(request.mock.calls.filter(([method]) => method === 'pty.hasPty')).toHaveLength(1) + expect( + request.mock.calls.filter(([method]) => method === 'pty.listProcesses') + ).toHaveLength(1) + for (const ptyId of ptyIds) { + deletePtyOwnership(ptyId) + } + }) + it('passes keepHistory through runtime controller stopAndWait', async () => { vi.useFakeTimers() const shutdown = vi.fn(async () => undefined) diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 961313462bd..4d9e6edc360 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -3603,7 +3603,26 @@ export function registerPtyHandlers( resetPtyRendererDeliveryDebug() }) - ipcMain.handle( + const registerWorktreePtySpawnHandler = ( + channel: string, + handler: (event: IpcMainInvokeEvent, args: T) => Promise + ): void => { + ipcMain.handle(channel, async (event, args: T) => { + const releaseWorktreeSpawn = + typeof args.worktreeId === 'string' && runtime?.beginWorktreePtySpawn + ? runtime.beginWorktreePtySpawn(args.worktreeId) + : null + try { + return await handler(event, args) + } finally { + // Why: deletion must own renderer launch preparation as well as provider + // spawn and registration, including failures before a PTY id exists. + releaseWorktreeSpawn?.() + } + }) + } + + registerWorktreePtySpawnHandler( 'pty:spawn', async ( _event, @@ -4027,10 +4046,6 @@ export function registerPtyHandlers( if (existingPaneSpawn) { return await existingPaneSpawn.promise } - const releaseWorktreeSpawn = - typeof args.worktreeId === 'string' && runtime?.beginWorktreePtySpawn - ? runtime.beginWorktreePtySpawn(args.worktreeId) - : null const paneSpawnReservation = reservationPaneKey ? reservePaneSpawn(reservationPaneKey) : null const initiallyHidden = args.initiallyHidden === true // Why pre-spawn for daemon-host sessions (id minted up front): daemon @@ -4433,8 +4448,6 @@ export function registerPtyHandlers( // no-op once the reservation has already resolved. rejectPaneSpawnReservation(reservationPaneKey, paneSpawnReservation, err) throw err - } finally { - releaseWorktreeSpawn?.() } } ) diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index f08b1e280ea..3e368074402 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -22214,6 +22214,25 @@ describe('OrcaRuntimeService', () => { expect(killed).toBe(false) }) + it('uses authoritative runtime PTYs for teardown while the renderer graph is reloading', async () => { + const stopAndWait = vi.fn(async () => true) + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + write: () => true, + kill: () => false, + stopAndWait, + getForegroundProcess: async () => null + }) + runtime.registerPty('ssh:ssh-1@@reloading', TEST_WORKTREE_ID, 'ssh-1') + runtime.attachWindow(1) + runtime.markRendererReloading(1) + + await expect( + runtime.stopTerminalsForWorktree(TEST_WORKTREE_ID, { worktreeTeardown: true }) + ).resolves.toEqual({ stopped: 1 }) + expect(stopAndWait).toHaveBeenCalledWith('ssh:ssh-1@@reloading') + }) + it('fails terminal listing closed if the graph reloads during selector resolution', async () => { const runtime = new OrcaRuntimeService(store) @@ -22324,7 +22343,7 @@ describe('OrcaRuntimeService', () => { let completed = false const stopping = runtime - .stopTerminalsForWorktree(TEST_WORKTREE_ID, { waitForProviderShutdown: true }) + .stopTerminalsForWorktree(TEST_WORKTREE_ID, { worktreeTeardown: true }) .then((result) => { completed = true return result @@ -29458,7 +29477,7 @@ describe('OrcaRuntimeService', () => { expect(preDaemonProvider.shutdown).not.toHaveBeenCalled() }) - it('drains a late SSH spawn and stops it before remote worktree removal', async () => { + it('drains headless and late SSH spawns before remote worktree removal', async () => { const remoteRepo = { ...store.getRepo(TEST_REPO_ID)!, path: '/remote/repo', @@ -29518,27 +29537,6 @@ describe('OrcaRuntimeService', () => { ), getForegroundProcess: async () => null }) - runtime.attachWindow(1) - runtime.syncWindowGraph(1, { - tabs: [ - { - tabId: 'tab-remote', - worktreeId: remoteWorktreeId, - title: 'Terminal', - activeLeafId: 'pane:remote', - layout: null - } - ], - leaves: [ - { - tabId: 'tab-remote', - worktreeId: remoteWorktreeId, - leafId: 'pane:remote', - paneRuntimeId: 1, - ptyId: 'ssh:ssh-1@@existing' - } - ] - }) runtime.registerPty('ssh:ssh-1@@existing', remoteWorktreeId, 'ssh-1') const releaseLateSpawn = runtime.beginWorktreePtySpawn(remoteWorktreeId) diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 4be376db10c..bd6941eb996 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -18818,17 +18818,21 @@ export class OrcaRuntimeService { async stopTerminalsForWorktree( worktreeSelector: string, - opts: { waitForProviderShutdown?: boolean } = {} + opts: { worktreeTeardown?: boolean } = {} ): Promise<{ stopped: number }> { - // Why: this mutates live PTYs, so the runtime must reject it while the - // renderer graph is reloading instead of acting on cached leaf ownership. - const graphEpoch = this.captureReadyGraphEpoch() + // Why: generic terminal.stop must not act on a stale renderer graph, while + // deletion runs under admission and owns main's authoritative PTY records. + const graphEpoch = opts.worktreeTeardown ? null : this.captureReadyGraphEpoch() const worktree = await this.resolveWorktreeSelector(worktreeSelector) - this.assertStableReadyGraph(graphEpoch) + if (graphEpoch !== null) { + this.assertStableReadyGraph(graphEpoch) + } const ptyIds = new Set() - for (const leaf of this.leaves.values()) { - if (leaf.worktreeId === worktree.id && leaf.ptyId) { - ptyIds.add(leaf.ptyId) + if (!opts.worktreeTeardown) { + for (const leaf of this.leaves.values()) { + if (leaf.worktreeId === worktree.id && leaf.ptyId) { + ptyIds.add(leaf.ptyId) + } } } for (const pty of this.ptysById.values()) { @@ -18844,7 +18848,7 @@ export class OrcaRuntimeService { // provider shutdown with a second ConPTY teardown on Windows (#8275). // Why: deletion needs provider acknowledgement before its fallback sweeps, // while the shared terminal.stop RPC must preserve graceful shell teardown. - if (opts.waitForProviderShutdown && this.ptyController?.stopAndWait) { + if (opts.worktreeTeardown && this.ptyController?.stopAndWait) { if (await this.ptyController.stopAndWait(ptyId)) { stopped += 1 } diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index a5a1af380be..4ba3d4c9050 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -158,7 +158,7 @@ describe('killAllProcessesForWorktree', () => { const result = await killAllProcessesForWorktree('w1', { runtime, localProvider }) expect(stopTerminalsForWorktree).toHaveBeenCalledWith('w1', { - waitForProviderShutdown: true + worktreeTeardown: true }) expect(result.runtimeStopped).toBe(3) }) @@ -212,7 +212,7 @@ describe('killAllProcessesForWorktree', () => { expect(liveSessions.has('witness@@unrelated')).toBe(true) }) - it('tolerates runtime.stopTerminalsForWorktree throwing (headless assertGraphReady reject)', async () => { + it('tolerates an unexpected runtime worktree-stop failure', async () => { const stopTerminalsForWorktree = vi.fn().mockRejectedValue(new Error('graph not ready')) const runtime = { stopTerminalsForWorktree diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index 3289a56d1de..1085ae30682 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -48,7 +48,7 @@ export async function killAllProcessesForWorktree( if (deps.runtime) { const r = await deps.runtime - .stopTerminalsForWorktree(worktreeId, { waitForProviderShutdown: true }) + .stopTerminalsForWorktree(worktreeId, { worktreeTeardown: true }) .catch(() => ({ stopped: 0 })) result.runtimeStopped = r.stopped } From 0c9809e6be72633bb8bf3882bb8525b4a1c3c0b9 Mon Sep 17 00:00:00 2001 From: Brennan Benson Date: Sun, 12 Jul 2026 14:15:10 -0700 Subject: [PATCH 07/36] fix: bound terminal teardown verification --- src/main/daemon/daemon-pty-router.test.ts | 37 +++++++ src/main/daemon/daemon-pty-router.ts | 19 +++- .../degraded-daemon-pty-provider.test.ts | 59 ++++++++++++ .../daemon/degraded-daemon-pty-provider.ts | 56 ++++++----- .../degraded-daemon-session-reconciliation.ts | 23 +++++ .../daemon/provider-replay-subscription.ts | 25 +++++ .../shutdown-verification-owner-cache.ts | 30 ++++++ src/main/ipc/worktrees.test.ts | 96 +++++++++++++++++++ src/main/ipc/worktrees.ts | 48 +++++++++- src/main/runtime/orca-runtime.test.ts | 44 +++++++++ src/main/runtime/orca-runtime.ts | 71 +++++++++++++- .../runtime/worktree-pty-admission.test.ts | 14 +-- src/main/runtime/worktree-pty-admission.ts | 19 ++-- 13 files changed, 488 insertions(+), 53 deletions(-) create mode 100644 src/main/daemon/degraded-daemon-session-reconciliation.ts create mode 100644 src/main/daemon/provider-replay-subscription.ts create mode 100644 src/main/daemon/shutdown-verification-owner-cache.ts diff --git a/src/main/daemon/daemon-pty-router.test.ts b/src/main/daemon/daemon-pty-router.test.ts index eb8acb86488..a5de628ef65 100644 --- a/src/main/daemon/daemon-pty-router.test.ts +++ b/src/main/daemon/daemon-pty-router.test.ts @@ -239,6 +239,43 @@ describe('DaemonPtyRouter', () => { expect(current.hasPty).not.toHaveBeenCalledWith('legacy-session') }) + it('keeps multi-PTY post-shutdown verification O(T) across retained adapters', async () => { + const adapters = Array.from({ length: 21 }, (_, index) => createAdapter(`adapter-${index}`)) + const sessionIds = Array.from({ length: 50 }, (_, index) => `session-${index}`) + for (const [index, sessionId] of sessionIds.entries()) { + const adapter = adapters[index % adapters.length]! + await adapter.spawn({ sessionId, cols: 80, rows: 24 }) + } + const router = new DaemonPtyRouter({ current: adapters[0]!, legacy: adapters.slice(1) }) + await router.discoverLegacySessions() + for (const adapter of adapters) { + vi.mocked(adapter.hasPty).mockClear() + } + + for (const sessionId of sessionIds) { + await router.shutdown(sessionId, { immediate: true }) + expect(router.hasPty(sessionId)).toBe(false) + } + + expect( + adapters.reduce((count, adapter) => count + vi.mocked(adapter.shutdown).mock.calls.length, 0) + ).toBe(50) + expect( + adapters.reduce((count, adapter) => count + vi.mocked(adapter.hasPty).mock.calls.length, 0) + ).toBe(50) + }) + + it('invalidates stopped evidence when the same session id spawns again', async () => { + const current = createAdapter('current', ['reused-session']) + const router = new DaemonPtyRouter({ current, legacy: [] }) + + await router.shutdown('reused-session', { immediate: true }) + expect(router.hasPty('reused-session')).toBe(false) + await router.spawn({ sessionId: 'reused-session', cols: 80, rows: 24 }) + + expect(router.hasPty('reused-session')).toBe(true) + }) + it('fails listProcesses closed when any routed adapter cannot list sessions', async () => { const current = createAdapter('current', ['current-session']) const legacy = createAdapter('legacy', ['legacy-session']) diff --git a/src/main/daemon/daemon-pty-router.ts b/src/main/daemon/daemon-pty-router.ts index 18bf0f926b5..dc17340f72a 100644 --- a/src/main/daemon/daemon-pty-router.ts +++ b/src/main/daemon/daemon-pty-router.ts @@ -7,11 +7,13 @@ import type { PtySpawnOptions, PtySpawnResult } from '../providers/types' +import { ShutdownVerificationOwnerCache } from './shutdown-verification-owner-cache' export class DaemonPtyRouter implements IPtyProvider { private current: DaemonPtyAdapter private legacy: DaemonPtyAdapter[] private sessionAdapters = new Map() + private shutdownVerificationAdapters = new ShutdownVerificationOwnerCache() private unsubscribers: (() => void)[] = [] private dataListeners: ((payload: { id: string @@ -47,6 +49,7 @@ export class DaemonPtyRouter implements IPtyProvider { const sessions = await adapter.listProcesses() for (const session of sessions) { this.sessionAdapters.set(session.id, adapter) + this.shutdownVerificationAdapters.delete(session.id) } } catch (error) { console.warn('[daemon] Failed to discover legacy daemon sessions', error) @@ -59,14 +62,20 @@ export class DaemonPtyRouter implements IPtyProvider { const target = adapter ?? this.current const result = await target.spawn(opts) this.sessionAdapters.set(result.id, target) + this.shutdownVerificationAdapters.delete(result.id) return result } async attach(id: string): Promise { await this.adapterFor(id).attach(id) + this.shutdownVerificationAdapters.delete(id) } hasPty(id: string): boolean { + const shutdownOwner = this.shutdownVerificationAdapters.take(id) + if (shutdownOwner) { + return shutdownOwner.hasPty(id) + } const routed = this.sessionAdapters.get(id) if (routed) { return routed.hasPty(id) @@ -95,7 +104,8 @@ export class DaemonPtyRouter implements IPtyProvider { } async shutdown(id: string, opts: { immediate?: boolean; keepHistory?: boolean }): Promise { - await this.adapterFor(id).shutdown(id, opts) + const owner = this.adapterFor(id) + await owner.shutdown(id, opts) // Why: sleep passes keepHistory=true and re-spawns against the same // sessionId on wake. If we delete the routing entry here, adapterFor() // falls back to `this.current` on wake — for a session that originally @@ -104,6 +114,7 @@ export class DaemonPtyRouter implements IPtyProvider { // losing the cold-restore from the legacy adapter's history dir. if (!opts.keepHistory) { this.sessionAdapters.delete(id) + this.shutdownVerificationAdapters.remember(id, owner) } } @@ -156,6 +167,9 @@ export class DaemonPtyRouter implements IPtyProvider { async revive(state: string): Promise { await this.current.revive(state) + // Why: serialized revive state is opaque here; discard negative evidence + // so a restored id is discovered from its owning adapter instead. + this.shutdownVerificationAdapters.clear() } async listProcesses(): Promise { @@ -236,15 +250,18 @@ export class DaemonPtyRouter implements IPtyProvider { } for (const id of result.alive) { this.sessionAdapters.set(id, adapter) + this.shutdownVerificationAdapters.delete(id) } for (const id of result.killed) { this.sessionAdapters.delete(id) + this.shutdownVerificationAdapters.delete(id) } } return { alive, killed } } dispose(): void { + this.shutdownVerificationAdapters.clear() for (const unsubscribe of this.unsubscribers.splice(0)) { unsubscribe() } diff --git a/src/main/daemon/degraded-daemon-pty-provider.test.ts b/src/main/daemon/degraded-daemon-pty-provider.test.ts index 188173816db..3a34db0c700 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.test.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.test.ts @@ -168,6 +168,65 @@ describe('DegradedDaemonPtyProvider', () => { expect(fallback.write).not.toHaveBeenCalled() }) + it('keeps multi-PTY post-shutdown verification O(T) across retained providers', async () => { + const adapters = Array.from({ length: 21 }, (_, index) => + createDaemonAdapter(`adapter-${index}`) + ) + const fallback = createProvider('fallback') + const sessionIds = Array.from({ length: 50 }, (_, index) => `session-${index}`) + for (const [index, sessionId] of sessionIds.entries()) { + const adapter = adapters[index % adapters.length]! + await adapter.spawn({ sessionId, cols: 80, rows: 24 }) + } + const provider = new DegradedDaemonPtyProvider({ + current: adapters[0]!, + legacy: adapters.slice(1), + fallback + }) + await provider.discoverDaemonSessions() + for (const adapter of adapters) { + vi.mocked(adapter.hasPty).mockClear() + } + vi.mocked(fallback.hasPty!).mockClear() + + for (const sessionId of sessionIds) { + await provider.shutdown(sessionId, { immediate: true }) + expect(provider.hasPty(sessionId)).toBe(false) + } + + expect( + adapters.reduce((count, adapter) => count + vi.mocked(adapter.shutdown).mock.calls.length, 0) + ).toBe(50) + expect( + adapters.reduce((count, adapter) => count + vi.mocked(adapter.hasPty).mock.calls.length, 0) + ).toBe(50) + expect(fallback.hasPty).not.toHaveBeenCalled() + }) + + it('invalidates stopped evidence when the same session id spawns again', async () => { + const current = createDaemonAdapter('daemon') + const fallback = createProvider('fallback', ['reused-session']) + const provider = new DegradedDaemonPtyProvider({ current, legacy: [], fallback }) + + await provider.shutdown('reused-session', { immediate: true }) + expect(provider.hasPty('reused-session')).toBe(false) + await provider.spawn({ sessionId: 'reused-session', cols: 80, rows: 24 }) + + expect(provider.hasPty('reused-session')).toBe(true) + }) + + it('keeps shutdown verification conservative when the cached owner lacks hasPty', async () => { + const current = createDaemonAdapter('daemon') + const fallback = createProvider('fallback') + const provider = new DegradedDaemonPtyProvider({ current, legacy: [], fallback }) + const session = await provider.spawn({ cols: 80, rows: 24 }) + delete (fallback as { hasPty?: (id: string) => boolean }).hasPty + + await provider.shutdown(session.id, { immediate: true }) + + expect(provider.hasPty(session.id)).toBe(true) + }) + it('routes authoritative recovery snapshots to the owning daemon', async () => { const current = createDaemonAdapter('daemon', ['daemon-session']) const fallback = createProvider('fallback') diff --git a/src/main/daemon/degraded-daemon-pty-provider.ts b/src/main/daemon/degraded-daemon-pty-provider.ts index 572cb0aca37..06696291114 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.ts @@ -8,6 +8,9 @@ import type { PtySpawnOptions, PtySpawnResult } from '../providers/types' +import { ShutdownVerificationOwnerCache } from './shutdown-verification-owner-cache' +import { reconcileDegradedDaemonSessions } from './degraded-daemon-session-reconciliation' +import { subscribeToProviderReplay } from './provider-replay-subscription' type ManagedPtyProvider = IPtyProvider & { disconnectOnly?: () => Promise @@ -25,6 +28,7 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { private legacy: DaemonPtyAdapter[] private fallback: ManagedPtyProvider private sessionProviders = new Map() + private shutdownVerificationProviders = new ShutdownVerificationOwnerCache() private unsubscribers: (() => void)[] = [] private dataListeners: ((payload: { id: string @@ -65,6 +69,7 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { const sessions = await adapter.listProcesses() for (const session of sessions) { this.sessionProviders.set(session.id, adapter) + this.shutdownVerificationProviders.delete(session.id) } } catch (error) { console.warn('[daemon] Failed to discover degraded daemon sessions', error) @@ -77,14 +82,20 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { const target = mapped ?? this.fallback const result = await target.spawn(opts) this.sessionProviders.set(result.id, target) + this.shutdownVerificationProviders.delete(result.id) return result } async attach(id: string): Promise { await this.providerFor(id).attach(id) + this.shutdownVerificationProviders.delete(id) } hasPty(id: string): boolean { + const shutdownOwner = this.shutdownVerificationProviders.take(id) + if (shutdownOwner) { + return shutdownOwner.hasPty?.(id) ?? true + } const mapped = this.sessionProviders.get(id) if (mapped) { return mapped.hasPty?.(id) ?? true @@ -113,9 +124,11 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { } async shutdown(id: string, opts: { immediate?: boolean; keepHistory?: boolean }): Promise { - await this.providerFor(id).shutdown(id, opts) + const owner = this.providerFor(id) + await owner.shutdown(id, opts) if (!opts.keepHistory) { this.sessionProviders.delete(id) + this.shutdownVerificationProviders.remember(id, owner) } } @@ -170,6 +183,7 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { async revive(state: string): Promise { await this.fallback.revive(state) + this.shutdownVerificationProviders.clear() } async listProcesses(): Promise { @@ -211,23 +225,7 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { } onReplay(callback: (payload: { id: string; data: string }) => void): () => void { - const unsubscribes = this.allProviders().map((provider) => provider.onReplay(callback)) - let active = true - const trackedUnsubscribe = (): void => { - if (!active) { - return - } - active = false - const idx = this.unsubscribers.indexOf(trackedUnsubscribe) - if (idx !== -1) { - this.unsubscribers.splice(idx, 1) - } - for (const unsubscribe of unsubscribes) { - unsubscribe() - } - } - this.unsubscribers.push(trackedUnsubscribe) - return trackedUnsubscribe + return subscribeToProviderReplay(this.allProviders(), callback, this.unsubscribers) } onExit(callback: (payload: { id: string; code: number }) => void): () => void { @@ -252,23 +250,22 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { alive: string[] killed: string[] }> { - const alive: string[] = [] - const killed: string[] = [] - for (const adapter of this.allDaemonAdapters()) { - const result = await adapter.reconcileOnStartup(validWorktreeIds) - for (const id of result.alive) { - alive.push(id) + return reconcileDegradedDaemonSessions( + this.allDaemonAdapters(), + validWorktreeIds, + (id, adapter) => { this.sessionProviders.set(id, adapter) - } - for (const id of result.killed) { - killed.push(id) + this.shutdownVerificationProviders.delete(id) + }, + (id) => { this.sessionProviders.delete(id) + this.shutdownVerificationProviders.delete(id) } - } - return { alive, killed } + ) } dispose(): void { + this.shutdownVerificationProviders.clear() this.disposeProviderOnly() for (const adapter of this.allDaemonAdapters()) { adapter.dispose() @@ -292,6 +289,7 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { fanoutCurrentDaemonSyntheticExits(code: number): void { for (const id of this.getCurrentDaemonSessionIds()) { this.sessionProviders.delete(id) + this.shutdownVerificationProviders.delete(id) // Why: sessions discovered from listProcesses may not exist in the // adapter's active-session set, but restart still kills that daemon. // oxlint-disable-next-line unicorn/no-useless-spread -- copy-safe: listeners may unsubscribe during iteration diff --git a/src/main/daemon/degraded-daemon-session-reconciliation.ts b/src/main/daemon/degraded-daemon-session-reconciliation.ts new file mode 100644 index 00000000000..04884240044 --- /dev/null +++ b/src/main/daemon/degraded-daemon-session-reconciliation.ts @@ -0,0 +1,23 @@ +import type { DaemonPtyAdapter } from './daemon-pty-adapter' + +export async function reconcileDegradedDaemonSessions( + adapters: readonly DaemonPtyAdapter[], + validWorktreeIds: Set, + onAlive: (id: string, adapter: DaemonPtyAdapter) => void, + onKilled: (id: string) => void +): Promise<{ alive: string[]; killed: string[] }> { + const alive: string[] = [] + const killed: string[] = [] + for (const adapter of adapters) { + const result = await adapter.reconcileOnStartup(validWorktreeIds) + for (const id of result.alive) { + alive.push(id) + onAlive(id, adapter) + } + for (const id of result.killed) { + killed.push(id) + onKilled(id) + } + } + return { alive, killed } +} diff --git a/src/main/daemon/provider-replay-subscription.ts b/src/main/daemon/provider-replay-subscription.ts new file mode 100644 index 00000000000..a87e459fd89 --- /dev/null +++ b/src/main/daemon/provider-replay-subscription.ts @@ -0,0 +1,25 @@ +import type { IPtyProvider } from '../providers/types' + +export function subscribeToProviderReplay( + providers: readonly IPtyProvider[], + callback: (payload: { id: string; data: string }) => void, + trackedSubscriptions: (() => void)[] +): () => void { + const unsubscribes = providers.map((provider) => provider.onReplay(callback)) + let active = true + const trackedUnsubscribe = (): void => { + if (!active) { + return + } + active = false + const index = trackedSubscriptions.indexOf(trackedUnsubscribe) + if (index !== -1) { + trackedSubscriptions.splice(index, 1) + } + for (const unsubscribe of unsubscribes) { + unsubscribe() + } + } + trackedSubscriptions.push(trackedUnsubscribe) + return trackedUnsubscribe +} diff --git a/src/main/daemon/shutdown-verification-owner-cache.ts b/src/main/daemon/shutdown-verification-owner-cache.ts new file mode 100644 index 00000000000..b928e707796 --- /dev/null +++ b/src/main/daemon/shutdown-verification-owner-cache.ts @@ -0,0 +1,30 @@ +export class ShutdownVerificationOwnerCache { + private readonly owners = new Map() + + constructor(private readonly limit = 4_096) {} + + delete(id: string): void { + this.owners.delete(id) + } + + clear(): void { + this.owners.clear() + } + + take(id: string): T | undefined { + const owner = this.owners.get(id) + this.owners.delete(id) + return owner + } + + remember(id: string, owner: T): void { + this.owners.delete(id) + this.owners.set(id, owner) + if (this.owners.size > this.limit) { + const oldest = this.owners.keys().next().value + if (oldest !== undefined) { + this.owners.delete(oldest) + } + } + } +} diff --git a/src/main/ipc/worktrees.test.ts b/src/main/ipc/worktrees.test.ts index 30a73ee3522..bed85f1853b 100644 --- a/src/main/ipc/worktrees.test.ts +++ b/src/main/ipc/worktrees.test.ts @@ -290,6 +290,7 @@ describe('registerWorktreeHandlers', () => { createTerminal: ReturnType splitTerminal: ReturnType notifyWorktreesChangedForRemoteClients: ReturnType + runWithWorktreeRemovalDedup: ReturnType runWithWorktreePtyTeardown: ReturnType beginWorktreePtyTeardown: ReturnType } @@ -504,6 +505,18 @@ describe('registerWorktreeHandlers', () => { paneRuntimeId: -1 }), notifyWorktreesChangedForRemoteClients: vi.fn(), + runWithWorktreeRemovalDedup: vi.fn( + async ( + _worktreeId: string, + _opts: { + force: boolean + runHooks: boolean + hostId?: string + mode?: 'remove' | 'forget' + }, + operation: () => Promise + ) => await operation() + ), runWithWorktreePtyTeardown: vi.fn( async (_worktreeId: string, operation: () => Promise) => await operation() ), @@ -8020,6 +8033,75 @@ describe('registerWorktreeHandlers', () => { expect(mainWindow.webContents.send).toHaveBeenCalledTimes(1) }) + it('joins concurrent renderer and runtime removals before either repeats preflight', async () => { + mockKnownFeatureWorktree() + const shared = new Map }>() + runtimeStub.runWithWorktreeRemovalDedup.mockImplementation( + async ( + worktreeId: string, + opts: { + force: boolean + runHooks: boolean + hostId?: string + mode?: 'remove' | 'forget' + }, + operation: () => Promise + ) => { + const key = `${opts.hostId ?? ''}\0${worktreeId}` + const optionsKey = `${opts.mode ?? 'remove'}:${opts.force}:${opts.runHooks}` + const existing = shared.get(key) + if (existing) { + if (existing.optionsKey !== optionsKey) { + throw new Error('Worktree deletion already in progress') + } + return await existing.promise + } + const promise = operation() + shared.set(key, { optionsKey, promise }) + try { + return await promise + } finally { + shared.delete(key) + } + } + ) + let finishRemoval = () => {} + removeWorktreeMock.mockImplementation( + () => + new Promise((resolve) => { + finishRemoval = resolve + }) + ) + const worktreeId = 'repo-1::/workspace/feature-wt' + const rendererRemoval = handlers['worktrees:remove'](null, { + worktreeId, + force: true + }) as Promise + await vi.waitFor(() => expect(removeWorktreeMock).toHaveBeenCalledTimes(1)) + const runtimeBackend = vi.fn(async () => ({ source: 'runtime' })) + const runRuntimeRemoval = runtimeStub.runWithWorktreeRemovalDedup as unknown as ( + worktreeId: string, + opts: { + force: boolean + runHooks: boolean + hostId?: string + mode?: 'remove' | 'forget' + }, + operation: () => Promise + ) => Promise + const runtimeRemoval = runRuntimeRemoval( + worktreeId, + { force: true, runHooks: true, hostId: 'local' }, + runtimeBackend + ) + + finishRemoval() + await expect(Promise.all([rendererRemoval, runtimeRemoval])).resolves.toEqual([{}, {}]) + expect(runtimeBackend).not.toHaveBeenCalled() + expect(killAllProcessesForWorktreeMock).toHaveBeenCalledTimes(1) + expect(removeWorktreeMock).toHaveBeenCalledTimes(1) + }) + it('rejects concurrent deletes for the same worktree id with different options', async () => { mockKnownFeatureWorktree() let removalStarted!: () => void @@ -8511,6 +8593,20 @@ describe('registerWorktreeHandlers', () => { }) describe('worktrees:forgetLocal', () => { + it('rejects a forget race while another removal owns PTY admission', async () => { + const worktreeId = 'repo-1::/workspace/feature-wt' + runtimeStub.runWithWorktreePtyTeardown.mockRejectedValueOnce( + new Error(`Worktree teardown is already in progress: ${worktreeId}`) + ) + + await expect(handlers['worktrees:forgetLocal'](null, { worktreeId })).rejects.toThrow( + 'Worktree teardown is already in progress' + ) + + expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() + expect(store.removeWorktreeMeta).not.toHaveBeenCalled() + }) + it('forgets a workspace pinned to a removed SSH target without touching the provider', async () => { const repo = { id: 'repo-1', diff --git a/src/main/ipc/worktrees.ts b/src/main/ipc/worktrees.ts index f2cd53ccbe8..cb0ebf14e39 100644 --- a/src/main/ipc/worktrees.ts +++ b/src/main/ipc/worktrees.ts @@ -1,5 +1,5 @@ /* oxlint-disable max-lines */ -import type { BrowserWindow } from 'electron' +import type { BrowserWindow, IpcMainInvokeEvent } from 'electron' import { ipcMain } from 'electron' import { readFile, stat } from 'node:fs/promises' import { randomUUID } from 'node:crypto' @@ -1405,7 +1405,26 @@ export function registerWorktreeHandlers( ? runtime.beginWorktreePtyTeardown(worktreeId) : Promise.resolve(() => {}) - ipcMain.handle( + const registerCrossSurfaceWorktreeRemovalHandler = ( + channel: string, + handler: (event: IpcMainInvokeEvent, args: RemoveWorktreeArgs) => Promise + ): void => { + ipcMain.handle(channel, (event, args: RemoveWorktreeArgs) => { + const { repoId } = parseWorktreeId(args.worktreeId) + const repo = getRepoForWorktreeRemoval(store, repoId, args.hostId) + return runtime.runWithWorktreeRemovalDedup( + args.worktreeId, + { + force: args.force === true, + runHooks: args.skipArchive !== true, + ...(repo ? { hostId: getRepoExecutionHostId(repo) } : {}) + }, + () => handler(event, args) + ) + }) + } + + registerCrossSurfaceWorktreeRemovalHandler( 'worktrees:remove', async (_event, args: RemoveWorktreeArgs): Promise => { const { repoId, worktreePath } = parseWorktreeId(args.worktreeId) @@ -1922,13 +1941,36 @@ export function registerWorktreeHandlers( } ) + const registerCrossSurfaceWorktreeForgetHandler = ( + channel: string, + handler: ( + event: IpcMainInvokeEvent, + args: Pick + ) => Promise + ): void => { + ipcMain.handle(channel, (event, args: Pick) => { + const { repoId } = parseWorktreeId(args.worktreeId) + const repo = getRepoForWorktreeRemoval(store, repoId, args.hostId) + return runtime.runWithWorktreeRemovalDedup( + args.worktreeId, + { + force: false, + runHooks: false, + mode: 'forget', + ...(repo ? { hostId: getRepoExecutionHostId(repo) } : {}) + }, + () => handler(event, args) + ) + }) + } + // Why: forget-locally drops a workspace from Orca without any remote Git or // filesystem work. It exists so a workspace pinned to a removed/disconnected // SSH target — whose provider is gone and whose `worktrees:remove` therefore // throws at requireSshGitProvider before any cleanup runs — can still be // cleared from the app. It never touches the remote: no worktree registration, // no branches, no files are deleted there. - ipcMain.handle( + registerCrossSurfaceWorktreeForgetHandler( 'worktrees:forgetLocal', async ( _event, diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 3e368074402..31a26113379 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -22377,6 +22377,50 @@ describe('OrcaRuntimeService', () => { expect(stopAndWait).not.toHaveBeenCalled() }) + it('joins identical cross-surface worktree removals in the runtime owner', async () => { + const runtime = new OrcaRuntimeService(store) + const removal = deferred<{ warning: string }>() + const operation = vi.fn(() => removal.promise) + const opts = { force: true, runHooks: false, hostId: 'local' } + + const first = runtime.runWithWorktreeRemovalDedup(TEST_WORKTREE_ID, opts, operation) + const second = runtime.runWithWorktreeRemovalDedup(TEST_WORKTREE_ID, opts, vi.fn()) + removal.resolve({ warning: 'joined' }) + + await expect(Promise.all([first, second])).resolves.toEqual([ + { warning: 'joined' }, + { warning: 'joined' } + ]) + expect(operation).toHaveBeenCalledTimes(1) + }) + + it('rejects cross-surface worktree removals with different options', async () => { + const runtime = new OrcaRuntimeService(store) + const removal = deferred>() + const first = runtime.runWithWorktreeRemovalDedup( + TEST_WORKTREE_ID, + { force: false, runHooks: true, hostId: 'local' }, + () => removal.promise + ) + + await expect( + runtime.runWithWorktreeRemovalDedup( + TEST_WORKTREE_ID, + { force: true, runHooks: true, hostId: 'local' }, + vi.fn() + ) + ).rejects.toThrow('Worktree deletion already in progress') + await expect( + runtime.runWithWorktreeRemovalDedup( + TEST_WORKTREE_ID, + { force: false, runHooks: false, hostId: 'local', mode: 'forget' }, + vi.fn() + ) + ).rejects.toThrow('Worktree deletion already in progress') + removal.resolve({}) + await first + }) + it('stops exactly the expected live PTYs for a worktree', async () => { const runtime = new OrcaRuntimeService(store) const stopped: string[] = [] diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index bd6941eb996..03b47a5c10e 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -2499,6 +2499,7 @@ export class OrcaRuntimeService { private canonicalFetchKeyCache = new Map() private optimisticReconcileTokens = new Map() private removeManagedWorktreeInFlight = new Map() + private crossSurfaceWorktreeRemovalInFlight = new Map() private readonly worktreePtyAdmission = new WorktreePtyAdmission() private preservedBranchCleanupByWorktreeId = new Map() private readonly getLocalProviderFn: (() => IPtyProvider) | null @@ -16888,8 +16889,45 @@ export class OrcaRuntimeService { if (!this.store) { throw new Error('runtime_unavailable') } - const store = this.store + const exactTarget = parseExactWorktreeIdSelector(worktreeSelector) + if (exactTarget) { + const repo = this.store.getRepo(exactTarget.repoId) + return this.runWithWorktreeRemovalDedup( + exactTarget.id, + { + force, + runHooks, + ...(repo ? { hostId: getRepoExecutionHostId(repo) } : {}) + }, + async () => { + const resolvedTarget = await this.resolveWorktreeRemovalTarget(worktreeSelector) + return this.removeManagedWorktreeCore(force, runHooks, resolvedTarget) + } + ) + } const removalTarget = await this.resolveWorktreeRemovalTarget(worktreeSelector) + const repo = this.store.getRepo(removalTarget.repoId) + return this.runWithWorktreeRemovalDedup( + removalTarget.id, + { + force, + runHooks, + ...(repo ? { hostId: getRepoExecutionHostId(repo) } : {}) + }, + () => this.removeManagedWorktreeCore(force, runHooks, removalTarget) + ) + } + + private async removeManagedWorktreeCore( + force: boolean, + runHooks: boolean, + resolvedRemovalTarget: RuntimeWorktreeRemovalTarget + ): Promise { + if (!this.store) { + throw new Error('runtime_unavailable') + } + const store = this.store + const removalTarget = resolvedRemovalTarget const optionsKey = getRuntimeWorktreeRemovalOptionsKey(force, runHooks) const inFlightRemoval = this.removeManagedWorktreeInFlight.get(removalTarget.id) if (inFlightRemoval) { @@ -18816,6 +18854,37 @@ export class OrcaRuntimeService { return this.worktreePtyAdmission.runTeardown(worktreeId, operation) } + async runWithWorktreeRemovalDedup( + worktreeId: string, + opts: { force: boolean; runHooks: boolean; hostId?: string; mode?: 'remove' | 'forget' }, + operation: () => Promise + ): Promise { + const optionsKey = `${opts.mode ?? 'remove'}:${getRuntimeWorktreeRemovalOptionsKey( + opts.force, + opts.runHooks + )}` + const removalKey = `${opts.hostId ?? ''}\0${worktreeId}` + const existing = this.crossSurfaceWorktreeRemovalInFlight.get(removalKey) + if (existing) { + if (existing.optionsKey === optionsKey) { + return existing.promise as Promise + } + throw new Error(`Worktree deletion already in progress: ${worktreeId}`) + } + const promise = operation() + this.crossSurfaceWorktreeRemovalInFlight.set(removalKey, { + optionsKey, + promise: promise as Promise + }) + try { + return await promise + } finally { + if (this.crossSurfaceWorktreeRemovalInFlight.get(removalKey)?.promise === promise) { + this.crossSurfaceWorktreeRemovalInFlight.delete(removalKey) + } + } + } + async stopTerminalsForWorktree( worktreeSelector: string, opts: { worktreeTeardown?: boolean } = {} diff --git a/src/main/runtime/worktree-pty-admission.test.ts b/src/main/runtime/worktree-pty-admission.test.ts index 35bf926a162..bb83ab9d956 100644 --- a/src/main/runtime/worktree-pty-admission.test.ts +++ b/src/main/runtime/worktree-pty-admission.test.ts @@ -39,7 +39,7 @@ describe('WorktreePtyAdmission', () => { await expect(teardown).resolves.toBeUndefined() }) - it('serializes teardown owners from different removal entry points', async () => { + it('rejects a duplicate teardown owner from another removal entry point', async () => { const admission = new WorktreePtyAdmission() let releaseFirst = () => {} const order: string[] = [] @@ -51,14 +51,16 @@ describe('WorktreePtyAdmission', () => { releaseFirst = resolve }) ) - const second = admission.runTeardown('w1', async () => { - order.push('second') - }) + const second = admission.runTeardown('w1', async () => order.push('second')) + const secondRejection = expect(second).rejects.toThrow( + 'Worktree teardown is already in progress' + ) await vi.waitFor(() => expect(order).toEqual(['first'])) + await secondRejection releaseFirst() - await Promise.all([first, second]) - expect(order).toEqual(['first', 'second']) + await first + expect(order).toEqual(['first']) }) }) diff --git a/src/main/runtime/worktree-pty-admission.ts b/src/main/runtime/worktree-pty-admission.ts index 7dadd76c8b2..f3fb2211cb9 100644 --- a/src/main/runtime/worktree-pty-admission.ts +++ b/src/main/runtime/worktree-pty-admission.ts @@ -2,7 +2,6 @@ type WorktreePtyAdmissionState = { activeSpawns: number teardownOwners: number drainWaiters: Set<() => void> - teardownTail: Promise } export class WorktreePtyAdmission { @@ -44,16 +43,12 @@ export class WorktreePtyAdmission { async closeForTeardown(worktreeId: string): Promise<() => void> { const state = this.getOrCreate(worktreeId) + if (state.teardownOwners > 0) { + // Why: teardown-only callers and mismatched remove/forget operations do + // not join the shared removal promise; never queue their stale state. + throw new Error(`Worktree teardown is already in progress: ${worktreeId}`) + } state.teardownOwners += 1 - const previousTeardown = state.teardownTail - let finishTeardown = () => {} - const thisTeardown = new Promise((resolve) => { - finishTeardown = resolve - }) - // Why: runtime RPC and renderer IPC have distinct dedupe maps; serialize - // them here so two removal owners cannot overlap provider or Git teardown. - state.teardownTail = previousTeardown.then(() => thisTeardown) - await previousTeardown if (state.activeSpawns > 0) { await new Promise((resolve) => state.drainWaiters.add(resolve)) } @@ -64,7 +59,6 @@ export class WorktreePtyAdmission { } released = true state.teardownOwners -= 1 - finishTeardown() this.deleteIfIdle(worktreeId, state) } } @@ -77,8 +71,7 @@ export class WorktreePtyAdmission { const state: WorktreePtyAdmissionState = { activeSpawns: 0, teardownOwners: 0, - drainWaiters: new Set(), - teardownTail: Promise.resolve() + drainWaiters: new Set() } this.states.set(worktreeId, state) return state From 61d9d2c128eef99812a54493339937f64cf035b2 Mon Sep 17 00:00:00 2001 From: Brennan Benson Date: Sun, 12 Jul 2026 14:36:21 -0700 Subject: [PATCH 08/36] fix: drain terminals during orphan cleanup --- src/main/ipc/worktrees.test.ts | 39 +++-- src/main/ipc/worktrees.ts | 239 ++++++++++++++------------ src/main/runtime/orca-runtime.test.ts | 65 ++++++- src/main/runtime/orca-runtime.ts | 232 ++++++++++++++----------- 4 files changed, 342 insertions(+), 233 deletions(-) diff --git a/src/main/ipc/worktrees.test.ts b/src/main/ipc/worktrees.test.ts index bed85f1853b..46263b4b942 100644 --- a/src/main/ipc/worktrees.test.ts +++ b/src/main/ipc/worktrees.test.ts @@ -6669,7 +6669,7 @@ describe('registerWorktreeHandlers', () => { preservedBranch: { branchName: 'feature', head: 'feature' } }) expect(runHookMock).not.toHaveBeenCalled() - expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() + expect(killAllProcessesForWorktreeMock).toHaveBeenCalledWith(worktreeId, expect.any(Object)) expect(removeWorktreeMock).not.toHaveBeenCalled() expect(gitExecFileAsyncMock).toHaveBeenCalledWith(['worktree', 'prune'], { cwd: '/workspace/repo' @@ -7675,7 +7675,10 @@ describe('registerWorktreeHandlers', () => { force: true }) - expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() + expect(killAllProcessesForWorktreeMock).toHaveBeenCalledWith( + 'repo-1::/workspace/already-deleted-wt', + expect.any(Object) + ) expect(runHookMock).not.toHaveBeenCalled() expect(removeWorktreeMock).not.toHaveBeenCalled() expect(runtimeStub.clearOptimisticReconcileToken).toHaveBeenCalledWith( @@ -7700,7 +7703,7 @@ describe('registerWorktreeHandlers', () => { await handlers['worktrees:remove'](null, { worktreeId, force: true }) - expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() + expect(killAllProcessesForWorktreeMock).toHaveBeenCalledWith(worktreeId, expect.any(Object)) expect(runHookMock).not.toHaveBeenCalled() expect(removeWorktreeMock).not.toHaveBeenCalled() expect(runtimeStub.clearOptimisticReconcileToken).toHaveBeenCalledWith(worktreeId) @@ -7714,12 +7717,20 @@ describe('registerWorktreeHandlers', () => { it('treats normal deletion of an already-missing unregistered worktree as cleanup', async () => { mockKnownFeatureWorktree('/workspace/real-feature') store.getWorktreeMeta.mockReturnValue(makeWorktreeMeta()) + const callOrder: string[] = [] + killAllProcessesForWorktreeMock.mockImplementation(async () => { + callOrder.push('pty-teardown') + return { runtimeStopped: 1, providerStopped: 0, registryStopped: 0 } + }) + store.removeWorktreeMeta.mockImplementation(() => { + callOrder.push('metadata-removal') + }) await handlers['worktrees:remove'](null, { worktreeId: 'repo-1::/workspace/already-deleted-wt' }) - expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() + expect(callOrder).toEqual(['pty-teardown', 'metadata-removal']) expect(runHookMock).not.toHaveBeenCalled() expect(removeWorktreeMock).not.toHaveBeenCalled() expect(runtimeStub.clearOptimisticReconcileToken).toHaveBeenCalledWith( @@ -7764,7 +7775,7 @@ describe('registerWorktreeHandlers', () => { }) await expect(lstat(orphanPath)).rejects.toMatchObject({ code: 'ENOENT' }) - expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() + expect(killAllProcessesForWorktreeMock).toHaveBeenCalledWith(worktreeId, expect.any(Object)) expect(runHookMock).not.toHaveBeenCalled() expect(removeWorktreeMock).not.toHaveBeenCalled() expect(runtimeStub.clearOptimisticReconcileToken).toHaveBeenCalledWith(worktreeId) @@ -7856,7 +7867,7 @@ describe('registerWorktreeHandlers', () => { ).resolves.toEqual({}) await expect(lstat(leftoverPath)).rejects.toMatchObject({ code: 'ENOENT' }) - expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() + expect(killAllProcessesForWorktreeMock).toHaveBeenCalledWith(worktreeId, expect.any(Object)) expect(runHookMock).not.toHaveBeenCalled() expect(removeWorktreeMock).not.toHaveBeenCalled() expect(runtimeStub.clearOptimisticReconcileToken).toHaveBeenCalledWith(worktreeId) @@ -8418,17 +8429,19 @@ describe('registerWorktreeHandlers', () => { worktreeId: 'repo-1::/workspace/feature-wt' }) - expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() + expect(killAllProcessesForWorktreeMock).toHaveBeenCalledWith( + 'repo-1::/workspace/feature-wt', + expect.any(Object) + ) expect(removeWorktreeMock).toHaveBeenCalled() expect(gitExecFileAsyncMock).toHaveBeenCalledWith(['worktree', 'prune'], { cwd: '/workspace/repo' }) }) - it('skips the PTY teardown for SSH-backed repos (design §6 out-of-scope)', async () => { - // Why: SSH-backed PTYs live on the remote host and are handled by the - // remote provider's own teardown. The local-host helper must not run for - // SSH repos, because it would sweep registry entries for other worktrees. + it('does not start teardown before resolving the SSH removal provider', async () => { + // Why: an unresolved SSH target is a rejected removal, not a successful + // cleanup path, so admission and PTY state must remain untouched. const repo = { id: 'repo-ssh', path: '/remote/repo', @@ -8441,9 +8454,7 @@ describe('registerWorktreeHandlers', () => { store.getRepos.mockReturnValue([repo]) store.getRepo.mockReturnValue(repo) - // The test can't easily mock the SSH provider without more plumbing — the - // call will throw about 'no git provider for connection'. What matters - // here is that the kill helper was NOT called for the SSH branch. + // The call rejects before it has an authoritative remote removal target. await ( handlers['worktrees:remove'](null, { worktreeId: 'repo-ssh::/remote/feature-wt' diff --git a/src/main/ipc/worktrees.ts b/src/main/ipc/worktrees.ts index cb0ebf14e39..a4c9f6df286 100644 --- a/src/main/ipc/worktrees.ts +++ b/src/main/ipc/worktrees.ts @@ -1404,6 +1404,20 @@ export function registerWorktreeHandlers( typeof runtime.beginWorktreePtyTeardown === 'function' ? runtime.beginWorktreePtyTeardown(worktreeId) : Promise.resolve(() => {}) + const runWithSuccessfulCleanupTeardown = ( + worktreeId: string, + operation: () => Promise + ): Promise => + runWithWorktreePtyTeardown(worktreeId, async () => { + // Why: orphan and stale-registration cleanup still retires an Orca + // workspace; its terminals must not outlive the last addressable owner. + await killAllProcessesForWorktree(worktreeId, { + runtime, + localProvider: getLocalPtyProvider(), + onPtyStopped: clearProviderPtyState + }) + return operation() + }) const registerCrossSurfaceWorktreeRemovalHandler = ( channel: string, @@ -1529,32 +1543,34 @@ export function registerWorktreeHandlers( if (!args.force) { throw new Error(ORPHANED_WORKTREE_DIRECTORY_MESSAGE) } - if (repo.connectionId) { - await fsProvider!.deletePath(worktreePath, true) - await cleanupUnusedWorktreePushTargetRemoteSsh( - provider!, - repo.path, - args.worktreeId, - removedPushTarget, - store - ) - } else { - await closeLocalWatcherForRemoval(worktreePath) - await removeLocalWorktreePath(worktreePath, localWorktreeGitOptions) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - args.worktreeId, - removedPushTarget, - store, - localWorktreeGitOptions - ) - invalidateAuthorizedRootsCache() - } - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - preservedBranchCleanupByWorktreeId.delete(args.worktreeId) - notifyWorktreesChanged(mainWindow, repoId) - return {} + return runWithSuccessfulCleanupTeardown(args.worktreeId, async () => { + if (repo.connectionId) { + await fsProvider!.deletePath(worktreePath, true) + await cleanupUnusedWorktreePushTargetRemoteSsh( + provider!, + repo.path, + args.worktreeId, + removedPushTarget, + store + ) + } else { + await closeLocalWatcherForRemoval(worktreePath) + await removeLocalWorktreePath(worktreePath, localWorktreeGitOptions) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + args.worktreeId, + removedPushTarget, + store, + localWorktreeGitOptions + ) + invalidateAuthorizedRootsCache() + } + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + preservedBranchCleanupByWorktreeId.delete(args.worktreeId) + notifyWorktreesChanged(mainWindow, repoId) + return {} + }) } if (!repo.connectionId) { const access = getLocalWorktreePathAccess(localWorktreeGitOptions) @@ -1577,21 +1593,23 @@ export function registerWorktreeHandlers( if (!args.force) { throw new Error(ORPHANED_WORKTREE_DIRECTORY_MESSAGE) } - await closeLocalWatcherForRemoval(worktreePath) - await removeLocalWorktreePath(worktreePath, localWorktreeGitOptions) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - args.worktreeId, - removedPushTarget, - store, - localWorktreeGitOptions - ) - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - preservedBranchCleanupByWorktreeId.delete(args.worktreeId) - invalidateAuthorizedRootsCache() - notifyWorktreesChanged(mainWindow, repoId) - return {} + return runWithSuccessfulCleanupTeardown(args.worktreeId, async () => { + await closeLocalWatcherForRemoval(worktreePath) + await removeLocalWorktreePath(worktreePath, localWorktreeGitOptions) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + args.worktreeId, + removedPushTarget, + store, + localWorktreeGitOptions + ) + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + preservedBranchCleanupByWorktreeId.delete(args.worktreeId) + invalidateAuthorizedRootsCache() + notifyWorktreesChanged(mainWindow, repoId) + return {} + }) } } if (await isAlreadyRemovedWorktreePath(repo, worktreePath, localWorktreeGitOptions)) { @@ -1603,29 +1621,31 @@ export function registerWorktreeHandlers( // Why: a manually deleted worktree is already gone from Git and disk. // The sidebar delete action has persisted metadata proving this was // an Orca-known row, so no force confirmation is needed. - if (repo.connectionId) { - await cleanupUnusedWorktreePushTargetRemoteSsh( - provider!, - repo.path, - args.worktreeId, - removedPushTarget, - store - ) - } else { - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - args.worktreeId, - removedPushTarget, - store, - localWorktreeGitOptions - ) - invalidateAuthorizedRootsCache() - } - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - preservedBranchCleanupByWorktreeId.delete(args.worktreeId) - notifyWorktreesChanged(mainWindow, repoId) - return {} + return runWithSuccessfulCleanupTeardown(args.worktreeId, async () => { + if (repo.connectionId) { + await cleanupUnusedWorktreePushTargetRemoteSsh( + provider!, + repo.path, + args.worktreeId, + removedPushTarget, + store + ) + } else { + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + args.worktreeId, + removedPushTarget, + store, + localWorktreeGitOptions + ) + invalidateAuthorizedRootsCache() + } + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + preservedBranchCleanupByWorktreeId.delete(args.worktreeId) + notifyWorktreesChanged(mainWindow, repoId) + return {} + }) } throw new Error(`Refusing to delete unregistered worktree path: ${worktreePath}`) } @@ -1653,35 +1673,35 @@ export function registerWorktreeHandlers( removedMeta && (await isAlreadyRemovedWorktreePath(repo, canonicalWorktreePath, localWorktreeGitOptions)) ) { - const removalResult = await removeStaleLocalWorktreeRegistrationAfterFilesystemRemoval({ - canonicalWorktreePath, - repoPath: repo.path, - localWorktreeGitOptions, - registeredWorktree, - deleteBranch + return runWithSuccessfulCleanupTeardown(args.worktreeId, async () => { + const removalResult = await removeStaleLocalWorktreeRegistrationAfterFilesystemRemoval({ + canonicalWorktreePath, + repoPath: repo.path, + localWorktreeGitOptions, + registeredWorktree, + deleteBranch + }) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + args.worktreeId, + removedPushTarget, + store, + localWorktreeGitOptions + ) + rememberPreservedBranchCleanupTarget( + args.worktreeId, + removalResult, + registeredWorktree.head, + removedPushTarget + ) + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + invalidateAuthorizedRootsCache() + notifyWorktreesChanged(mainWindow, repoId) + return removalResult ?? {} }) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - args.worktreeId, - removedPushTarget, - store, - localWorktreeGitOptions - ) - rememberPreservedBranchCleanupTarget( - args.worktreeId, - removalResult, - registeredWorktree.head, - removedPushTarget - ) - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - invalidateAuthorizedRootsCache() - notifyWorktreesChanged(mainWindow, repoId) - return removalResult ?? {} } - let shouldTearDownPtys = true - // Run archive hook before removal so teardown scripts still see the worktree directory. const hooks = await getArchiveHooksForRemoval(repo) if (hooks?.scripts.archive && !args.skipArchive) { @@ -1793,9 +1813,8 @@ export function registerWorktreeHandlers( formatWorktreeRemovalError(error, canonicalWorktreePath, args.force ?? false) ) } - // Why: orphan cleanup does not need live shells to be killed first, - // and preflight did not prove the worktree is cleanly removable. - shouldTearDownPtys = false + // Why: missing/non-repo paths may still have a live deleted-cwd PTY; + // continue to the fenced teardown before orphan recovery mutates state. } const releaseWorktreePtyTeardown = await beginWorktreePtyTeardown(args.worktreeId) @@ -1803,26 +1822,22 @@ export function registerWorktreeHandlers( try { await closeLocalWatcherForRemoval(canonicalWorktreePath) - if (shouldTearDownPtys) { - // Why: once preflight proves normal deletion is clean, kill PTYs before - // git-level removal so Windows handles cannot keep the directory busy. - await killAllProcessesForWorktree(args.worktreeId, { - runtime, - localProvider: getLocalPtyProvider(), - onPtyStopped: clearProviderPtyState + await killAllProcessesForWorktree(args.worktreeId, { + runtime, + localProvider: getLocalPtyProvider(), + onPtyStopped: clearProviderPtyState + }) + .then((r) => { + const total = r.runtimeStopped + r.providerStopped + r.registryStopped + if (total > 0) { + console.info( + `[worktree-teardown] ${args.worktreeId} killed runtime=${r.runtimeStopped} provider=${r.providerStopped} registry=${r.registryStopped}` + ) + } + }) + .catch((err) => { + console.warn(`[worktree-teardown] failed for ${args.worktreeId}:`, err) }) - .then((r) => { - const total = r.runtimeStopped + r.providerStopped + r.registryStopped - if (total > 0) { - console.info( - `[worktree-teardown] ${args.worktreeId} killed runtime=${r.runtimeStopped} provider=${r.providerStopped} registry=${r.registryStopped}` - ) - } - }) - .catch((err) => { - console.warn(`[worktree-teardown] failed for ${args.worktreeId}:`, err) - }) - } try { const removeOptions = { diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 31a26113379..b3ab954a0a6 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -27896,7 +27896,24 @@ describe('OrcaRuntimeService', () => { const missingWorktreePath = 'C:\\workspace\\already-removed' const worktreeId = `${TEST_REPO_ID}::${missingWorktreePath}` const { runtimeStore, removeWorktreeMeta } = createStaleRuntimeWorktreeStore(worktreeId) - const runtime = new OrcaRuntimeService(runtimeStore as never) + const callOrder: string[] = [] + const localProvider = { listProcesses: vi.fn().mockResolvedValue([]) } + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { + getLocalProvider: () => localProvider as never + }) + runtime.setPtyController({ + write: () => true, + kill: () => false, + stopAndWait: vi.fn(async (ptyId: string) => { + callOrder.push(`stop:${ptyId}`) + return true + }), + getForegroundProcess: async () => null + }) + runtime.registerPty('stale-registration-pty', worktreeId) + removeWorktreeMeta.mockImplementation(() => { + callOrder.push('metadata-removal') + }) const registeredWorktrees = [ { path: TEST_REPO_PATH, @@ -27931,6 +27948,8 @@ describe('OrcaRuntimeService', () => { expect(result).toEqual({ preservedBranch: { branchName: 'feature/foo', head: 'abc' } }) + expect(callOrder).toEqual(['stop:stale-registration-pty', 'metadata-removal']) + expect(localProvider.listProcesses).toHaveBeenCalledTimes(1) expect(runHook).not.toHaveBeenCalled() expect(removeWorktree).not.toHaveBeenCalled() expect(gitSpy).toHaveBeenCalledWith(['worktree', 'prune'], { @@ -28337,7 +28356,24 @@ describe('OrcaRuntimeService', () => { const missingWorktreePath = join(parentDir, 'already-deleted') const worktreeId = `${TEST_REPO_ID}::${missingWorktreePath}` const { runtimeStore, removeWorktreeMeta } = createStaleRuntimeWorktreeStore(worktreeId) - const runtime = new OrcaRuntimeService(runtimeStore as never) + const callOrder: string[] = [] + const localProvider = { listProcesses: vi.fn().mockResolvedValue([]) } + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { + getLocalProvider: () => localProvider as never + }) + runtime.setPtyController({ + write: () => true, + kill: () => false, + stopAndWait: vi.fn(async (ptyId: string) => { + callOrder.push(`stop:${ptyId}`) + return true + }), + getForegroundProcess: async () => null + }) + runtime.registerPty('missing-worktree-pty', worktreeId) + removeWorktreeMeta.mockImplementation(() => { + callOrder.push('metadata-removal') + }) const notifier = { worktreesChanged: vi.fn() } runtime.setNotifier(notifier as never) @@ -28346,6 +28382,8 @@ describe('OrcaRuntimeService', () => { await expect(runtime.removeManagedWorktree(worktreeId)).resolves.toEqual({}) + expect(callOrder).toEqual(['stop:missing-worktree-pty', 'metadata-removal']) + expect(localProvider.listProcesses).toHaveBeenCalledTimes(1) expect(removeWorktree).not.toHaveBeenCalled() expect(removeWorktreeMeta).toHaveBeenCalledWith(worktreeId) expect(deleteWorktreeHistoryDirMock).toHaveBeenCalledWith(worktreeId) @@ -28800,21 +28838,36 @@ describe('OrcaRuntimeService', () => { }) it('falls through to orphan cleanup when preflight reports missing/non-repo worktree', async () => { - const runtime = new OrcaRuntimeService(store) + const callOrder: string[] = [] + const provider = { + listProcesses: vi + .fn() + .mockResolvedValue([ + { id: `${TEST_WORKTREE_ID}@@orphan`, cwd: TEST_WORKTREE_PATH, title: 'shell' } + ]), + shutdown: vi.fn(async () => { + callOrder.push('pty-shutdown') + }) + } + const runtime = new OrcaRuntimeService(store, undefined, { + getLocalProvider: () => provider as never + }) vi.mocked(getEffectiveHooks).mockReturnValue(null) vi.mocked(assertWorktreeCleanForRemoval).mockRejectedValue( Object.assign(new Error('status failed'), { stderr: 'fatal: not a git repository (or any of the parent directories): .git\n' }) ) - vi.mocked(removeWorktree).mockRejectedValue( - Object.assign(new Error('git worktree remove failed'), { + vi.mocked(removeWorktree).mockImplementation(async () => { + callOrder.push('git-remove') + throw Object.assign(new Error('git worktree remove failed'), { stderr: `fatal: '${TEST_WORKTREE_PATH}' is not a working tree` }) - ) + }) vi.spyOn(gitRunner, 'gitExecFileAsync').mockResolvedValue({ stdout: '', stderr: '' }) await expect(runtime.removeManagedWorktree(TEST_WORKTREE_ID)).resolves.toEqual({}) + expect(callOrder).toEqual(['pty-shutdown', 'git-remove']) expect(removeWorktree).toHaveBeenCalledWith( TEST_REPO_PATH, TEST_WORKTREE_PATH, diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 03b47a5c10e..5b89ca0f7d0 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -17025,32 +17025,34 @@ export class OrcaRuntimeService { if (!force) { throw new Error(ORPHANED_WORKTREE_DIRECTORY_MESSAGE) } - if (repo.connectionId) { - await fsProvider!.deletePath(removalTarget.path, true) - await cleanupUnusedWorktreePushTargetRemoteSsh( - provider!, - repo.path, - removalTarget.id, - removedPushTarget, - store - ) - } else { - await removeLocalWorktreePath(removalTarget.path, localWorktreeGitOptions) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - removalTarget.id, - removedPushTarget, - store, - localWorktreeGitOptions - ) - } - this.clearOptimisticReconcileToken(removalTarget.id) - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) - this.invalidateResolvedWorktreeCache() - invalidateAuthorizedRootsCache() - this.notifyWorktreesChanged(repo.id) - return {} + return this.runWithSuccessfulCleanupTeardown(removalTarget.id, async () => { + if (repo.connectionId) { + await fsProvider!.deletePath(removalTarget.path, true) + await cleanupUnusedWorktreePushTargetRemoteSsh( + provider!, + repo.path, + removalTarget.id, + removedPushTarget, + store + ) + } else { + await removeLocalWorktreePath(removalTarget.path, localWorktreeGitOptions) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + removalTarget.id, + removedPushTarget, + store, + localWorktreeGitOptions + ) + } + this.clearOptimisticReconcileToken(removalTarget.id) + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) + this.invalidateResolvedWorktreeCache() + invalidateAuthorizedRootsCache() + this.notifyWorktreesChanged(repo.id) + return {} + }) } if (!repo.connectionId) { const access = getLocalWorktreePathAccess(localWorktreeGitOptions) @@ -17073,24 +17075,26 @@ export class OrcaRuntimeService { if (!force) { throw new Error(ORPHANED_WORKTREE_DIRECTORY_MESSAGE) } - await closeLocalWatcherForWorktreePath(removalTarget.path).catch((err) => { - console.warn(`[filesystem-watcher] failed to close ${removalTarget.path}:`, err) + return this.runWithSuccessfulCleanupTeardown(removalTarget.id, async () => { + await closeLocalWatcherForWorktreePath(removalTarget.path).catch((err) => { + console.warn(`[filesystem-watcher] failed to close ${removalTarget.path}:`, err) + }) + await removeLocalWorktreePath(removalTarget.path, localWorktreeGitOptions) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + removalTarget.id, + removedPushTarget, + store, + localWorktreeGitOptions + ) + this.clearOptimisticReconcileToken(removalTarget.id) + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) + this.invalidateResolvedWorktreeCache() + invalidateAuthorizedRootsCache() + this.notifyWorktreesChanged(repo.id) + return {} }) - await removeLocalWorktreePath(removalTarget.path, localWorktreeGitOptions) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - removalTarget.id, - removedPushTarget, - store, - localWorktreeGitOptions - ) - this.clearOptimisticReconcileToken(removalTarget.id) - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) - this.invalidateResolvedWorktreeCache() - invalidateAuthorizedRootsCache() - this.notifyWorktreesChanged(repo.id) - return {} } } if (await isRuntimeWorktreePathMissing(repo, removalTarget.path, localWorktreeGitOptions)) { @@ -17102,28 +17106,30 @@ export class OrcaRuntimeService { // Why: a manually deleted worktree is already gone from Git and disk. // Finish runtime metadata cleanup without requiring force or touching // any unregistered path that still exists. - await (repo.connectionId - ? cleanupUnusedWorktreePushTargetRemoteSsh( - provider!, - repo.path, - removalTarget.id, - removedPushTarget, - store - ) - : cleanupUnusedWorktreePushTargetRemote( - repo.path, - removalTarget.id, - removedPushTarget, - store, - localWorktreeGitOptions - )) - this.clearOptimisticReconcileToken(removalTarget.id) - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) - this.invalidateResolvedWorktreeCache() - invalidateAuthorizedRootsCache() - this.notifyWorktreesChanged(repo.id) - return {} + return this.runWithSuccessfulCleanupTeardown(removalTarget.id, async () => { + await (repo.connectionId + ? cleanupUnusedWorktreePushTargetRemoteSsh( + provider!, + repo.path, + removalTarget.id, + removedPushTarget, + store + ) + : cleanupUnusedWorktreePushTargetRemote( + repo.path, + removalTarget.id, + removedPushTarget, + store, + localWorktreeGitOptions + )) + this.clearOptimisticReconcileToken(removalTarget.id) + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) + this.invalidateResolvedWorktreeCache() + invalidateAuthorizedRootsCache() + this.notifyWorktreesChanged(repo.id) + return {} + }) } throw new Error(`Refusing to delete unregistered worktree path: ${removalTarget.path}`) } @@ -17148,32 +17154,34 @@ export class OrcaRuntimeService { removedMeta && (await isRuntimeWorktreePathMissing(repo, canonicalWorktreePath, localWorktreeGitOptions)) ) { - const removalResult = await removeStaleLocalWorktreeRegistrationAfterFilesystemRemoval({ - canonicalWorktreePath, - repoPath: repo.path, - localWorktreeGitOptions, - registeredWorktree, - deleteBranch + return this.runWithSuccessfulCleanupTeardown(removalTarget.id, async () => { + const removalResult = await removeStaleLocalWorktreeRegistrationAfterFilesystemRemoval({ + canonicalWorktreePath, + repoPath: repo.path, + localWorktreeGitOptions, + registeredWorktree, + deleteBranch + }) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + removalTarget.id, + removedPushTarget, + store, + localWorktreeGitOptions + ) + this.rememberPreservedBranchCleanupTarget( + removalTarget.id, + removalResult, + registeredWorktree.head, + removedPushTarget + ) + this.clearOptimisticReconcileToken(removalTarget.id) + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.invalidateResolvedWorktreeCache() + invalidateAuthorizedRootsCache() + this.notifyWorktreesChanged(repo.id) + return removalResult ?? {} }) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - removalTarget.id, - removedPushTarget, - store, - localWorktreeGitOptions - ) - this.rememberPreservedBranchCleanupTarget( - removalTarget.id, - removalResult, - registeredWorktree.head, - removedPushTarget - ) - this.clearOptimisticReconcileToken(removalTarget.id) - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.invalidateResolvedWorktreeCache() - invalidateAuthorizedRootsCache() - this.notifyWorktreesChanged(repo.id) - return removalResult ?? {} } if (repo.connectionId) { const remoteRemoveOptions = !deleteBranch ? { deleteBranch } : {} @@ -17258,7 +17266,6 @@ export class OrcaRuntimeService { throw new Error(formatWorktreeRemovalError(error, canonicalWorktreePath, force)) } - let shouldTearDownPtys = true if (repo.symlinkPaths && repo.symlinkPaths.length > 0) { await removeWorktreeLinkedPaths(canonicalWorktreePath, repo.symlinkPaths) } @@ -17270,9 +17277,8 @@ export class OrcaRuntimeService { if (!isOrphanCompatiblePreflightError(error)) { throw new Error(formatWorktreeRemovalError(error, canonicalWorktreePath, force)) } - // Why: orphan cleanup does not need live shells to be killed first, - // and preflight did not prove the worktree is cleanly removable. - shouldTearDownPtys = false + // Why: missing/non-repo paths may still have a live deleted-cwd PTY; + // continue to the fenced teardown before orphan recovery mutates state. } const releaseWorktreePtyTeardown = await this.beginWorktreePtyTeardown(removalTarget.id) @@ -17282,10 +17288,9 @@ export class OrcaRuntimeService { await closeLocalWatcherForWorktreePath(canonicalWorktreePath).catch((err) => { console.warn(`[filesystem-watcher] failed to close ${canonicalWorktreePath}:`, err) }) - if (localProvider && shouldTearDownPtys) { - // Why: once preflight proves normal deletion is clean, kill PTYs before - // git-level removal so Windows handles cannot keep the directory busy. This also - // closes the headless-CLI leak for confirmed-removable worktrees. + if (localProvider) { + // Why: kill PTYs before git-level removal so Windows handles cannot + // keep the directory busy and deleted-cwd shells cannot survive recovery. await killAllProcessesForWorktree(removalTarget.id, { runtime: this, localProvider, @@ -18854,6 +18859,25 @@ export class OrcaRuntimeService { return this.worktreePtyAdmission.runTeardown(worktreeId, operation) } + private runWithSuccessfulCleanupTeardown( + worktreeId: string, + operation: () => Promise + ): Promise { + return this.runWithWorktreePtyTeardown(worktreeId, async () => { + // Why: orphan and stale-registration cleanup still retires an Orca + // workspace; its terminals must not outlive the last addressable owner. + const localProvider = this.getLocalProvider() + if (localProvider) { + await killAllProcessesForWorktree(worktreeId, { + runtime: this, + localProvider, + onPtyStopped: this.onPtyStopped ?? undefined + }) + } + return operation() + }) + } + async runWithWorktreeRemovalDedup( worktreeId: string, opts: { force: boolean; runHooks: boolean; hostId?: string; mode?: 'remove' | 'forget' }, @@ -18892,20 +18916,26 @@ export class OrcaRuntimeService { // Why: generic terminal.stop must not act on a stale renderer graph, while // deletion runs under admission and owns main's authoritative PTY records. const graphEpoch = opts.worktreeTeardown ? null : this.captureReadyGraphEpoch() - const worktree = await this.resolveWorktreeSelector(worktreeSelector) + const exactTeardownTarget = opts.worktreeTeardown + ? parseExactWorktreeIdSelector(worktreeSelector) + : null + // Why: successful orphan/missing-path cleanup has authoritative metadata + // but no live Git row for selector resolution; deletion owns the exact ID. + const worktreeId = + exactTeardownTarget?.id ?? (await this.resolveWorktreeSelector(worktreeSelector)).id if (graphEpoch !== null) { this.assertStableReadyGraph(graphEpoch) } const ptyIds = new Set() if (!opts.worktreeTeardown) { for (const leaf of this.leaves.values()) { - if (leaf.worktreeId === worktree.id && leaf.ptyId) { + if (leaf.worktreeId === worktreeId && leaf.ptyId) { ptyIds.add(leaf.ptyId) } } } for (const pty of this.ptysById.values()) { - if (pty.worktreeId === worktree.id && pty.connected) { + if (pty.worktreeId === worktreeId && pty.connected) { ptyIds.add(pty.ptyId) } } From 48b3dc9be022fc46c9299504a234eee34c85dd88 Mon Sep 17 00:00:00 2001 From: Brennan Benson Date: Sun, 12 Jul 2026 15:32:16 -0700 Subject: [PATCH 09/36] fix: fail closed on unverified SSH teardown --- .../ipc/pty-startup-barrier-ordering.test.ts | 2 +- src/main/ipc/worktrees.ts | 12 ++- src/main/runtime/orca-runtime.test.ts | 73 +++++++++++++++++++ src/main/runtime/orca-runtime.ts | 7 +- src/main/runtime/worktree-teardown.test.ts | 17 +++++ src/main/runtime/worktree-teardown.ts | 12 +++ 6 files changed, 118 insertions(+), 5 deletions(-) diff --git a/src/main/ipc/pty-startup-barrier-ordering.test.ts b/src/main/ipc/pty-startup-barrier-ordering.test.ts index 78f7477f468..3ab97765291 100644 --- a/src/main/ipc/pty-startup-barrier-ordering.test.ts +++ b/src/main/ipc/pty-startup-barrier-ordering.test.ts @@ -11,7 +11,7 @@ describe('PTY startup barrier ordering', () => { const runtimeSpawnStart = source.indexOf('spawn: async (args) => {') const runtimeSpawnEnd = source.indexOf(' write:', runtimeSpawnStart) const runtimeSpawn = source.slice(runtimeSpawnStart, runtimeSpawnEnd) - const rendererSpawnStart = source.indexOf("ipcMain.handle(\n 'pty:spawn'") + const rendererSpawnStart = source.indexOf("registerWorktreePtySpawnHandler(\n 'pty:spawn'") const rendererSpawnEnd = source.indexOf("ipcMain.handle(\n 'pty:kill'", rendererSpawnStart) const rendererSpawn = source.slice(rendererSpawnStart, rendererSpawnEnd) diff --git a/src/main/ipc/worktrees.ts b/src/main/ipc/worktrees.ts index a4c9f6df286..172cc740939 100644 --- a/src/main/ipc/worktrees.ts +++ b/src/main/ipc/worktrees.ts @@ -1404,6 +1404,14 @@ export function registerWorktreeHandlers( typeof runtime.beginWorktreePtyTeardown === 'function' ? runtime.beginWorktreePtyTeardown(worktreeId) : Promise.resolve(() => {}) + const runWithWorktreeRemovalDedup = ( + worktreeId: string, + opts: { force: boolean; runHooks: boolean; hostId?: string; mode?: 'remove' | 'forget' }, + operation: () => Promise + ): Promise => + typeof runtime.runWithWorktreeRemovalDedup === 'function' + ? runtime.runWithWorktreeRemovalDedup(worktreeId, opts, operation) + : operation() const runWithSuccessfulCleanupTeardown = ( worktreeId: string, operation: () => Promise @@ -1426,7 +1434,7 @@ export function registerWorktreeHandlers( ipcMain.handle(channel, (event, args: RemoveWorktreeArgs) => { const { repoId } = parseWorktreeId(args.worktreeId) const repo = getRepoForWorktreeRemoval(store, repoId, args.hostId) - return runtime.runWithWorktreeRemovalDedup( + return runWithWorktreeRemovalDedup( args.worktreeId, { force: args.force === true, @@ -1966,7 +1974,7 @@ export function registerWorktreeHandlers( ipcMain.handle(channel, (event, args: Pick) => { const { repoId } = parseWorktreeId(args.worktreeId) const repo = getRepoForWorktreeRemoval(store, repoId, args.hostId) - return runtime.runWithWorktreeRemovalDedup( + return runWithWorktreeRemovalDedup( args.worktreeId, { force: false, diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index b3ab954a0a6..6589ff3519b 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -22357,6 +22357,22 @@ describe('OrcaRuntimeService', () => { await expect(stopping).resolves.toEqual({ stopped: 1 }) }) + it('reports exact teardown PTYs whose verified stop failed', async () => { + const runtime = new OrcaRuntimeService(store) + const stopAndWait = vi.fn(async () => false) + runtime.setPtyController({ + write: () => true, + kill: () => false, + stopAndWait, + getForegroundProcess: async () => null + }) + runtime.registerPty('ssh:ssh-1@@failed', TEST_WORKTREE_ID, 'ssh-1') + + await expect( + runtime.stopTerminalsForWorktree(TEST_WORKTREE_ID, { worktreeTeardown: true }) + ).resolves.toEqual({ stopped: 0, failedPtyIds: ['ssh:ssh-1@@failed'] }) + }) + it('keeps generic worktree terminal stops graceful', async () => { const runtime = new OrcaRuntimeService(store) const kill = vi.fn(() => true) @@ -29660,6 +29676,63 @@ describe('OrcaRuntimeService', () => { unregisterSshGitProvider('ssh-1') } }) + + it('does not remove a remote worktree when its SSH PTY stop is unverified', async () => { + const remoteRepo = { + ...store.getRepo(TEST_REPO_ID)!, + path: '/remote/repo', + connectionId: 'ssh-1' + } + const remoteWorktree = { + path: '/remote/feature-wt', + head: 'def456', + branch: 'feature/test', + isBare: false, + isMainWorktree: false + } + const remoteWorktreeId = `${remoteRepo.id}::${remoteWorktree.path}` + const runtimeStore = { + ...store, + getRepos: () => [remoteRepo], + getRepo: (id: string) => (id === remoteRepo.id ? remoteRepo : undefined), + getAllWorktreeMeta: () => ({ [remoteWorktreeId]: makeWorktreeMeta() }), + getWorktreeMeta: (id: string) => (id === remoteWorktreeId ? makeWorktreeMeta() : undefined) + } + const gitProvider = { + listWorktrees: vi.fn().mockResolvedValue([ + { + path: remoteRepo.path, + head: 'main', + branch: 'main', + isBare: false, + isMainWorktree: true + }, + remoteWorktree + ]), + removeWorktree: vi.fn() + } + const localProvider = createProviderStub(async () => []) + registerSshGitProvider('ssh-1', gitProvider as never) + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { + getLocalProvider: () => localProvider as never + }) + runtime.setPtyController({ + write: () => true, + kill: () => false, + stopAndWait: vi.fn(async () => false), + getForegroundProcess: async () => null + }) + runtime.registerPty('ssh:ssh-1@@survivor', remoteWorktreeId, 'ssh-1') + + try { + await expect(runtime.removeManagedWorktree(remoteWorktreeId)).rejects.toThrow( + 'Failed to stop remote worktree terminals: ssh:ssh-1@@survivor' + ) + expect(gitProvider.removeWorktree).not.toHaveBeenCalled() + } finally { + unregisterSshGitProvider('ssh-1') + } + }) }) describe('stale terminal handle resolution (#7718)', () => { function syncSingleTerminalGraph(runtime: OrcaRuntimeService, ptyId: string): void { diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 5b89ca0f7d0..de909eb149e 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -18912,7 +18912,7 @@ export class OrcaRuntimeService { async stopTerminalsForWorktree( worktreeSelector: string, opts: { worktreeTeardown?: boolean } = {} - ): Promise<{ stopped: number }> { + ): Promise<{ stopped: number; failedPtyIds?: string[] }> { // Why: generic terminal.stop must not act on a stale renderer graph, while // deletion runs under admission and owns main's authoritative PTY records. const graphEpoch = opts.worktreeTeardown ? null : this.captureReadyGraphEpoch() @@ -18941,6 +18941,7 @@ export class OrcaRuntimeService { } let stopped = 0 + const failedPtyIds: string[] = [] for (const ptyId of ptyIds) { // Why: worktree removal immediately follows this with provider/registry // sweeps. Await the verified stop so those sweeps cannot overlap a @@ -18950,12 +18951,14 @@ export class OrcaRuntimeService { if (opts.worktreeTeardown && this.ptyController?.stopAndWait) { if (await this.ptyController.stopAndWait(ptyId)) { stopped += 1 + } else { + failedPtyIds.push(ptyId) } } else if (this.ptyController?.kill(ptyId)) { stopped += 1 } } - return { stopped } + return failedPtyIds.length > 0 ? { stopped, failedPtyIds } : { stopped } } async stopExactTerminalsForWorktree( diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index 4ba3d4c9050..386e0224749 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -163,6 +163,23 @@ describe('killAllProcessesForWorktree', () => { expect(result.runtimeStopped).toBe(3) }) + it('fails closed when an SSH-owned runtime PTY cannot be verified stopped', async () => { + const stopTerminalsForWorktree = vi.fn().mockResolvedValue({ + stopped: 0, + failedPtyIds: ['ssh:ssh-1@@relay-pty'] + }) + const runtime = { + stopTerminalsForWorktree + } as unknown as Parameters[1]['runtime'] + const localProvider = createProviderStub(async () => []) + listRegisteredPtysMock.mockReturnValue([]) + + await expect(killAllProcessesForWorktree('w1', { runtime, localProvider })).rejects.toThrow( + 'Failed to stop remote worktree terminals: ssh:ssh-1@@relay-pty' + ) + expect(localProvider.listProcesses).toHaveBeenCalledTimes(1) + }) + it('awaits each runtime stop before fallback sweeps and preserves unrelated sessions', async () => { const releases = new Map void>() const liveSessions = new Set(['w1@@target', 'w2@@target', 'witness@@unrelated']) diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index 1085ae30682..e60079a783a 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -1,6 +1,7 @@ import type { IPtyProvider } from '../providers/types' import type { OrcaRuntimeService } from './orca-runtime' import { listRegisteredPtys } from '../memory/pty-registry' +import { parseAppSshPtyId } from '../../shared/ssh-pty-id' export type WorktreeTeardownDeps = { runtime?: OrcaRuntimeService @@ -45,12 +46,17 @@ export async function killAllProcessesForWorktree( providerStopped: 0, registryStopped: 0 } + let failedRemotePtyIds: string[] = [] if (deps.runtime) { const r = await deps.runtime .stopTerminalsForWorktree(worktreeId, { worktreeTeardown: true }) .catch(() => ({ stopped: 0 })) result.runtimeStopped = r.stopped + failedRemotePtyIds = + 'failedPtyIds' in r + ? (r.failedPtyIds ?? []).filter((ptyId) => parseAppSshPtyId(ptyId) !== null) + : [] } result.providerStopped = await sweepProviderByPrefix( @@ -64,6 +70,12 @@ export async function killAllProcessesForWorktree( deps.onPtyStopped ) + if (failedRemotePtyIds.length > 0) { + // Why: local prefix/registry sweeps cannot prove an SSH-owned PTY dead; + // remote Git removal must not proceed after an unverified exact stop. + throw new Error(`Failed to stop remote worktree terminals: ${failedRemotePtyIds.join(', ')}`) + } + return result } From 60e7cb336f04b86d0d9459f90fd22bb9d7f3a96e Mon Sep 17 00:00:00 2001 From: Brennan Benson Date: Sun, 12 Jul 2026 15:49:53 -0700 Subject: [PATCH 10/36] docs(runtime): clarify SSH teardown contract --- src/main/runtime/worktree-teardown.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index e60079a783a..eb984400a81 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -33,9 +33,9 @@ export type WorktreeTeardownResult = { * canonical source for memory attribution; it also redundantly backstops * daemon spawns. * - * Best-effort throughout: each sweep catches its own errors. The caller - * (removeManagedWorktree, worktrees:remove IPC) must run the git-level - * removal regardless of what this returns. + * Local provider and registry sweeps remain best-effort because they overlap + * local ownership sources. An unverified SSH-owned stop fails closed because + * those local fallbacks cannot prove the remote PTY is gone before removal. */ export async function killAllProcessesForWorktree( worktreeId: string, From 4993b8ca885574bc9b60a3b0751bf5e7992780fc Mon Sep 17 00:00:00 2001 From: Brennan Benson Date: Sun, 12 Jul 2026 18:30:01 -0700 Subject: [PATCH 11/36] fix(ssh): reap full PTY session on teardown --- src/relay/pty-handler.test.ts | 28 ++++++++++++++++++++++++++++ src/relay/pty-handler.ts | 13 ++++++++++--- src/relay/pty-session-kill.test.ts | 25 +++++++++++++++++++++++++ src/relay/pty-session-kill.ts | 26 ++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 src/relay/pty-session-kill.test.ts create mode 100644 src/relay/pty-session-kill.ts diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index 3ce8702cd6b..10fb1febe2a 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -27,10 +27,18 @@ const { mockPtySpawn, mockPtyInstance } = vi.hoisted(() => ({ } })) +const { mockKillPosixPtySession } = vi.hoisted(() => ({ + mockKillPosixPtySession: vi.fn().mockResolvedValue(false) +})) + vi.mock('node-pty', () => ({ spawn: mockPtySpawn })) +vi.mock('./pty-session-kill', () => ({ + killPosixPtySession: mockKillPosixPtySession +})) + import { PtyHandler, attachIdentityMismatches } from './pty-handler' import type { RelayDispatcher } from './dispatcher' @@ -100,6 +108,8 @@ describe('PtyHandler', () => { mockPtyInstance.resize.mockReset() mockPtyInstance.kill.mockReset() mockPtyInstance.clear.mockReset() + mockKillPosixPtySession.mockReset() + mockKillPosixPtySession.mockResolvedValue(false) mockPtySpawn.mockReturnValue({ ...mockPtyInstance }) @@ -1086,9 +1096,27 @@ describe('PtyHandler', () => { await dispatcher.callRequest('pty.spawn', {}) await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + expect(mockKillPosixPtySession).toHaveBeenCalledWith(process.pid) expect(mockKill).toHaveBeenCalledWith('SIGKILL') }) + it('does not signal a recycled root pid after the POSIX session kill succeeds', async () => { + const mockKill = vi.fn() + mockKillPosixPtySession.mockResolvedValue(true) + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn() + }) + + await dispatcher.callRequest('pty.spawn', {}) + await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + + expect(mockKillPosixPtySession).toHaveBeenCalledWith(process.pid) + expect(mockKill).not.toHaveBeenCalled() + }) + it('throws for attach on nonexistent PTY', async () => { await expect(dispatcher.callRequest('pty.attach', { id: 'pty-999' })).rejects.toThrow( 'PTY "pty-999" not found' diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index f3d5c9e8d65..fca6117208a 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -24,6 +24,7 @@ import { scanForShellReady, type ShellReadyScanState } from '../main/shell-ready-marker-scanner' +import { killPosixPtySession } from './pty-session-kill' // Why: node-pty is a native addon that may not be installed on the remote. // Dynamic import keeps the require() lazy so loadPty() returns null gracefully @@ -840,9 +841,15 @@ export class PtyHandler { if (immediate) { this.releaseStartupCommand(managed) this.flushPtyOutput(id) - managed.pty.kill('SIGKILL') - // Why: SIGKILL has already reaped the child; release the ptmx fd on the - // same tick. Deferring to onExit leaves a window where the fd is live + const killedSession = await killPosixPtySession(managed.pty.pid) + if (!killedSession && !managed.disposed) { + managed.pty.kill('SIGKILL') + } + if (managed.disposed) { + return + } + // Why: SIGKILL has been delivered to the session; release the ptmx fd on + // the same tick. Deferring to onExit leaves a window where the fd is live // with a dead child. Idempotent via the disposed guard — if onExit fires // later and also calls disposeManagedPty, the second call is a no-op. disposeManagedPty(managed) diff --git a/src/relay/pty-session-kill.test.ts b/src/relay/pty-session-kill.test.ts new file mode 100644 index 00000000000..53cbbee3c08 --- /dev/null +++ b/src/relay/pty-session-kill.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it, vi } from 'vitest' +import { killPosixPtySession } from './pty-session-kill' + +describe('killPosixPtySession', () => { + it('targets the full POSIX session with a bounded argument list', async () => { + const run = vi.fn().mockResolvedValue(undefined) + + await expect(killPosixPtySession(4242, 'linux', run)).resolves.toBe(true) + + expect(run).toHaveBeenCalledWith('pkill', ['-KILL', '-s', '4242'], { timeout: 3000 }) + }) + + it('leaves Windows ConPTY teardown to node-pty', async () => { + const run = vi.fn() + + await expect(killPosixPtySession(4242, 'win32', run)).resolves.toBe(false) + expect(run).not.toHaveBeenCalled() + }) + + it('falls back cleanly when the session is already empty or pkill is unavailable', async () => { + const run = vi.fn().mockRejectedValue(new Error('pkill failed')) + + await expect(killPosixPtySession(4242, 'darwin', run)).resolves.toBe(false) + }) +}) diff --git a/src/relay/pty-session-kill.ts b/src/relay/pty-session-kill.ts new file mode 100644 index 00000000000..89b606fa317 --- /dev/null +++ b/src/relay/pty-session-kill.ts @@ -0,0 +1,26 @@ +import { execFile as execFileCallback } from 'node:child_process' +import { promisify } from 'node:util' + +type ExecFile = (file: string, args: string[], options: { timeout: number }) => Promise + +const execFile = promisify(execFileCallback) as ExecFile + +export async function killPosixPtySession( + pid: number, + platform: NodeJS.Platform = process.platform, + run: ExecFile = execFile +): Promise { + if (platform === 'win32' || !Number.isSafeInteger(pid) || pid <= 0) { + return false + } + + try { + // Why: forkpty makes the shell a session leader. Killing by SID also reaps + // background job groups that a direct node-pty SIGKILL would orphan. + await run('pkill', ['-KILL', '-s', String(pid)], { timeout: 3000 }) + return true + } catch { + // Missing pkill and an already-empty session both fall back to node-pty. + return false + } +} From 1599f5a53dd29aef2051b036b291fe6aa974ca0e Mon Sep 17 00:00:00 2001 From: Brennan Benson Date: Sun, 12 Jul 2026 18:48:01 -0700 Subject: [PATCH 12/36] fix(ssh): support Darwin PTY session teardown --- src/relay/pty-handler.test.ts | 4 +- src/relay/pty-handler.ts | 5 ++- src/relay/pty-session-kill.test.ts | 46 ++++++++++++++++++++--- src/relay/pty-session-kill.ts | 60 +++++++++++++++++++++++++++--- 4 files changed, 101 insertions(+), 14 deletions(-) diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index 10fb1febe2a..c8e9813b03a 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -1096,7 +1096,7 @@ describe('PtyHandler', () => { await dispatcher.callRequest('pty.spawn', {}) await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) - expect(mockKillPosixPtySession).toHaveBeenCalledWith(process.pid) + expect(mockKillPosixPtySession).toHaveBeenCalledWith(process.pid, undefined) expect(mockKill).toHaveBeenCalledWith('SIGKILL') }) @@ -1113,7 +1113,7 @@ describe('PtyHandler', () => { await dispatcher.callRequest('pty.spawn', {}) await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) - expect(mockKillPosixPtySession).toHaveBeenCalledWith(process.pid) + expect(mockKillPosixPtySession).toHaveBeenCalledWith(process.pid, undefined) expect(mockKill).not.toHaveBeenCalled() }) diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index fca6117208a..c7661cf1e9b 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -841,7 +841,10 @@ export class PtyHandler { if (immediate) { this.releaseStartupCommand(managed) this.flushPtyOutput(id) - const killedSession = await killPosixPtySession(managed.pty.pid) + const killedSession = await killPosixPtySession( + managed.pty.pid, + (managed.pty as unknown as { ptsName?: unknown }).ptsName + ) if (!killedSession && !managed.disposed) { managed.pty.kill('SIGKILL') } diff --git a/src/relay/pty-session-kill.test.ts b/src/relay/pty-session-kill.test.ts index 53cbbee3c08..0973db3773b 100644 --- a/src/relay/pty-session-kill.test.ts +++ b/src/relay/pty-session-kill.test.ts @@ -5,21 +5,57 @@ describe('killPosixPtySession', () => { it('targets the full POSIX session with a bounded argument list', async () => { const run = vi.fn().mockResolvedValue(undefined) - await expect(killPosixPtySession(4242, 'linux', run)).resolves.toBe(true) + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run)).resolves.toBe(true) - expect(run).toHaveBeenCalledWith('pkill', ['-KILL', '-s', '4242'], { timeout: 3000 }) + expect(run).toHaveBeenCalledWith('pkill', ['-KILL', '-s', '4242', '.*'], { + timeout: 3000 + }) + }) + + it('targets Darwin forkpty jobs through their controlling TTY', async () => { + const run = vi.fn().mockResolvedValue({ stdout: '4242\n4243\n' }) + const killProcess = vi.fn() + + await expect( + killPosixPtySession(4242, '/dev/ttys042', 'darwin', run, killProcess) + ).resolves.toBe(true) + + expect(run).toHaveBeenCalledWith('ps', ['-t', 'ttys042', '-o', 'pid='], { + timeout: 3000 + }) + expect(killProcess.mock.calls).toEqual([ + [4243, 'SIGKILL'], + [4242, 'SIGKILL'] + ]) }) it('leaves Windows ConPTY teardown to node-pty', async () => { const run = vi.fn() - await expect(killPosixPtySession(4242, 'win32', run)).resolves.toBe(false) + await expect(killPosixPtySession(4242, undefined, 'win32', run)).resolves.toBe(false) expect(run).not.toHaveBeenCalled() }) - it('falls back cleanly when the session is already empty or pkill is unavailable', async () => { + it('falls back cleanly when pkill is unavailable', async () => { const run = vi.fn().mockRejectedValue(new Error('pkill failed')) - await expect(killPosixPtySession(4242, 'darwin', run)).resolves.toBe(false) + await expect(killPosixPtySession(4242, '/dev/ttys042', 'darwin', run)).resolves.toBe(false) + }) + + it('rejects an absent or ambiguous Darwin PTY name', async () => { + const run = vi.fn() + + await expect(killPosixPtySession(4242, '/dev/ttys1,ttys2', 'darwin', run)).resolves.toBe(false) + expect(run).not.toHaveBeenCalled() + }) + + it('does not signal a Darwin TTY that no longer owns the root pid', async () => { + const run = vi.fn().mockResolvedValue({ stdout: '9999\n' }) + const killProcess = vi.fn() + + await expect( + killPosixPtySession(4242, '/dev/ttys042', 'darwin', run, killProcess) + ).resolves.toBe(false) + expect(killProcess).not.toHaveBeenCalled() }) }) diff --git a/src/relay/pty-session-kill.ts b/src/relay/pty-session-kill.ts index 89b606fa317..b2288eff55b 100644 --- a/src/relay/pty-session-kill.ts +++ b/src/relay/pty-session-kill.ts @@ -2,25 +2,73 @@ import { execFile as execFileCallback } from 'node:child_process' import { promisify } from 'node:util' type ExecFile = (file: string, args: string[], options: { timeout: number }) => Promise +type KillProcess = (pid: number, signal: NodeJS.Signals) => void const execFile = promisify(execFileCallback) as ExecFile export async function killPosixPtySession( pid: number, + ptsName: unknown, platform: NodeJS.Platform = process.platform, - run: ExecFile = execFile + run: ExecFile = execFile, + killProcess: KillProcess = process.kill ): Promise { if (platform === 'win32' || !Number.isSafeInteger(pid) || pid <= 0) { return false } try { - // Why: forkpty makes the shell a session leader. Killing by SID also reaps - // background job groups that a direct node-pty SIGKILL would orphan. - await run('pkill', ['-KILL', '-s', String(pid)], { timeout: 3000 }) - return true + if (platform === 'linux') { + // Why: forkpty makes the Linux shell a session leader. SID targeting + // includes background job groups without a broad process inventory. + await run('pkill', ['-KILL', '-s', String(pid), '.*'], { timeout: 3000 }) + return true + } + if (platform === 'darwin') { + return await killDarwinPtyProcesses(pid, ptsName, run, killProcess) + } + return false } catch { - // Missing pkill and an already-empty session both fall back to node-pty. + // Missing platform tools and already-empty sessions fall back to node-pty. + return false + } +} + +async function killDarwinPtyProcesses( + rootPid: number, + ptsName: unknown, + run: ExecFile, + killProcess: KillProcess +): Promise { + if (typeof ptsName !== 'string') { return false } + const tty = ptsName.startsWith('/dev/') ? ptsName.slice('/dev/'.length) : ptsName + if (!/^[A-Za-z0-9._-]+$/.test(tty)) { + return false + } + // Why: Darwin pkill has no SID selector and its TTY filter does not match + // forkpty children. A targeted ps query avoids a system-wide inventory. + const result = (await run('ps', ['-t', tty, '-o', 'pid='], { timeout: 3000 })) as { + stdout?: string | Buffer + } + const pids = String(result.stdout ?? '') + .split(/\s+/) + .map(Number) + .filter((candidate) => Number.isSafeInteger(candidate) && candidate > 0) + if (!pids.includes(rootPid)) { + return false + } + // Kill the leader last so its child job groups cannot be reparented between + // the ownership snapshot and their signals. + for (const candidate of [...pids.filter((entry) => entry !== rootPid), rootPid]) { + try { + killProcess(candidate, 'SIGKILL') + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') { + return false + } + } + } + return true } From d4126819537c1d05b1d19ed274fd509b4de3c563 Mon Sep 17 00:00:00 2001 From: Brennan Benson Date: Sun, 12 Jul 2026 19:07:23 -0700 Subject: [PATCH 13/36] fix(terminals): fail closed on incomplete teardown --- src/main/ipc/worktrees.test.ts | 16 +++ src/main/ipc/worktrees.ts | 20 ++-- src/main/runtime/orca-runtime.test.ts | 62 ++++++++++- src/main/runtime/orca-runtime.ts | 75 ++++++++----- src/main/runtime/worktree-teardown.test.ts | 79 ++++++++++++-- src/main/runtime/worktree-teardown.ts | 119 ++++++++++++--------- src/relay/pty-handler.test.ts | 31 ++++-- src/relay/pty-handler.ts | 5 + src/relay/pty-session-kill.test.ts | 7 +- src/relay/pty-session-kill.ts | 53 +++++++-- 10 files changed, 347 insertions(+), 120 deletions(-) diff --git a/src/main/ipc/worktrees.test.ts b/src/main/ipc/worktrees.test.ts index 46263b4b942..92818b01ba0 100644 --- a/src/main/ipc/worktrees.test.ts +++ b/src/main/ipc/worktrees.test.ts @@ -6762,6 +6762,22 @@ describe('registerWorktreeHandlers', () => { }) }) + it('does not remove worktree state when verified PTY teardown fails', async () => { + const worktreeId = 'repo-1::/workspace/feature-wt' + mockKnownFeatureWorktree() + store.getWorktreeMeta.mockReturnValue(makeWorktreeMeta()) + killAllProcessesForWorktreeMock.mockRejectedValue( + new Error(`Failed to stop local worktree terminals: ${worktreeId}@@daemon`) + ) + + await expect(handlers['worktrees:remove'](null, { worktreeId })).rejects.toThrow( + 'Failed to stop local worktree terminals' + ) + + expect(removeWorktreeMock).not.toHaveBeenCalled() + expect(store.removeWorktreeMeta).not.toHaveBeenCalled() + }) + it('runs the archive hook on remove when skipArchive is not set', async () => { mockKnownFeatureWorktree() removeWorktreeMock.mockResolvedValue(undefined) diff --git a/src/main/ipc/worktrees.ts b/src/main/ipc/worktrees.ts index 172cc740939..7f08a2c3a85 100644 --- a/src/main/ipc/worktrees.ts +++ b/src/main/ipc/worktrees.ts @@ -1484,8 +1484,6 @@ export function registerWorktreeHandlers( runtime, localProvider: getLocalPtyProvider(), onPtyStopped: clearProviderPtyState - }).catch((err) => { - console.warn(`[worktree-teardown] failed for ${args.worktreeId}:`, err) }) removeWorktreeMetadataAndTransientState(store, args.worktreeId) preservedBranchCleanupByWorktreeId.delete(args.worktreeId) @@ -1834,18 +1832,14 @@ export function registerWorktreeHandlers( runtime, localProvider: getLocalPtyProvider(), onPtyStopped: clearProviderPtyState + }).then((r) => { + const total = r.runtimeStopped + r.providerStopped + r.registryStopped + if (total > 0) { + console.info( + `[worktree-teardown] ${args.worktreeId} killed runtime=${r.runtimeStopped} provider=${r.providerStopped} registry=${r.registryStopped}` + ) + } }) - .then((r) => { - const total = r.runtimeStopped + r.providerStopped + r.registryStopped - if (total > 0) { - console.info( - `[worktree-teardown] ${args.worktreeId} killed runtime=${r.runtimeStopped} provider=${r.providerStopped} registry=${r.registryStopped}` - ) - } - }) - .catch((err) => { - console.warn(`[worktree-teardown] failed for ${args.worktreeId}:`, err) - }) try { const removeOptions = { diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 6589ff3519b..9c02997ff8b 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -22373,6 +22373,49 @@ describe('OrcaRuntimeService', () => { ).resolves.toEqual({ stopped: 0, failedPtyIds: ['ssh:ssh-1@@failed'] }) }) + it('bounds concurrent verified worktree stops', async () => { + const runtime = new OrcaRuntimeService(store) + const releases: (() => void)[] = [] + let active = 0 + let peak = 0 + const stopAndWait = vi.fn( + () => + new Promise((resolve) => { + active += 1 + peak = Math.max(peak, active) + releases.push(() => { + active -= 1 + resolve(true) + }) + }) + ) + runtime.setPtyController({ + write: () => true, + kill: () => false, + stopAndWait, + getForegroundProcess: async () => null + }) + for (let index = 0; index < 20; index += 1) { + runtime.registerPty(`pty-${index}`, TEST_WORKTREE_ID, null) + } + + const stopping = runtime.stopTerminalsForWorktree(TEST_WORKTREE_ID, { + worktreeTeardown: true + }) + await vi.waitFor(() => expect(stopAndWait).toHaveBeenCalledTimes(8)) + while (stopAndWait.mock.calls.length < 20) { + const callsBeforeRelease = stopAndWait.mock.calls.length + releases.splice(0).forEach((release) => release()) + await vi.waitFor(() => + expect(stopAndWait.mock.calls.length).toBeGreaterThan(callsBeforeRelease) + ) + } + releases.splice(0).forEach((release) => release()) + + await expect(stopping).resolves.toEqual({ stopped: 20 }) + expect(peak).toBe(8) + }) + it('keeps generic worktree terminal stops graceful', async () => { const runtime = new OrcaRuntimeService(store) const kill = vi.fn(() => true) @@ -29555,6 +29598,23 @@ describe('OrcaRuntimeService', () => { expect(gitIdx).toBeGreaterThan(killIdx) }) + it('does not invoke Git removal when local provider teardown remains unverified', async () => { + const ptyId = `${TEST_WORKTREE_ID}@@daemon` + const localProvider = createProviderStub(async () => [ + { id: ptyId, cwd: '/tmp/worktree-a', title: 'shell' } + ]) + localProvider.shutdown.mockRejectedValue(new Error('daemon still alive')) + const runtime = new OrcaRuntimeService(store, undefined, { + getLocalProvider: () => localProvider as never + }) + + await expect(runtime.removeManagedWorktree(TEST_WORKTREE_ID)).rejects.toThrow( + `Failed to stop local worktree terminals: ${ptyId}` + ) + + expect(removeWorktree).not.toHaveBeenCalled() + }) + it('thunk resolves the installed provider lazily, not at construction time', async () => { // Simulates the daemon adapter being installed AFTER OrcaRuntimeService // construction (setLocalPtyProvider(routedAdapter) in daemon-init). @@ -29661,7 +29721,7 @@ describe('OrcaRuntimeService', () => { runtime.registerPty('ssh:ssh-1@@late', remoteWorktreeId, 'ssh-1') releaseLateSpawn() await vi.waitFor(() => expect(callOrder).toContain('stop:ssh:ssh-1@@existing')) - expect(callOrder).not.toContain('stop:ssh:ssh-1@@late') + expect(callOrder).toContain('stop:ssh:ssh-1@@late') expect(callOrder).not.toContain('git-remove') releaseFirstStop() diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index de909eb149e..954b3d52e14 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -2125,6 +2125,27 @@ async function hasLocalWorktreeBaseRef( ) } +async function mapPtyStopsWithConcurrency( + ptyIds: readonly string[], + concurrency: number, + stopPty: (ptyId: string) => Promise +): Promise { + const results = Array(ptyIds.length) + let nextIndex = 0 + const stopNext = async (): Promise => { + while (nextIndex < ptyIds.length) { + const index = nextIndex + nextIndex += 1 + results[index] = await stopPty(ptyIds[index]) + } + } + // Why: remote tools may each consume the full timeout. A small worker pool + // bounds resource use without making teardown latency linear in PTY count. + const workers = Array.from({ length: Math.min(concurrency, ptyIds.length) }, () => stopNext()) + await Promise.all(workers) + return results +} + export class OrcaRuntimeService { private readonly runtimeId = randomUUID() private readonly startedAt = Date.now() @@ -16959,8 +16980,6 @@ export class OrcaRuntimeService { runtime: this, localProvider, onPtyStopped: this.onPtyStopped ?? undefined - }).catch((err) => { - console.warn(`[worktree-teardown] failed for ${removalTarget.id}:`, err) }) } this.removeWorktreeMetadataAndHistory(store, removalTarget.id) @@ -17295,23 +17314,19 @@ export class OrcaRuntimeService { runtime: this, localProvider, onPtyStopped: this.onPtyStopped ?? undefined + }).then((r) => { + const total = r.runtimeStopped + r.providerStopped + r.registryStopped + if (total > 0) { + // Why (design §4.4 observability): breadcrumb lets ops + // distinguish a renderer-state-induced leak (diff-path purge + // non-empty) from a backend-induced one (nothing to kill but + // memory still pinned). Emit only when the sweep actually did + // work so steady-state logs stay quiet. + console.info( + `[worktree-teardown] ${removalTarget.id} killed runtime=${r.runtimeStopped} provider=${r.providerStopped} registry=${r.registryStopped}` + ) + } }) - .then((r) => { - const total = r.runtimeStopped + r.providerStopped + r.registryStopped - if (total > 0) { - // Why (design §4.4 observability): breadcrumb lets ops - // distinguish a renderer-state-induced leak (diff-path purge - // non-empty) from a backend-induced one (nothing to kill but - // memory still pinned). Emit only when the sweep actually did - // work so steady-state logs stay quiet. - console.info( - `[worktree-teardown] ${removalTarget.id} killed runtime=${r.runtimeStopped} provider=${r.providerStopped} registry=${r.registryStopped}` - ) - } - }) - .catch((err) => { - console.warn(`[worktree-teardown] failed for ${removalTarget.id}:`, err) - }) } try { @@ -18940,22 +18955,28 @@ export class OrcaRuntimeService { } } - let stopped = 0 - const failedPtyIds: string[] = [] - for (const ptyId of ptyIds) { + const stopPty = async (ptyId: string): Promise => { // Why: worktree removal immediately follows this with provider/registry // sweeps. Await the verified stop so those sweeps cannot overlap a // provider shutdown with a second ConPTY teardown on Windows (#8275). // Why: deletion needs provider acknowledgement before its fallback sweeps, // while the shared terminal.stop RPC must preserve graceful shell teardown. if (opts.worktreeTeardown && this.ptyController?.stopAndWait) { - if (await this.ptyController.stopAndWait(ptyId)) { - stopped += 1 - } else { - failedPtyIds.push(ptyId) - } - } else if (this.ptyController?.kill(ptyId)) { + return this.ptyController.stopAndWait(ptyId) + } + return this.ptyController?.kill(ptyId) ?? false + } + const ids = [...ptyIds] + const stopResults = opts.worktreeTeardown + ? await mapPtyStopsWithConcurrency(ids, 8, stopPty) + : await Promise.all(ids.map(stopPty)) + let stopped = 0 + const failedPtyIds: string[] = [] + for (const [index, didStop] of stopResults.entries()) { + if (didStop) { stopped += 1 + } else if (opts.worktreeTeardown) { + failedPtyIds.push(ids[index]) } } return failedPtyIds.length > 0 ? { stopped, failedPtyIds } : { stopped } diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index 386e0224749..d0ee5ba5653 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -92,7 +92,7 @@ describe('killAllProcessesForWorktree', () => { expect(onPtyStopped).toHaveBeenCalledWith('1') }) - it('best-effort: swallows errors from listProcesses and shutdown', async () => { + it('fails closed when the provider inventory cannot be read', async () => { const localProvider = createProviderStub(() => Promise.reject(new Error('boom'))) listRegisteredPtysMock.mockReturnValue([ { ptyId: 'x', worktreeId: 'w1', sessionId: null, paneKey: null, pid: 10 } @@ -101,12 +101,7 @@ describe('killAllProcessesForWorktree', () => { new Error('already dead') ) - const result = await killAllProcessesForWorktree('w1', { localProvider }) - - // listProcesses rejected → provider sweep returns 0; registry shutdown - // rejected → counted as not-killed (registry sweep currently swallows). - expect(result.providerStopped).toBe(0) - expect(result.registryStopped).toBe(0) + await expect(killAllProcessesForWorktree('w1', { localProvider })).rejects.toThrow('boom') }) it('does not let cleanup hook failures abort teardown', async () => { @@ -180,6 +175,67 @@ describe('killAllProcessesForWorktree', () => { expect(localProvider.listProcesses).toHaveBeenCalledTimes(1) }) + it('fails closed when a local runtime PTY remains after fallback shutdown rejects', async () => { + const runtime = { + stopTerminalsForWorktree: vi.fn().mockResolvedValue({ + stopped: 0, + failedPtyIds: ['w1@@daemon'] + }) + } as unknown as Parameters[1]['runtime'] + const localProvider = createProviderStub(async () => [ + { id: 'w1@@daemon', cwd: '/tmp/w1', title: 'shell' } + ]) + vi.mocked(localProvider.shutdown).mockRejectedValue(new Error('daemon still alive')) + listRegisteredPtysMock.mockReturnValue([]) + + await expect(killAllProcessesForWorktree('w1', { runtime, localProvider })).rejects.toThrow( + 'Failed to stop local worktree terminals: w1@@daemon' + ) + }) + + it('recovers a failed local runtime stop through one deduplicated provider shutdown', async () => { + const runtime = { + stopTerminalsForWorktree: vi.fn().mockResolvedValue({ + stopped: 0, + failedPtyIds: ['w1@@daemon'] + }) + } as unknown as Parameters[1]['runtime'] + const localProvider = createProviderStub(async () => [ + { id: 'w1@@daemon', cwd: '/tmp/w1', title: 'shell' } + ]) + listRegisteredPtysMock.mockReturnValue([ + { ptyId: 'w1@@daemon', worktreeId: 'w1', sessionId: null, paneKey: null, pid: 10 } + ]) + + await expect(killAllProcessesForWorktree('w1', { runtime, localProvider })).resolves.toEqual({ + runtimeStopped: 0, + providerStopped: 1, + registryStopped: 0 + }) + expect(localProvider.shutdown).toHaveBeenCalledTimes(1) + }) + + it('accepts a stale duplicate registry row only after authoritative absence verification', async () => { + const localProvider = createProviderStub( + vi + .fn() + .mockResolvedValueOnce([{ id: 'w1@@daemon', cwd: '/tmp/w1', title: 'shell' }]) + .mockResolvedValueOnce([]) + ) + vi.mocked(localProvider.shutdown).mockRejectedValue(new Error('not found')) + listRegisteredPtysMock.mockReturnValue([ + { ptyId: 'w1@@daemon', worktreeId: 'w1', sessionId: null, paneKey: null, pid: 10 } + ]) + + await expect(killAllProcessesForWorktree('w1', { localProvider })).resolves.toEqual({ + runtimeStopped: 0, + providerStopped: 0, + registryStopped: 0 + }) + expect(localProvider.listProcesses).toHaveBeenCalledTimes(2) + expect(localProvider.shutdown).toHaveBeenCalledTimes(1) + }) + it('awaits each runtime stop before fallback sweeps and preserves unrelated sessions', async () => { const releases = new Map void>() const liveSessions = new Set(['w1@@target', 'w2@@target', 'witness@@unrelated']) @@ -229,7 +285,7 @@ describe('killAllProcessesForWorktree', () => { expect(liveSessions.has('witness@@unrelated')).toBe(true) }) - it('tolerates an unexpected runtime worktree-stop failure', async () => { + it('fails closed on an unexpected runtime worktree-stop failure', async () => { const stopTerminalsForWorktree = vi.fn().mockRejectedValue(new Error('graph not ready')) const runtime = { stopTerminalsForWorktree @@ -238,8 +294,9 @@ describe('killAllProcessesForWorktree', () => { const localProvider = createProviderStub(async () => []) listRegisteredPtysMock.mockReturnValue([]) - const result = await killAllProcessesForWorktree('w1', { runtime, localProvider }) - - expect(result.runtimeStopped).toBe(0) + await expect(killAllProcessesForWorktree('w1', { runtime, localProvider })).rejects.toThrow( + 'graph not ready' + ) + expect(localProvider.listProcesses).not.toHaveBeenCalled() }) }) diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index eb984400a81..713933df091 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -33,9 +33,9 @@ export type WorktreeTeardownResult = { * canonical source for memory attribution; it also redundantly backstops * daemon spawns. * - * Local provider and registry sweeps remain best-effort because they overlap - * local ownership sources. An unverified SSH-owned stop fails closed because - * those local fallbacks cannot prove the remote PTY is gone before removal. + * Provider and registry ownership overlap, so their targets are deduplicated + * before shutdown. Any stop that cannot be verified fails closed: Git removal + * must never race a shell whose cwd still belongs to the worktree. */ export async function killAllProcessesForWorktree( worktreeId: string, @@ -46,80 +46,99 @@ export async function killAllProcessesForWorktree( providerStopped: 0, registryStopped: 0 } - let failedRemotePtyIds: string[] = [] + let failedPtyIds: string[] = [] if (deps.runtime) { - const r = await deps.runtime - .stopTerminalsForWorktree(worktreeId, { worktreeTeardown: true }) - .catch(() => ({ stopped: 0 })) + const r = await deps.runtime.stopTerminalsForWorktree(worktreeId, { + worktreeTeardown: true + }) result.runtimeStopped = r.stopped - failedRemotePtyIds = - 'failedPtyIds' in r - ? (r.failedPtyIds ?? []).filter((ptyId) => parseAppSshPtyId(ptyId) !== null) - : [] + failedPtyIds = 'failedPtyIds' in r ? (r.failedPtyIds ?? []) : [] } - result.providerStopped = await sweepProviderByPrefix( - worktreeId, - deps.localProvider, - deps.onPtyStopped - ) - result.registryStopped = await sweepRegistryForWorktree( + const failedRemotePtyIds = failedPtyIds.filter((ptyId) => parseAppSshPtyId(ptyId) !== null) + const failedLocalPtyIds = failedPtyIds.filter((ptyId) => parseAppSshPtyId(ptyId) === null) + const fallbackResult = await sweepLocalProvider( worktreeId, deps.localProvider, + failedLocalPtyIds, deps.onPtyStopped ) + result.providerStopped = fallbackResult.providerStopped + result.registryStopped = fallbackResult.registryStopped if (failedRemotePtyIds.length > 0) { - // Why: local prefix/registry sweeps cannot prove an SSH-owned PTY dead; - // remote Git removal must not proceed after an unverified exact stop. + // Why: the local provider cannot prove an SSH-owned PTY dead; remote Git + // removal must not proceed after an unverified exact stop. throw new Error(`Failed to stop remote worktree terminals: ${failedRemotePtyIds.join(', ')}`) } return result } -async function sweepProviderByPrefix( +async function sweepLocalProvider( worktreeId: string, provider: IPtyProvider, + failedLocalPtyIds: readonly string[], onPtyStopped?: (ptyId: string) => void -): Promise { +): Promise<{ providerStopped: number; registryStopped: number }> { const prefix = `${worktreeId}@@` - const sessions = await provider.listProcesses().catch(() => []) - let killed = 0 - for (const s of sessions) { - if (!s.id.startsWith(prefix)) { - continue + const sessions = await provider.listProcesses() + const targets = new Map() + for (const session of sessions) { + if (session.id.startsWith(prefix)) { + targets.set(session.id, 'provider') } - try { - await provider.shutdown(s.id, { immediate: true }) - clearStoppedPtyState(s.id, onPtyStopped) - killed += 1 - } catch { - // Already dead, or the backend dropped the session — treat as success. - killed += 1 + } + for (const entry of listRegisteredPtys()) { + if (entry.worktreeId === worktreeId && !targets.has(entry.ptyId)) { + targets.set(entry.ptyId, 'registry') + } + } + for (const ptyId of failedLocalPtyIds) { + if (!targets.has(ptyId)) { + targets.set(ptyId, 'registry') } } - return killed -} -async function sweepRegistryForWorktree( - worktreeId: string, - localProvider: IPtyProvider, - onPtyStopped?: (ptyId: string) => void -): Promise { - const entries = listRegisteredPtys().filter((r) => r.worktreeId === worktreeId) - let killed = 0 - for (const entry of entries) { - try { - await localProvider.shutdown(entry.ptyId, { immediate: true }) - clearStoppedPtyState(entry.ptyId, onPtyStopped) - killed += 1 - } catch { - /* ignore — best-effort */ + const failedShutdowns = new Map() + const stopped = new Set() + await Promise.all( + [...targets].map(async ([ptyId]) => { + try { + await provider.shutdown(ptyId, { immediate: true }) + stopped.add(ptyId) + clearStoppedPtyState(ptyId, onPtyStopped) + } catch (error) { + failedShutdowns.set(ptyId, error) + } + }) + ) + + if (failedShutdowns.size > 0) { + // Why: duplicate/stale registry rows may report "not found" after another + // owner already stopped the PTY. Only an authoritative post-sweep absence + // converts that rejection into verified success. + const remainingIds = new Set((await provider.listProcesses()).map((session) => session.id)) + const unverified = [...failedShutdowns.keys()].filter((ptyId) => remainingIds.has(ptyId)) + if (unverified.length > 0) { + throw new Error(`Failed to stop local worktree terminals: ${unverified.join(', ')}`) + } + } + + let providerStopped = 0 + let registryStopped = 0 + for (const [ptyId, owner] of targets) { + if (!stopped.has(ptyId)) { + continue + } + if (owner === 'provider') { + providerStopped += 1 + } else { + registryStopped += 1 } } - return killed + return { providerStopped, registryStopped } } function clearStoppedPtyState(ptyId: string, onPtyStopped?: (ptyId: string) => void): void { diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index c8e9813b03a..c343545d503 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -109,7 +109,7 @@ describe('PtyHandler', () => { mockPtyInstance.kill.mockReset() mockPtyInstance.clear.mockReset() mockKillPosixPtySession.mockReset() - mockKillPosixPtySession.mockResolvedValue(false) + mockKillPosixPtySession.mockResolvedValue(true) mockPtySpawn.mockReturnValue({ ...mockPtyInstance }) @@ -745,7 +745,7 @@ describe('PtyHandler', () => { expect(handler.retainedStartupCommandCount).toBe(0) expect(term.write).not.toHaveBeenCalled() - expect(killSpy).toHaveBeenCalledWith('SIGKILL') + expect(killSpy).not.toHaveBeenCalledWith('SIGKILL') }) it('increments PTY ids on each spawn', async () => { @@ -1056,7 +1056,7 @@ describe('PtyHandler', () => { id: 'pty-1', data: 'last words' }) - expect(mockKill).toHaveBeenCalledWith('SIGKILL') + expect(mockKill).not.toHaveBeenCalledWith('SIGKILL') }) it('notifies pty.exit when graceful shutdown falls back to SIGKILL', async () => { @@ -1085,7 +1085,7 @@ describe('PtyHandler', () => { expect(handler.activePtyCount).toBe(0) }) - it('kills PTY on shutdown with SIGKILL when immediate', async () => { + it('kills the full POSIX PTY session when shutdown is immediate', async () => { const mockKill = vi.fn() mockPtySpawn.mockReturnValue({ ...mockPtyInstance, @@ -1097,7 +1097,26 @@ describe('PtyHandler', () => { await dispatcher.callRequest('pty.spawn', {}) await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) expect(mockKillPosixPtySession).toHaveBeenCalledWith(process.pid, undefined) - expect(mockKill).toHaveBeenCalledWith('SIGKILL') + expect(mockKill).not.toHaveBeenCalledWith('SIGKILL') + }) + + it('fails closed when full POSIX PTY session teardown cannot be verified', async () => { + const mockKill = vi.fn() + mockKillPosixPtySession.mockResolvedValue(false) + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn() + }) + + await dispatcher.callRequest('pty.spawn', {}) + + await expect( + dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + ).rejects.toThrow('Unable to verify full PTY session teardown: pty-1') + expect(mockKill).not.toHaveBeenCalled() + expect(handler.activePtyCount).toBe(1) }) it('does not signal a recycled root pid after the POSIX session kill succeeds', async () => { @@ -1779,7 +1798,7 @@ describe('PtyHandler', () => { await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) onExitCb!({ exitCode: 0 }) - expect(mockKill).toHaveBeenCalledWith('SIGKILL') + expect(mockKill).not.toHaveBeenCalledWith('SIGKILL') expect(exits).toEqual([{ id: 'pty-1', paneKey: 'tab-shutdown:0' }]) expect(handler.activePtyCount).toBe(0) }) diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index c7661cf1e9b..95a25e01734 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -845,6 +845,11 @@ export class PtyHandler { managed.pty.pid, (managed.pty as unknown as { ptsName?: unknown }).ptsName ) + if (!killedSession && process.platform !== 'win32' && !managed.disposed) { + // Why: killing only the forkpty leader can strand HUP-resistant job + // groups in a deleted cwd. Fail closed so worktree removal is aborted. + throw new Error(`Unable to verify full PTY session teardown: ${id}`) + } if (!killedSession && !managed.disposed) { managed.pty.kill('SIGKILL') } diff --git a/src/relay/pty-session-kill.test.ts b/src/relay/pty-session-kill.test.ts index 0973db3773b..33499d13140 100644 --- a/src/relay/pty-session-kill.test.ts +++ b/src/relay/pty-session-kill.test.ts @@ -13,17 +13,18 @@ describe('killPosixPtySession', () => { }) it('targets Darwin forkpty jobs through their controlling TTY', async () => { - const run = vi.fn().mockResolvedValue({ stdout: '4242\n4243\n' }) + const run = vi.fn().mockResolvedValue({ stdout: '4242 1\n4243 4242\n4244 4243\n' }) const killProcess = vi.fn() await expect( killPosixPtySession(4242, '/dev/ttys042', 'darwin', run, killProcess) ).resolves.toBe(true) - expect(run).toHaveBeenCalledWith('ps', ['-t', 'ttys042', '-o', 'pid='], { + expect(run).toHaveBeenCalledWith('/bin/ps', ['-t', 'ttys042', '-o', 'pid=', '-o', 'ppid='], { timeout: 3000 }) expect(killProcess.mock.calls).toEqual([ + [4244, 'SIGKILL'], [4243, 'SIGKILL'], [4242, 'SIGKILL'] ]) @@ -50,7 +51,7 @@ describe('killPosixPtySession', () => { }) it('does not signal a Darwin TTY that no longer owns the root pid', async () => { - const run = vi.fn().mockResolvedValue({ stdout: '9999\n' }) + const run = vi.fn().mockResolvedValue({ stdout: '9999 1\n' }) const killProcess = vi.fn() await expect( diff --git a/src/relay/pty-session-kill.ts b/src/relay/pty-session-kill.ts index b2288eff55b..ed9d20107a0 100644 --- a/src/relay/pty-session-kill.ts +++ b/src/relay/pty-session-kill.ts @@ -49,19 +49,54 @@ async function killDarwinPtyProcesses( } // Why: Darwin pkill has no SID selector and its TTY filter does not match // forkpty children. A targeted ps query avoids a system-wide inventory. - const result = (await run('ps', ['-t', tty, '-o', 'pid='], { timeout: 3000 })) as { + const result = (await run('/bin/ps', ['-t', tty, '-o', 'pid=', '-o', 'ppid='], { + timeout: 3000 + })) as { stdout?: string | Buffer } - const pids = String(result.stdout ?? '') - .split(/\s+/) - .map(Number) - .filter((candidate) => Number.isSafeInteger(candidate) && candidate > 0) - if (!pids.includes(rootPid)) { + const rows = String(result.stdout ?? '') + .trim() + .split('\n') + .map((line) => line.trim().split(/\s+/).map(Number)) + .filter( + (row): row is [number, number] => + row.length === 2 && + row.every((candidate) => Number.isSafeInteger(candidate) && candidate >= 0) && + row[0] > 0 + ) + if (!rows.some(([candidate]) => candidate === rootPid)) { return false } - // Kill the leader last so its child job groups cannot be reparented between - // the ownership snapshot and their signals. - for (const candidate of [...pids.filter((entry) => entry !== rootPid), rootPid]) { + const parentByPid = new Map(rows) + const depth = (pid: number): number => { + let current = pid + let result = 0 + const visited = new Set() + while (current !== rootPid && !visited.has(current)) { + visited.add(current) + const parent = parentByPid.get(current) + if (!parent || !parentByPid.has(parent)) { + break + } + current = parent + result += 1 + } + return result + } + // Why: descendants must be signalled before their parents so the snapshot + // cannot be invalidated by reparenting while teardown is in progress. + const pids = rows + .map(([candidate]) => candidate) + .sort((a, b) => { + if (a === rootPid) { + return 1 + } + if (b === rootPid) { + return -1 + } + return depth(b) - depth(a) + }) + for (const candidate of pids) { try { killProcess(candidate, 'SIGKILL') } catch (error) { From 8fa167fb5203a16e03d7d02b6616e0b0c54775de Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 12 Jul 2026 23:30:39 -0700 Subject: [PATCH 14/36] fix(terminals): verify bounded teardown to completion --- src/main/daemon/daemon-server.ts | 4 +- src/main/daemon/pty-subprocess.test.ts | 30 +++- src/main/daemon/pty-subprocess.ts | 36 +++++ src/main/daemon/session.ts | 16 ++ src/main/daemon/terminal-host.test.ts | 35 +++++ src/main/daemon/terminal-host.ts | 11 ++ src/main/ipc/pty.test.ts | 23 ++- src/main/ipc/worktrees.test.ts | 40 +++++ src/main/ipc/worktrees.ts | 93 ++++++------ src/main/providers/local-pty-provider.test.ts | 86 ++++++++++- src/main/providers/local-pty-provider.ts | 93 +++++++++++- src/main/providers/ssh-pty-provider.test.ts | 41 +++-- src/main/providers/ssh-pty-provider.ts | 40 ++++- src/main/runtime/orca-runtime.test.ts | 40 ++++- src/main/runtime/orca-runtime.ts | 142 ++++++++---------- src/main/runtime/pty-stop-concurrency.ts | 24 +++ src/main/runtime/worktree-teardown.test.ts | 97 ++++++++++++ src/main/runtime/worktree-teardown.ts | 21 ++- src/relay/pty-session-kill.test.ts | 59 +++++++- src/relay/pty-session-kill.ts | 62 +++++++- 20 files changed, 801 insertions(+), 192 deletions(-) create mode 100644 src/main/runtime/pty-stop-concurrency.ts diff --git a/src/main/daemon/daemon-server.ts b/src/main/daemon/daemon-server.ts index fe87022e5c5..3f22d49d916 100644 --- a/src/main/daemon/daemon-server.ts +++ b/src/main/daemon/daemon-server.ts @@ -498,7 +498,9 @@ export class DaemonServer { sessionId: request.payload.sessionId, immediate: request.payload.immediate === true }) - this.host.kill(request.payload.sessionId, { immediate: request.payload.immediate }) + await this.host.killAndWait(request.payload.sessionId, { + immediate: request.payload.immediate + }) return {} case 'signal': diff --git a/src/main/daemon/pty-subprocess.test.ts b/src/main/daemon/pty-subprocess.test.ts index 99a35047743..cf2695348b1 100644 --- a/src/main/daemon/pty-subprocess.test.ts +++ b/src/main/daemon/pty-subprocess.test.ts @@ -10,12 +10,14 @@ const { isPwshAvailableMock, validateWorkingDirectoryMock, resolveUnixShellPathMock, - resolveAgentForegroundProcessMock + resolveAgentForegroundProcessMock, + killPosixPtySessionMock } = vi.hoisted(() => ({ spawnMock: vi.fn(), isPwshAvailableMock: vi.fn(), resolveUnixShellPathMock: vi.fn((shellPath: string) => shellPath), resolveAgentForegroundProcessMock: vi.fn(), + killPosixPtySessionMock: vi.fn(), validateWorkingDirectoryMock: vi.fn((cwd: string) => { if (cwd.includes('definitely-missing')) { throw new Error( @@ -29,6 +31,10 @@ vi.mock('node-pty', () => ({ spawn: spawnMock })) +vi.mock('../../relay/pty-session-kill', () => ({ + killPosixPtySession: killPosixPtySessionMock +})) + vi.mock('../pwsh', () => ({ isPwshAvailable: isPwshAvailableMock })) @@ -113,6 +119,8 @@ describe('createPtySubprocess', () => { beforeEach(() => { spawnMock.mockReset() + killPosixPtySessionMock.mockReset() + killPosixPtySessionMock.mockResolvedValue(true) isPwshAvailableMock.mockReset() resolveAgentForegroundProcessMock.mockReset() resolveAgentForegroundProcessMock.mockImplementation( @@ -1251,6 +1259,26 @@ describe('createPtySubprocess', () => { killSpy.mockRestore() }) + itOnPosixHost('proves the complete POSIX PTY session stopped before acknowledging', async () => { + const proc = mockPtyProcess(77) + Object.assign(proc, { ptsName: '/dev/ttys042' }) + spawnMock.mockReturnValue(proc) + const handle = createPtySubprocess({ sessionId: 'test', cols: 80, rows: 24 }) + + await expect(handle.forceKillAndWait?.()).resolves.toBe(true) + + expect(killPosixPtySessionMock).toHaveBeenCalledWith(77, '/dev/ttys042') + }) + + itOnPosixHost('fails closed when complete POSIX PTY session death is unverified', async () => { + const proc = mockPtyProcess(77) + spawnMock.mockReturnValue(proc) + killPosixPtySessionMock.mockResolvedValue(false) + const handle = createPtySubprocess({ sessionId: 'test', cols: 80, rows: 24 }) + + await expect(handle.forceKillAndWait?.()).resolves.toBe(false) + }) + it('routes onData events', () => { const proc = mockPtyProcess() spawnMock.mockReturnValue(proc) diff --git a/src/main/daemon/pty-subprocess.ts b/src/main/daemon/pty-subprocess.ts index ce1b1faf35c..0d4fa1841d7 100644 --- a/src/main/daemon/pty-subprocess.ts +++ b/src/main/daemon/pty-subprocess.ts @@ -58,6 +58,7 @@ import { parsePtySessionId } from './pty-session-id' import { getAgentForegroundContextPaths } from '../providers/agent-foreground-context-paths' import { assertSafeAgentStartupCwd, resolveSafePtyDefaultCwd } from '../providers/pty-default-cwd' import { ORCA_HERMES_STARTUP_QUERY_ENV } from '../../shared/hermes-startup-query' +import { killPosixPtySession } from '../../relay/pty-session-kill' const PANE_IDENTITY_ENV_KEYS = [ 'ORCA_PANE_KEY', @@ -893,6 +894,7 @@ export function createPtySubprocess(opts: PtySubprocessOptions): SubprocessHandl let dead = false let disposed = false let nodePtyKillIssued = false + const exitWaiters = new Set<() => void>() let cachedAgentForeground: { processName: string; refreshedAt: number } | null = null const agentForegroundContextPaths = getAgentForegroundContextPaths({ cwd: opts.cwd, @@ -1001,6 +1003,10 @@ export function createPtySubprocess(opts: PtySubprocessOptions): SubprocessHandl } proc.onExit(() => { dead = true + for (const resolve of exitWaiters) { + resolve() + } + exitWaiters.clear() cachedAgentForeground = null startupAgentForeground = null // Why: UnixTerminal.destroy() registers `_socket.once('close', () => this.kill('SIGHUP'))` @@ -1204,6 +1210,36 @@ export function createPtySubprocess(opts: PtySubprocessOptions): SubprocessHandl } } }, + forceKillAndWait: async () => { + if (dead) { + return true + } + if (process.platform !== 'win32') { + return killPosixPtySession(proc.pid, (proc as unknown as { ptsName?: unknown }).ptsName) + } + if (!nodePtyKillIssued) { + try { + nodePtyKillIssued = true + proc.kill() + } catch { + return false + } + } + if (dead) { + return true + } + return new Promise((resolve) => { + const onExit = (): void => { + clearTimeout(timeout) + resolve(true) + } + const timeout = setTimeout(() => { + exitWaiters.delete(onExit) + resolve(false) + }, 3000) + exitWaiters.add(onExit) + }) + }, signal: (sig) => { // Why: same recycled-pid hazard as forceKill. Once dead, silently drop // the signal rather than risk sending it to an unrelated process. diff --git a/src/main/daemon/session.ts b/src/main/daemon/session.ts index b9c82a48174..3aa64b097ec 100644 --- a/src/main/daemon/session.ts +++ b/src/main/daemon/session.ts @@ -65,6 +65,8 @@ export type SubprocessHandle = { clear?(): void kill(): void forceKill(): void + /** Force-stop the complete native PTY and prove no runnable owner remains. */ + forceKillAndWait?(): Promise signal(sig: string): void onData(cb: (data: string) => void): void onExit(cb: (code: number) => void): void @@ -494,6 +496,20 @@ export class Session { this.emulator.dispose() } + async forceKillAndDisposeSubprocessAndWait(): Promise { + const stopped = this.subprocess.forceKillAndWait + ? await this.subprocess.forceKillAndWait() + : (this.subprocess.forceKill(), true) + if (!stopped) { + // Why: disposing here would erase the only retryable owner while its + // process may still hold the worktree. The daemon request must fail. + throw new Error(`Unable to verify full PTY session teardown: ${this.sessionId}`) + } + this.#teardownSubprocess() + this._state = 'exited' + this.emulator.dispose() + } + /** Private: shared teardown helper called by dispose(), forceDispose(), and * forceKillAndDisposeSubprocess(). Flips `_disposed`, clears pending timers, * and forwards to subprocess.dispose() exactly once. Does NOT set `_state` — diff --git a/src/main/daemon/terminal-host.test.ts b/src/main/daemon/terminal-host.test.ts index 46e495e5f2e..6aba258dc48 100644 --- a/src/main/daemon/terminal-host.test.ts +++ b/src/main/daemon/terminal-host.test.ts @@ -390,6 +390,41 @@ describe('TerminalHost', () => { expect(host.isKilled('session-1')).toBe(true) }) + it('awaits verified immediate teardown before reaping the session', async () => { + await host.createOrAttach({ + sessionId: 'session-1', + cols: 80, + rows: 24, + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + lastSubprocess.forceKillAndWait = vi.fn(async () => true) + + await host.killAndWait('session-1', { immediate: true }) + + expect(lastSubprocess.forceKillAndWait).toHaveBeenCalledOnce() + expect(lastSubprocess.dispose).toHaveBeenCalled() + expect(host.listSessions()).toEqual([]) + expect(host.isKilled('session-1')).toBe(true) + }) + + it('retains a retryable owner when immediate teardown cannot be verified', async () => { + await host.createOrAttach({ + sessionId: 'session-1', + cols: 80, + rows: 24, + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + lastSubprocess.forceKillAndWait = vi.fn(async () => false) + + await expect(host.killAndWait('session-1', { immediate: true })).rejects.toThrow( + 'Unable to verify full PTY session teardown: session-1' + ) + + expect(lastSubprocess.dispose).not.toHaveBeenCalled() + expect(host.listSessions()).toHaveLength(1) + expect(host.isKilled('session-1')).toBe(false) + }) + it('throws for non-existent session', () => { expect(() => host.kill('missing')).toThrow('Session not found') }) diff --git a/src/main/daemon/terminal-host.ts b/src/main/daemon/terminal-host.ts index b15420fed14..92c5e6b036f 100644 --- a/src/main/daemon/terminal-host.ts +++ b/src/main/daemon/terminal-host.ts @@ -207,6 +207,17 @@ export class TerminalHost { session.kill() } + async killAndWait(sessionId: string, opts: { immediate?: boolean } = {}): Promise { + if (!opts.immediate) { + this.kill(sessionId, opts) + return + } + const session = this.getAliveSession(sessionId) + await session.forceKillAndDisposeSubprocessAndWait() + this.recordTombstone(sessionId) + this.reapSession(sessionId) + } + // Why: dispose a dead session's headless emulator and drop it from the map so // exited terminals don't pin ~5000 rows of scrollback for the daemon's life. // No-ops on live sessions (a live session must never be disposed here) and on diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index 280d727e904..9e7cac28933 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -57,7 +57,8 @@ const { setMigrationUnsupportedPtyMock, clearMigrationUnsupportedPtyMock, clearMigrationUnsupportedPtysForPaneKeyMock, - clearPaneKeyAliasesForPtyMock + clearPaneKeyAliasesForPtyMock, + killPosixPtySessionMock } = vi.hoisted(() => ({ handleMock: vi.fn(), onMock: vi.fn(), @@ -88,7 +89,8 @@ const { setMigrationUnsupportedPtyMock: vi.fn(), clearMigrationUnsupportedPtyMock: vi.fn(), clearMigrationUnsupportedPtysForPaneKeyMock: vi.fn(), - clearPaneKeyAliasesForPtyMock: vi.fn() + clearPaneKeyAliasesForPtyMock: vi.fn(), + killPosixPtySessionMock: vi.fn() })) vi.mock('electron', () => ({ @@ -128,6 +130,10 @@ vi.mock('node-pty', () => ({ spawn: spawnMock })) +vi.mock('../../relay/pty-session-kill', () => ({ + killPosixPtySession: killPosixPtySessionMock +})) + vi.mock('../opencode/hook-service', () => ({ openCodeHookService: { buildPtyEnv: openCodeBuildPtyEnvMock, @@ -327,6 +333,8 @@ describe('registerPtyHandlers', () => { chmodSyncMock.mockReset() getPathMock.mockReset() spawnMock.mockReset() + killPosixPtySessionMock.mockReset() + killPosixPtySessionMock.mockResolvedValue(true) openCodeBuildPtyEnvMock.mockReset() mimoCodeBuildPtyEnvMock.mockReset() openCodeClearPtyMock.mockReset() @@ -10868,7 +10876,7 @@ describe('registerPtyHandlers', () => { expect(piClearPtyMock).toHaveBeenCalledWith(spawnResult.id) }) - it('disposes PTY listeners before manual kill IPC', async () => { + it('keeps the retryable owner until manual kill IPC verifies session death', async () => { const onDataDisposable = makeDisposable() const onExitDisposable = makeDisposable() // Why: hold a stable reference to the kill spy. On POSIX, destroyPtyProcess @@ -10895,12 +10903,13 @@ describe('registerPtyHandlers', () => { await handlers.get('pty:kill')!(null, { id: spawnResult.id }) - expect(onDataDisposable.dispose.mock.invocationCallOrder[0]).toBeLessThan( - killSpy.mock.invocationCallOrder[0] + expect(killPosixPtySessionMock.mock.invocationCallOrder[0]).toBeLessThan( + onDataDisposable.dispose.mock.invocationCallOrder[0] ) - expect(onExitDisposable.dispose.mock.invocationCallOrder[0]).toBeLessThan( - killSpy.mock.invocationCallOrder[0] + expect(killPosixPtySessionMock.mock.invocationCallOrder[0]).toBeLessThan( + onExitDisposable.dispose.mock.invocationCallOrder[0] ) + expect(killSpy).not.toHaveBeenCalled() }) it('disposes PTY listeners before runtime controller kill', async () => { diff --git a/src/main/ipc/worktrees.test.ts b/src/main/ipc/worktrees.test.ts index 92818b01ba0..a0012e7298a 100644 --- a/src/main/ipc/worktrees.test.ts +++ b/src/main/ipc/worktrees.test.ts @@ -6778,6 +6778,24 @@ describe('registerWorktreeHandlers', () => { expect(store.removeWorktreeMeta).not.toHaveBeenCalled() }) + it('holds PTY admission until Git and metadata removal both complete', async () => { + const worktreeId = 'repo-1::/workspace/feature-wt' + const releaseTeardown = vi.fn() + mockKnownFeatureWorktree() + store.getWorktreeMeta.mockReturnValue(makeWorktreeMeta()) + removeWorktreeMock.mockResolvedValue({}) + runtimeStub.beginWorktreePtyTeardown.mockResolvedValue(releaseTeardown) + + await handlers['worktrees:remove'](null, { worktreeId }) + + expect(removeWorktreeMock.mock.invocationCallOrder[0]).toBeLessThan( + store.removeWorktreeMeta.mock.invocationCallOrder[0] + ) + expect(store.removeWorktreeMeta.mock.invocationCallOrder[0]).toBeLessThan( + releaseTeardown.mock.invocationCallOrder[0] + ) + }) + it('runs the archive hook on remove when skipArchive is not set', async () => { mockKnownFeatureWorktree() removeWorktreeMock.mockResolvedValue(undefined) @@ -8691,6 +8709,28 @@ describe('registerWorktreeHandlers', () => { expect(getSshGitProviderMock).not.toHaveBeenCalled() }) + it('keeps metadata when local terminal teardown cannot be verified', async () => { + const worktreeId = 'repo-1::/workspace/feature-wt' + store.getRepo.mockReturnValue({ + id: 'repo-1', + path: '/workspace/repo', + displayName: 'repo', + badgeColor: '#000', + addedAt: 0, + connectionId: 'removed-ssh-target' + }) + killAllProcessesForWorktreeMock.mockRejectedValue( + new Error('Failed to stop remote worktree terminals') + ) + + await expect(handlers['worktrees:forgetLocal'](null, { worktreeId })).rejects.toThrow( + 'Failed to stop remote worktree terminals' + ) + + expect(store.removeWorktreeMeta).not.toHaveBeenCalled() + expect(deleteWorktreeHistoryDirMock).not.toHaveBeenCalled() + }) + it('rejects forgetting a folder project root', async () => { const repo = { id: 'repo-folder', diff --git a/src/main/ipc/worktrees.ts b/src/main/ipc/worktrees.ts index 7f08a2c3a85..30cf404b102 100644 --- a/src/main/ipc/worktrees.ts +++ b/src/main/ipc/worktrees.ts @@ -1741,37 +1741,37 @@ export function registerWorktreeHandlers( } const remoteRemoveOptions = !deleteBranch ? { deleteBranch } : {} - const rawRemovalResult = await runWithWorktreePtyTeardown(args.worktreeId, async () => { + return runWithWorktreePtyTeardown(args.worktreeId, async () => { await killAllProcessesForWorktree(args.worktreeId, { runtime, localProvider: getLocalPtyProvider(), onPtyStopped: clearProviderPtyState }) - return Object.keys(remoteRemoveOptions).length > 0 + const rawRemovalResult = await (Object.keys(remoteRemoveOptions).length > 0 ? provider!.removeWorktree(canonicalWorktreePath, args.force, remoteRemoveOptions) - : provider!.removeWorktree(canonicalWorktreePath, args.force) + : provider!.removeWorktree(canonicalWorktreePath, args.force)) + const removalResult = preserveBranchHeadFallback( + rawRemovalResult, + registeredWorktree.head + ) + await cleanupUnusedWorktreePushTargetRemoteSsh( + provider!, + repo.path, + args.worktreeId, + removedPushTarget, + store + ) + rememberPreservedBranchCleanupTarget( + args.worktreeId, + removalResult, + registeredWorktree.head, + removedPushTarget + ) + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + notifyWorktreesChanged(mainWindow, repoId) + return removalResult ?? {} }) - const removalResult = preserveBranchHeadFallback( - rawRemovalResult, - registeredWorktree.head - ) - await cleanupUnusedWorktreePushTargetRemoteSsh( - provider!, - repo.path, - args.worktreeId, - removedPushTarget, - store - ) - rememberPreservedBranchCleanupTarget( - args.worktreeId, - removalResult, - registeredWorktree.head, - removedPushTarget - ) - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - notifyWorktreesChanged(mainWindow, repoId) - return removalResult ?? {} } const refreshedWorktrees = hasLocalWorktreeGitOptions @@ -1924,28 +1924,28 @@ export function registerWorktreeHandlers( ) } } + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + args.worktreeId, + removedPushTarget, + store, + localWorktreeGitOptions + ) + rememberPreservedBranchCleanupTarget( + args.worktreeId, + removalResult, + refreshedRegisteredWorktree.head, + removedPushTarget + ) + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + invalidateAuthorizedRootsCache() + + notifyWorktreesChanged(mainWindow, repoId) + return removalResult ?? {} } finally { releaseWorktreePtyTeardown() } - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - args.worktreeId, - removedPushTarget, - store, - localWorktreeGitOptions - ) - rememberPreservedBranchCleanupTarget( - args.worktreeId, - removalResult, - refreshedRegisteredWorktree.head, - removedPushTarget - ) - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - invalidateAuthorizedRootsCache() - - notifyWorktreesChanged(mainWindow, repoId) - return removalResult ?? {} })() worktreeRemovalsInFlight.set(inFlightKey, { optionsKey, promise: removal }) try { @@ -2021,15 +2021,12 @@ export function registerWorktreeHandlers( ) } - // Why: best-effort PTY sweep. killAllProcessesForWorktree resolves - // synchronously for a dead SSH relay (the provider tombstones the lease - // and returns without awaiting the remote), so this never hangs. + // Why: forgetting metadata releases Orca's last addressable owner. An + // unverified terminal must keep that owner intact so cleanup can retry. await killAllProcessesForWorktree(args.worktreeId, { runtime, localProvider: getLocalPtyProvider(), onPtyStopped: clearProviderPtyState - }).catch((err) => { - console.warn(`[worktree-teardown] forget-local failed for ${args.worktreeId}:`, err) }) runtime.clearOptimisticReconcileToken(args.worktreeId) diff --git a/src/main/providers/local-pty-provider.test.ts b/src/main/providers/local-pty-provider.test.ts index 66f68d6e868..b07ccea9222 100644 --- a/src/main/providers/local-pty-provider.test.ts +++ b/src/main/providers/local-pty-provider.test.ts @@ -9,7 +9,8 @@ const { mkdirSyncMock, writeFileSyncMock, spawnMock, - resolveAgentForegroundProcessMock + resolveAgentForegroundProcessMock, + killPosixPtySessionMock } = vi.hoisted(() => ({ existsSyncMock: vi.fn(), statSyncMock: vi.fn(), @@ -17,7 +18,8 @@ const { mkdirSyncMock: vi.fn(), writeFileSyncMock: vi.fn(), spawnMock: vi.fn(), - resolveAgentForegroundProcessMock: vi.fn() + resolveAgentForegroundProcessMock: vi.fn(), + killPosixPtySessionMock: vi.fn() })) vi.mock('fs', () => ({ @@ -40,6 +42,10 @@ vi.mock('node-pty', () => ({ spawn: spawnMock })) +vi.mock('../../relay/pty-session-kill', () => ({ + killPosixPtySession: killPosixPtySessionMock +})) + // Resolve PowerShell family names to deterministic absolute paths (the fs mock // above otherwise makes every probe miss). The real resolver — which skips the // Store App Execution Alias stub — is covered in @@ -123,6 +129,8 @@ describe('LocalPtyProvider', () => { accessSyncMock.mockReturnValue(undefined) mkdirSyncMock.mockReset() writeFileSyncMock.mockReset() + killPosixPtySessionMock.mockReset() + killPosixPtySessionMock.mockResolvedValue(true) resolveAgentForegroundProcessMock.mockReset() resolveAgentForegroundProcessMock.mockImplementation( async (_pid: number, fallbackProcess: string | null) => fallbackProcess @@ -942,7 +950,7 @@ describe('LocalPtyProvider', () => { }) describe('shutdown', () => { - it('kills the PTY process', async () => { + it('kills the complete POSIX PTY session', async () => { // Why: capture the spy reference before shutdown triggers onExit → // POSIX kill neutralization. After neutralization, mockProc.kill is // replaced with a non-spy no-op to close the UnixTerminal.destroy() → @@ -950,7 +958,8 @@ describe('LocalPtyProvider', () => { const killSpy = mockProc.kill const { id } = await provider.spawn({ cols: 80, rows: 24 }) await provider.shutdown(id, { immediate: true }) - expect(killSpy).toHaveBeenCalled() + expect(killPosixPtySessionMock).toHaveBeenCalledWith(12345, undefined) + expect(killSpy).not.toHaveBeenCalled() }) it('invokes onExit callback via the node-pty exit handler', async () => { @@ -963,7 +972,9 @@ describe('LocalPtyProvider', () => { it('does not destroy after an intentional Windows shutdown kill', async () => { Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) - const killSpy = vi.fn() + const killSpy = vi.fn(() => { + exitCb?.({ exitCode: -1 }) + }) const destroySpy = vi.fn(() => { killSpy() }) @@ -980,6 +991,56 @@ describe('LocalPtyProvider', () => { expect(destroySpy).not.toHaveBeenCalled() }) + it('awaits Windows native exit and does not double-kill after a timeout', async () => { + vi.useFakeTimers() + try { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const killSpy = vi.fn() + spawnMock.mockReturnValue({ + ...mockProc, + kill: killSpy + }) + const { id } = await provider.spawn({ cols: 80, rows: 24 }) + + const firstStop = provider.shutdown(id, { immediate: true }) + const firstStopRejected = expect(firstStop).rejects.toThrow( + `Unable to verify full PTY session teardown: ${id}` + ) + await vi.advanceTimersByTimeAsync(3000) + await firstStopRejected + expect((await provider.listProcesses()).some((session) => session.id === id)).toBe(true) + + const retry = provider.shutdown(id, { immediate: false }) + expect(killSpy).toHaveBeenCalledTimes(1) + exitCb?.({ exitCode: -1 }) + await expect(retry).resolves.toBeUndefined() + expect((await provider.listProcesses()).some((session) => session.id === id)).toBe(false) + } finally { + vi.useRealTimers() + } + }) + + it('retains the ConPTY owner when the native Windows kill throws', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const killError = new Error('native close failed') + spawnMock.mockReturnValue({ + ...mockProc, + kill: vi.fn(() => { + throw killError + }) + }) + const { id } = await provider.spawn({ cols: 80, rows: 24 }) + + await expect(provider.shutdown(id, { immediate: true })).rejects.toThrow(killError) + + expect((await provider.listProcesses()).some((session) => session.id === String(id))).toBe( + true + ) + Object.defineProperty(process, 'platform', { configurable: true, value: 'linux' }) + killPosixPtySessionMock.mockResolvedValue(true) + await provider.shutdown(id, { immediate: true }) + }) + it('cancels pending shell-ready startup delivery on forced shutdown', async () => { vi.useFakeTimers() try { @@ -999,6 +1060,21 @@ describe('LocalPtyProvider', () => { await provider.shutdown('nonexistent', { immediate: true }) expect(mockProc.kill).not.toHaveBeenCalled() }) + + it('retains the PTY owner when complete POSIX teardown is unverified', async () => { + killPosixPtySessionMock.mockResolvedValue(false) + const { id } = await provider.spawn({ cols: 80, rows: 24 }) + + await expect(provider.shutdown(id, { immediate: true })).rejects.toThrow( + `Unable to verify full PTY session teardown: ${id}` + ) + + expect((await provider.listProcesses()).some((session) => session.id === String(id))).toBe( + true + ) + killPosixPtySessionMock.mockResolvedValue(true) + await provider.shutdown(id, { immediate: true }) + }) }) describe('hasChildProcesses', () => { diff --git a/src/main/providers/local-pty-provider.ts b/src/main/providers/local-pty-provider.ts index 90ccae11ea6..b8f3adfe377 100644 --- a/src/main/providers/local-pty-provider.ts +++ b/src/main/providers/local-pty-provider.ts @@ -58,6 +58,7 @@ import { readWindowsConptyProcessIds } from './windows-conpty-process-membership import { shouldUseShellReadyStartupDelivery } from '../../shared/codex-startup-delivery' import { assertSafeAgentStartupCwd, resolveSafePtyDefaultCwd } from './pty-default-cwd' import { ORCA_HERMES_STARTUP_QUERY_ENV } from '../../shared/hermes-startup-query' +import { killPosixPtySession } from '../../relay/pty-session-kill' const PANE_IDENTITY_ENV_KEYS = [ 'ORCA_PANE_KEY', @@ -77,6 +78,9 @@ const ptyTerminalHandle = new Map() // to invoke/clean them up on a destroyed environment, triggering a SIGABRT. const ptyDisposables = new Map void }[]>() const ptyCleanupCallbacks = new Map void>() +const windowsImmediateKills = new WeakSet() + +const WINDOWS_PTY_EXIT_TIMEOUT_MS = 3000 let loadGeneration = 0 const ptyLoadGeneration = new Map() @@ -182,6 +186,44 @@ function clearPtyState(id: string): void { ptyLoadGeneration.delete(id) } +function waitForPtyExit( + id: string, + proc: pty.IPty +): { + promise: Promise + cancel: () => void +} { + let settled = false + let timer: NodeJS.Timeout | undefined + let resolvePromise: (exited: boolean) => void = () => {} + const finish = (exited: boolean): void => { + if (settled) { + return + } + settled = true + if (timer) { + clearTimeout(timer) + } + exitListeners.delete(onExit) + resolvePromise(exited) + } + const onExit: ExitCallback = (payload) => { + if (payload.id === id) { + finish(true) + } + } + const promise = new Promise((resolve) => { + resolvePromise = resolve + exitListeners.add(onExit) + if (ptyProcesses.get(id) !== proc) { + finish(true) + return + } + timer = setTimeout(() => finish(false), WINDOWS_PTY_EXIT_TIMEOUT_MS) + }) + return { promise, cancel: () => finish(false) } +} + /** * Allocates either a stable caller-provided PTY id or a new numeric id. */ @@ -800,7 +842,7 @@ export class LocalPtyProvider implements IPtyProvider { // Why: release the master ptmx fd on the natural-exit path — without // this, a shell that exits cleanly (the common case) never releases its // fd until the next GC. See docs/fix-pty-fd-leak.md. - destroyPtyProcess(proc) + destroyPtyProcess(proc, { alreadyKilled: windowsImmediateKills.has(proc) }) this.opts.onExit?.(id, exitCode) for (const cb of exitListeners) { cb({ id, code: exitCode }) @@ -882,11 +924,48 @@ export class LocalPtyProvider implements IPtyProvider { return { cols: proc.cols, rows: proc.rows } } - async shutdown(id: string, _opts: { immediate?: boolean; keepHistory?: boolean }): Promise { + async shutdown(id: string, opts: { immediate?: boolean; keepHistory?: boolean }): Promise { const proc = ptyProcesses.get(id) if (!proc) { return } + if (process.platform === 'win32' && windowsImmediateKills.has(proc)) { + if (!(await waitForPtyExit(id, proc).promise)) { + throw new Error(`Unable to verify full PTY session teardown: ${id}`) + } + return + } + let alreadyKilled = false + if (opts.immediate) { + if (process.platform === 'win32') { + // Why: ConPTY close completes asynchronously. Deletion must await the + // native exit event, while a timeout keeps the owner retryable. + const exitWaiter = waitForPtyExit(id, proc) + try { + windowsImmediateKills.add(proc) + proc.kill() + } catch (error) { + exitWaiter.cancel() + windowsImmediateKills.delete(proc) + if (ptyProcesses.get(id) === proc) { + throw error + } + } + if (!(await exitWaiter.promise)) { + throw new Error(`Unable to verify full PTY session teardown: ${id}`) + } + return + } else { + const killedSession = await killPosixPtySession( + proc.pid, + (proc as unknown as { ptsName?: unknown }).ptsName + ) + if (!killedSession) { + throw new Error(`Unable to verify full PTY session teardown: ${id}`) + } + alreadyKilled = true + } + } // Why: disposePtyListeners removes the onExit callback, so the natural // exit cleanup path from node-pty won't fire. Cleanup and notification // must happen unconditionally after the try/catch. @@ -894,10 +973,12 @@ export class LocalPtyProvider implements IPtyProvider { // the natural onExit callback from running the usual clearPtyState path. runPtyCleanup(id) disposePtyListeners(id) - try { - proc.kill() - } catch { - /* Process may already be dead */ + if (!alreadyKilled) { + try { + proc.kill() + } catch { + /* Generic terminal close preserves the previous already-dead behavior. */ + } } destroyPtyProcess(proc, { alreadyKilled: true }) clearPtyState(id) diff --git a/src/main/providers/ssh-pty-provider.test.ts b/src/main/providers/ssh-pty-provider.test.ts index c754c68ab85..438433af08e 100644 --- a/src/main/providers/ssh-pty-provider.test.ts +++ b/src/main/providers/ssh-pty-provider.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it, vi, beforeEach } from 'vitest' -import { SshPtyProvider } from './ssh-pty-provider' +import { + SSH_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS, + SSH_PTY_LEGACY_INVENTORY_TIMEOUT_MS, + SSH_PTY_TEARDOWN_LIVENESS_TIMEOUT_MS, + SshPtyProvider +} from './ssh-pty-provider' import { POWERLEVEL10K_WIZARD_DISABLE_ENV } from '../pty/powerlevel10k-wizard-env' import { JsonRpcErrorCode } from '../ssh/relay-protocol' @@ -419,20 +424,28 @@ describe('SshPtyProvider', () => { it('shutdown sends pty.shutdown request', async () => { await provider.shutdown(scopedPty1, { immediate: true }) - expect(mux.request).toHaveBeenCalledWith('pty.shutdown', { - id: 'pty-1', - immediate: true, - keepHistory: false - }) + expect(mux.request).toHaveBeenCalledWith( + 'pty.shutdown', + { + id: 'pty-1', + immediate: true, + keepHistory: false + }, + { timeoutMs: SSH_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS } + ) }) it('shutdown forwards keepHistory: true over the relay', async () => { await provider.shutdown(scopedPty1, { immediate: true, keepHistory: true }) - expect(mux.request).toHaveBeenCalledWith('pty.shutdown', { - id: 'pty-1', - immediate: true, - keepHistory: true - }) + expect(mux.request).toHaveBeenCalledWith( + 'pty.shutdown', + { + id: 'pty-1', + immediate: true, + keepHistory: true + }, + { timeoutMs: SSH_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS } + ) }) it('sendSignal sends pty.sendSignal request', async () => { @@ -505,6 +518,9 @@ describe('SshPtyProvider', () => { await expect(provider.hasPtyAsync('ssh:conn-1@@pty-2')).resolves.toBe(false) expect(mux.request.mock.calls.filter(([method]) => method === 'pty.hasPty')).toHaveLength(2) + for (const call of mux.request.mock.calls.filter(([method]) => method === 'pty.hasPty')) { + expect(call[2]).toEqual({ timeoutMs: SSH_PTY_TEARDOWN_LIVENESS_TIMEOUT_MS }) + } expect( mux.request.mock.calls.filter(([method]) => method === 'pty.listProcesses') ).toHaveLength(0) @@ -540,6 +556,9 @@ describe('SshPtyProvider', () => { expect( mux.request.mock.calls.filter(([method]) => method === 'pty.listProcesses') ).toHaveLength(1) + expect(mux.request.mock.calls.find(([method]) => method === 'pty.listProcesses')?.[2]).toEqual({ + timeoutMs: SSH_PTY_LEGACY_INVENTORY_TIMEOUT_MS + }) await provider.shutdown(scopedPty1, { immediate: true }) await expect(provider.hasPtyAsync(scopedPty1)).resolves.toBe(false) diff --git a/src/main/providers/ssh-pty-provider.ts b/src/main/providers/ssh-pty-provider.ts index bc271574cc8..7069527dfb8 100644 --- a/src/main/providers/ssh-pty-provider.ts +++ b/src/main/providers/ssh-pty-provider.ts @@ -17,6 +17,11 @@ type RemoteCliBridgeEnv = { export const SSH_SESSION_EXPIRED_ERROR = 'SSH_SESSION_EXPIRED' export const SSH_PTY_IDENTITY_MISMATCH_ERROR = 'SSH_PTY_IDENTITY_MISMATCH' +// Why: eight bounded workers keep a 50-PTY failure near one 30s batch, +// while leaving enough time for the relay's targeted 3s kill-and-verify path. +export const SSH_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS = 3250 +export const SSH_PTY_TEARDOWN_LIVENESS_TIMEOUT_MS = 1000 +export const SSH_PTY_LEGACY_INVENTORY_TIMEOUT_MS = 2000 export function isSshPtyNotFoundError(err: unknown): boolean { const message = err instanceof Error ? err.message : String(err) @@ -54,8 +59,15 @@ export class SshPtyProvider implements IPtyProvider { this.mux = mux this.ptyLiveness = new SshPtyLiveness({ probe: async (id) => - (await this.mux.request('pty.hasPty', { id: this.toRelayPtyId(id) })) === true, - listIds: async () => (await this.listProcesses()).map((session) => session.id) + (await this.mux.request( + 'pty.hasPty', + { id: this.toRelayPtyId(id) }, + { timeoutMs: SSH_PTY_TEARDOWN_LIVENESS_TIMEOUT_MS } + )) === true, + listIds: async () => + (await this.requestProcessList(SSH_PTY_LEGACY_INVENTORY_TIMEOUT_MS)).map( + (session) => session.id + ) }) // Subscribe to relay notifications for PTY events @@ -250,11 +262,15 @@ export class SshPtyProvider implements IPtyProvider { } async shutdown(id: string, opts: { immediate?: boolean; keepHistory?: boolean }): Promise { - await this.mux.request('pty.shutdown', { - id: this.toRelayPtyId(id), - immediate: opts.immediate ?? false, - keepHistory: opts.keepHistory ?? false - }) + await this.mux.request( + 'pty.shutdown', + { + id: this.toRelayPtyId(id), + immediate: opts.immediate ?? false, + keepHistory: opts.keepHistory ?? false + }, + opts.immediate ? { timeoutMs: SSH_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS } : undefined + ) this.ptyLiveness.markStopped(id) } @@ -302,7 +318,15 @@ export class SshPtyProvider implements IPtyProvider { } async listProcesses(): Promise { - const result = await this.mux.request('pty.listProcesses') + return this.requestProcessList() + } + + private async requestProcessList(timeoutMs?: number): Promise { + const result = await this.mux.request( + 'pty.listProcesses', + undefined, + timeoutMs === undefined ? undefined : { timeoutMs } + ) const sessions = (result as PtyProcessInfo[]).map((session) => ({ ...session, id: this.toAppPtyId(session.id) diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 9c02997ff8b..200999b5c72 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -29615,6 +29615,31 @@ describe('OrcaRuntimeService', () => { expect(removeWorktree).not.toHaveBeenCalled() }) + it('keeps spawn admission closed through post-Git metadata cleanup', async () => { + const localProvider = createProviderStub(async () => []) + const runtime = new OrcaRuntimeService(store, undefined, { + getLocalProvider: () => localProvider as never + }) + const originalRemoveWorktreeMeta = store.removeWorktreeMeta + let rejectedDuringMetadataCleanup = false + store.removeWorktreeMeta = () => { + expect(() => runtime.beginWorktreePtySpawn(TEST_WORKTREE_ID)).toThrow( + 'Worktree teardown is in progress' + ) + rejectedDuringMetadataCleanup = true + } + + try { + await runtime.removeManagedWorktree(TEST_WORKTREE_ID) + const release = runtime.beginWorktreePtySpawn(TEST_WORKTREE_ID) + release() + + expect(rejectedDuringMetadataCleanup).toBe(true) + } finally { + store.removeWorktreeMeta = originalRemoveWorktreeMeta + } + }) + it('thunk resolves the installed provider lazily, not at construction time', async () => { // Simulates the daemon adapter being installed AFTER OrcaRuntimeService // construction (setLocalPtyProvider(routedAdapter) in daemon-init). @@ -29664,12 +29689,20 @@ describe('OrcaRuntimeService', () => { isMainWorktree: false } const remoteWorktreeId = `${remoteRepo.id}::${remoteWorktree.path}` + let runtime: OrcaRuntimeService + let rejectedDuringRemoteMetadataCleanup = false const runtimeStore = { ...store, getRepos: () => [remoteRepo], getRepo: (id: string) => (id === remoteRepo.id ? remoteRepo : undefined), getAllWorktreeMeta: () => ({ [remoteWorktreeId]: makeWorktreeMeta() }), - getWorktreeMeta: (id: string) => (id === remoteWorktreeId ? makeWorktreeMeta() : undefined) + getWorktreeMeta: (id: string) => (id === remoteWorktreeId ? makeWorktreeMeta() : undefined), + removeWorktreeMeta: () => { + expect(() => runtime.beginWorktreePtySpawn(remoteWorktreeId)).toThrow( + 'Worktree teardown is in progress' + ) + rejectedDuringRemoteMetadataCleanup = true + } } const callOrder: string[] = [] const gitProvider = { @@ -29690,7 +29723,7 @@ describe('OrcaRuntimeService', () => { } const localProvider = createProviderStub(async () => []) registerSshGitProvider('ssh-1', gitProvider as never) - const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { + runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { getLocalProvider: () => localProvider as never }) let releaseFirstStop = () => {} @@ -29726,12 +29759,15 @@ describe('OrcaRuntimeService', () => { releaseFirstStop() await removal + const releasePostRemovalSpawn = runtime.beginWorktreePtySpawn(remoteWorktreeId) + releasePostRemovalSpawn() expect(callOrder).toEqual([ 'stop:ssh:ssh-1@@existing', 'stop:ssh:ssh-1@@late', 'git-remove' ]) + expect(rejectedDuringRemoteMetadataCleanup).toBe(true) } finally { unregisterSshGitProvider('ssh-1') } diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 954b3d52e14..a60c288f565 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -714,6 +714,7 @@ import { registerTerminalViewAttributesApplier } from './terminal-view-attribute-store' import { killAllProcessesForWorktree } from './worktree-teardown' +import { mapPtyStopsWithConcurrency } from './pty-stop-concurrency' import { WorktreePtyAdmission } from './worktree-pty-admission' import { MOBILE_SUBSCRIBE_SCROLLBACK_ROWS } from './scrollback-limits' import { @@ -2125,27 +2126,6 @@ async function hasLocalWorktreeBaseRef( ) } -async function mapPtyStopsWithConcurrency( - ptyIds: readonly string[], - concurrency: number, - stopPty: (ptyId: string) => Promise -): Promise { - const results = Array(ptyIds.length) - let nextIndex = 0 - const stopNext = async (): Promise => { - while (nextIndex < ptyIds.length) { - const index = nextIndex - nextIndex += 1 - results[index] = await stopPty(ptyIds[index]) - } - } - // Why: remote tools may each consume the full timeout. A small worker pool - // bounds resource use without making teardown latency linear in PTY count. - const workers = Array.from({ length: Math.min(concurrency, ptyIds.length) }, () => stopNext()) - await Promise.all(workers) - return results -} - export class OrcaRuntimeService { private readonly runtimeId = randomUUID() private readonly startedAt = Date.now() @@ -17204,45 +17184,42 @@ export class OrcaRuntimeService { } if (repo.connectionId) { const remoteRemoveOptions = !deleteBranch ? { deleteBranch } : {} - const rawRemovalResult = await this.runWithWorktreePtyTeardown( - removalTarget.id, - async () => { - const localProvider = this.getLocalProvider() - if (localProvider) { - await killAllProcessesForWorktree(removalTarget.id, { - runtime: this, - localProvider, - onPtyStopped: this.onPtyStopped ?? undefined - }) - } - return Object.keys(remoteRemoveOptions).length > 0 - ? provider!.removeWorktree(canonicalWorktreePath, force, remoteRemoveOptions) - : provider!.removeWorktree(canonicalWorktreePath, force) + return this.runWithWorktreePtyTeardown(removalTarget.id, async () => { + const localProvider = this.getLocalProvider() + if (localProvider) { + await killAllProcessesForWorktree(removalTarget.id, { + runtime: this, + localProvider, + onPtyStopped: this.onPtyStopped ?? undefined + }) } - ) - const removalResult = this.preserveBranchHeadFallback( - rawRemovalResult, - registeredWorktree.head - ) - await cleanupUnusedWorktreePushTargetRemoteSsh( - provider!, - repo.path, - removalTarget.id, - removedPushTarget, - store - ) - this.rememberPreservedBranchCleanupTarget( - removalTarget.id, - removalResult, - registeredWorktree.head, - removedPushTarget - ) - this.clearOptimisticReconcileToken(removalTarget.id) - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.invalidateResolvedWorktreeCache() - invalidateAuthorizedRootsCache() - this.notifyWorktreesChanged(repo.id) - return removalResult ?? {} + const rawRemovalResult = await (Object.keys(remoteRemoveOptions).length > 0 + ? provider!.removeWorktree(canonicalWorktreePath, force, remoteRemoveOptions) + : provider!.removeWorktree(canonicalWorktreePath, force)) + const removalResult = this.preserveBranchHeadFallback( + rawRemovalResult, + registeredWorktree.head + ) + await cleanupUnusedWorktreePushTargetRemoteSsh( + provider!, + repo.path, + removalTarget.id, + removedPushTarget, + store + ) + this.rememberPreservedBranchCleanupTarget( + removalTarget.id, + removalResult, + registeredWorktree.head, + removedPushTarget + ) + this.clearOptimisticReconcileToken(removalTarget.id) + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.invalidateResolvedWorktreeCache() + invalidateAuthorizedRootsCache() + this.notifyWorktreesChanged(repo.id) + return removalResult ?? {} + }) } const hooks = getEffectiveHooks(repo) @@ -17408,32 +17385,31 @@ export class OrcaRuntimeService { throw new Error(formatWorktreeRemovalError(error, canonicalWorktreePath, force)) } } + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + removalTarget.id, + removedPushTarget, + store, + localWorktreeGitOptions + ) + this.rememberPreservedBranchCleanupTarget( + removalTarget.id, + removalResult, + refreshedRegisteredWorktree.head, + removedPushTarget + ) + this.clearOptimisticReconcileToken(removalTarget.id) + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.invalidateResolvedWorktreeCache() + invalidateAuthorizedRootsCache() + this.notifyWorktreesChanged(repo.id) + return { + ...removalResult, + ...(warning ? { warning } : {}) + } } finally { releaseWorktreePtyTeardown() } - - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - removalTarget.id, - removedPushTarget, - store, - localWorktreeGitOptions - ) - this.rememberPreservedBranchCleanupTarget( - removalTarget.id, - removalResult, - refreshedRegisteredWorktree.head, - removedPushTarget - ) - this.clearOptimisticReconcileToken(removalTarget.id) - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.invalidateResolvedWorktreeCache() - invalidateAuthorizedRootsCache() - this.notifyWorktreesChanged(repo.id) - return { - ...removalResult, - ...(warning ? { warning } : {}) - } })() this.removeManagedWorktreeInFlight.set(removalTarget.id, { optionsKey, promise: removal }) try { @@ -18968,7 +18944,7 @@ export class OrcaRuntimeService { } const ids = [...ptyIds] const stopResults = opts.worktreeTeardown - ? await mapPtyStopsWithConcurrency(ids, 8, stopPty) + ? await mapPtyStopsWithConcurrency(ids, stopPty) : await Promise.all(ids.map(stopPty)) let stopped = 0 const failedPtyIds: string[] = [] diff --git a/src/main/runtime/pty-stop-concurrency.ts b/src/main/runtime/pty-stop-concurrency.ts new file mode 100644 index 00000000000..7784766a19a --- /dev/null +++ b/src/main/runtime/pty-stop-concurrency.ts @@ -0,0 +1,24 @@ +export const WORKTREE_PTY_STOP_CONCURRENCY = 8 + +export async function mapPtyStopsWithConcurrency( + ptyIds: readonly string[], + stopPty: (ptyId: string) => Promise +): Promise { + const results = Array(ptyIds.length) + let nextIndex = 0 + const stopNext = async (): Promise => { + while (nextIndex < ptyIds.length) { + const index = nextIndex + nextIndex += 1 + results[index] = await stopPty(ptyIds[index]) + } + } + // Why: remote tools may each consume the full timeout. A small worker pool + // bounds resource use without making teardown latency linear in PTY count. + const workers = Array.from( + { length: Math.min(WORKTREE_PTY_STOP_CONCURRENCY, ptyIds.length) }, + () => stopNext() + ) + await Promise.all(workers) + return results +} diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index d0ee5ba5653..0f2fccfd3f6 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -38,6 +38,24 @@ function createProviderStub( } as unknown as IPtyProvider } +async function releaseBoundedShutdownWaves( + shutdown: ReturnType, + pending: { settle: () => void }[], + targetCount: number +): Promise { + await vi.waitFor(() => expect(shutdown).toHaveBeenCalledTimes(8)) + let waves = 0 + while (shutdown.mock.calls.length < targetCount) { + waves += 1 + const callsBeforeRelease = shutdown.mock.calls.length + pending.splice(0).forEach(({ settle }) => settle()) + await vi.waitFor(() => expect(shutdown.mock.calls.length).toBeGreaterThan(callsBeforeRelease)) + } + waves += 1 + pending.splice(0).forEach(({ settle }) => settle()) + return waves +} + describe('killAllProcessesForWorktree', () => { beforeEach(() => { listRegisteredPtysMock.mockReset() @@ -236,6 +254,85 @@ describe('killAllProcessesForWorktree', () => { expect(localProvider.shutdown).toHaveBeenCalledTimes(1) }) + it('bounds a 50-session headless fallback sweep to seven shutdown waves', async () => { + const sessions = Array.from({ length: 50 }, (_, index) => ({ + id: `w1@@${index}`, + cwd: '/tmp/w1', + title: 'shell' + })) + const localProvider = createProviderStub(async () => sessions) + const pending: { settle: () => void }[] = [] + let active = 0 + let peak = 0 + vi.mocked(localProvider.shutdown).mockImplementation( + () => + new Promise((resolve) => { + active += 1 + peak = Math.max(peak, active) + pending.push({ + settle: () => { + active -= 1 + resolve() + } + }) + }) + ) + listRegisteredPtysMock.mockReturnValue([]) + + const teardown = killAllProcessesForWorktree('w1', { localProvider }) + const waves = await releaseBoundedShutdownWaves( + vi.mocked(localProvider.shutdown), + pending, + sessions.length + ) + + await expect(teardown).resolves.toEqual({ + runtimeStopped: 0, + providerStopped: 50, + registryStopped: 0 + }) + expect(peak).toBe(8) + expect(waves).toBe(7) + }) + + it('waits for seven bounded failure waves before rejecting a 50-session sweep', async () => { + const sessions = Array.from({ length: 50 }, (_, index) => ({ + id: `w1@@${index}`, + cwd: '/tmp/w1', + title: 'shell' + })) + const localProvider = createProviderStub(async () => sessions) + const pending: { settle: () => void }[] = [] + let active = 0 + let peak = 0 + vi.mocked(localProvider.shutdown).mockImplementation( + () => + new Promise((_resolve, reject) => { + active += 1 + peak = Math.max(peak, active) + pending.push({ + settle: () => { + active -= 1 + reject(new Error('shutdown timed out after 3000ms')) + } + }) + }) + ) + listRegisteredPtysMock.mockReturnValue([]) + + const teardown = killAllProcessesForWorktree('w1', { localProvider }) + const waves = await releaseBoundedShutdownWaves( + vi.mocked(localProvider.shutdown), + pending, + sessions.length + ) + + await expect(teardown).rejects.toThrow('Failed to stop local worktree terminals') + expect(localProvider.listProcesses).toHaveBeenCalledTimes(2) + expect(peak).toBe(8) + expect(waves).toBe(7) + }) + it('awaits each runtime stop before fallback sweeps and preserves unrelated sessions', async () => { const releases = new Map void>() const liveSessions = new Set(['w1@@target', 'w2@@target', 'witness@@unrelated']) diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index 713933df091..57ba4676455 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -2,6 +2,7 @@ import type { IPtyProvider } from '../providers/types' import type { OrcaRuntimeService } from './orca-runtime' import { listRegisteredPtys } from '../memory/pty-registry' import { parseAppSshPtyId } from '../../shared/ssh-pty-id' +import { mapPtyStopsWithConcurrency } from './pty-stop-concurrency' export type WorktreeTeardownDeps = { runtime?: OrcaRuntimeService @@ -103,17 +104,15 @@ async function sweepLocalProvider( const failedShutdowns = new Map() const stopped = new Set() - await Promise.all( - [...targets].map(async ([ptyId]) => { - try { - await provider.shutdown(ptyId, { immediate: true }) - stopped.add(ptyId) - clearStoppedPtyState(ptyId, onPtyStopped) - } catch (error) { - failedShutdowns.set(ptyId, error) - } - }) - ) + await mapPtyStopsWithConcurrency([...targets.keys()], async (ptyId) => { + try { + await provider.shutdown(ptyId, { immediate: true }) + stopped.add(ptyId) + clearStoppedPtyState(ptyId, onPtyStopped) + } catch (error) { + failedShutdowns.set(ptyId, error) + } + }) if (failedShutdowns.size > 0) { // Why: duplicate/stale registry rows may report "not found" after another diff --git a/src/relay/pty-session-kill.test.ts b/src/relay/pty-session-kill.test.ts index 33499d13140..56dc133cd25 100644 --- a/src/relay/pty-session-kill.test.ts +++ b/src/relay/pty-session-kill.test.ts @@ -1,19 +1,29 @@ import { describe, expect, it, vi } from 'vitest' -import { killPosixPtySession } from './pty-session-kill' +import { + killPosixPtySession, + PTY_SESSION_COMMAND_TIMEOUT_MS, + PTY_SESSION_VERIFY_TIMEOUT_MS +} from './pty-session-kill' describe('killPosixPtySession', () => { it('targets the full POSIX session with a bounded argument list', async () => { - const run = vi.fn().mockResolvedValue(undefined) + const run = vi.fn().mockResolvedValueOnce(undefined).mockResolvedValueOnce({ stdout: 'Z\n' }) await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run)).resolves.toBe(true) expect(run).toHaveBeenCalledWith('pkill', ['-KILL', '-s', '4242', '.*'], { - timeout: 3000 + timeout: PTY_SESSION_COMMAND_TIMEOUT_MS + }) + expect(run).toHaveBeenCalledWith('/bin/ps', ['-s', '4242', '-o', 'stat='], { + timeout: PTY_SESSION_VERIFY_TIMEOUT_MS }) }) it('targets Darwin forkpty jobs through their controlling TTY', async () => { - const run = vi.fn().mockResolvedValue({ stdout: '4242 1\n4243 4242\n4244 4243\n' }) + const run = vi + .fn() + .mockResolvedValueOnce({ stdout: '4242 1\n4243 4242\n4244 4243\n' }) + .mockResolvedValueOnce({ stdout: '4242 Z\n4243 Z\n4244 Z\n' }) const killProcess = vi.fn() await expect( @@ -21,13 +31,18 @@ describe('killPosixPtySession', () => { ).resolves.toBe(true) expect(run).toHaveBeenCalledWith('/bin/ps', ['-t', 'ttys042', '-o', 'pid=', '-o', 'ppid='], { - timeout: 3000 + timeout: PTY_SESSION_COMMAND_TIMEOUT_MS }) expect(killProcess.mock.calls).toEqual([ [4244, 'SIGKILL'], [4243, 'SIGKILL'], [4242, 'SIGKILL'] ]) + expect(run).toHaveBeenLastCalledWith( + '/bin/ps', + ['-t', 'ttys042', '-o', 'pid=', '-o', 'stat='], + { timeout: PTY_SESSION_VERIFY_TIMEOUT_MS } + ) }) it('leaves Windows ConPTY teardown to node-pty', async () => { @@ -59,4 +74,38 @@ describe('killPosixPtySession', () => { ).resolves.toBe(false) expect(killProcess).not.toHaveBeenCalled() }) + + it('fails closed when Linux session members remain runnable after SIGKILL', async () => { + const run = vi.fn().mockResolvedValueOnce(undefined).mockResolvedValueOnce({ stdout: 'D\n' }) + + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run)).resolves.toBe(false) + }) + + it('fails closed when Darwin signals do not remove the captured processes', async () => { + const run = vi + .fn() + .mockResolvedValueOnce({ stdout: '4242 1\n4243 4242\n' }) + .mockResolvedValueOnce({ stdout: '4242 S\n4243 S\n' }) + const killProcess = vi.fn() + + await expect( + killPosixPtySession(4242, '/dev/ttys042', 'darwin', run, killProcess) + ).resolves.toBe(false) + expect(killProcess).toHaveBeenCalledTimes(2) + }) + + it('accepts an exact empty-process ps exit after signalling', async () => { + const emptySelection = Object.assign(new Error('no processes'), { code: 1, stdout: '' }) + const run = vi.fn().mockResolvedValueOnce(undefined).mockRejectedValueOnce(emptySelection) + + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run)).resolves.toBe(true) + }) + + it('verifies an already-empty Linux session after pkill finds no match', async () => { + const emptySelection = Object.assign(new Error('no processes'), { code: 1, stdout: '' }) + const run = vi.fn().mockRejectedValueOnce(emptySelection).mockRejectedValueOnce(emptySelection) + + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run)).resolves.toBe(true) + expect(run).toHaveBeenCalledTimes(2) + }) }) diff --git a/src/relay/pty-session-kill.ts b/src/relay/pty-session-kill.ts index ed9d20107a0..2aa551b0b19 100644 --- a/src/relay/pty-session-kill.ts +++ b/src/relay/pty-session-kill.ts @@ -5,6 +5,8 @@ type ExecFile = (file: string, args: string[], options: { timeout: number }) => type KillProcess = (pid: number, signal: NodeJS.Signals) => void const execFile = promisify(execFileCallback) as ExecFile +export const PTY_SESSION_COMMAND_TIMEOUT_MS = 2500 +export const PTY_SESSION_VERIFY_TIMEOUT_MS = 500 export async function killPosixPtySession( pid: number, @@ -21,8 +23,16 @@ export async function killPosixPtySession( if (platform === 'linux') { // Why: forkpty makes the Linux shell a session leader. SID targeting // includes background job groups without a broad process inventory. - await run('pkill', ['-KILL', '-s', String(pid), '.*'], { timeout: 3000 }) - return true + try { + await run('pkill', ['-KILL', '-s', String(pid), '.*'], { + timeout: PTY_SESSION_COMMAND_TIMEOUT_MS + }) + } catch (error) { + if (!isEmptyProcessSelection(error)) { + throw error + } + } + return await verifyLinuxSessionStopped(pid, run) } if (platform === 'darwin') { return await killDarwinPtyProcesses(pid, ptsName, run, killProcess) @@ -50,7 +60,7 @@ async function killDarwinPtyProcesses( // Why: Darwin pkill has no SID selector and its TTY filter does not match // forkpty children. A targeted ps query avoids a system-wide inventory. const result = (await run('/bin/ps', ['-t', tty, '-o', 'pid=', '-o', 'ppid='], { - timeout: 3000 + timeout: PTY_SESSION_COMMAND_TIMEOUT_MS })) as { stdout?: string | Buffer } @@ -105,5 +115,49 @@ async function killDarwinPtyProcesses( } } } - return true + return await verifyDarwinTtyStopped(tty, run) +} + +async function verifyLinuxSessionStopped(pid: number, run: ExecFile): Promise { + try { + const result = (await run('/bin/ps', ['-s', String(pid), '-o', 'stat='], { + timeout: PTY_SESSION_VERIFY_TIMEOUT_MS + })) as { stdout?: string | Buffer } + return hasOnlyExitedProcesses(result.stdout) + } catch (error) { + return isEmptyProcessSelection(error) + } +} + +async function verifyDarwinTtyStopped(tty: string, run: ExecFile): Promise { + try { + const result = (await run('/bin/ps', ['-t', tty, '-o', 'pid=', '-o', 'stat='], { + timeout: PTY_SESSION_VERIFY_TIMEOUT_MS + })) as { stdout?: string | Buffer } + return hasOnlyExitedProcesses(result.stdout, true) + } catch (error) { + return isEmptyProcessSelection(error) + } +} + +function hasOnlyExitedProcesses(stdout: string | Buffer | undefined, pidColumn = false): boolean { + const lines = String(stdout ?? '') + .trim() + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + return lines.every((line) => { + const stat = pidColumn ? line.split(/\s+/).at(-1) : line + return stat?.startsWith('Z') === true + }) +} + +function isEmptyProcessSelection(error: unknown): boolean { + if (!error || typeof error !== 'object') { + return false + } + const processError = error as { code?: unknown; stdout?: unknown } + // Why: BSD/procps ps exits 1 for an empty selector. Only that exact empty + // result proves absence; ENOENT, timeout, and malformed output fail closed. + return processError.code === 1 && String(processError.stdout ?? '').trim() === '' } From 7dc6aea265380258feee5ada48233bffbba26d63 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 13 Jul 2026 00:04:21 -0700 Subject: [PATCH 15/36] fix(terminals): fail closed across remote teardown gaps --- src/main/ipc/pty.test.ts | 82 +++++++++++++- src/main/ipc/pty.ts | 35 +++--- src/main/providers/ssh-pty-liveness.test.ts | 65 +++++++++++ src/main/providers/ssh-pty-liveness.ts | 45 ++++++-- src/main/providers/ssh-pty-provider.test.ts | 29 +++++ src/main/providers/ssh-pty-provider.ts | 6 +- src/main/runtime/orca-runtime.test.ts | 102 ++++++++++++++++++ src/main/runtime/orca-runtime.ts | 16 ++- .../runtime/worktree-pty-admission.test.ts | 22 +++- src/main/runtime/worktree-pty-admission.ts | 29 ++++- src/relay/pty-handler.test.ts | 90 ++++++++++++++++ src/relay/pty-handler.ts | 95 ++++++++++++++-- 12 files changed, 583 insertions(+), 33 deletions(-) create mode 100644 src/main/providers/ssh-pty-liveness.test.ts diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index 9e7cac28933..7980c751a8d 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -2736,6 +2736,59 @@ describe('registerPtyHandlers', () => { expect(runtime.onPtyExit).toHaveBeenCalledWith('remote-pty', -1) }) + it('keeps graceful SSH ownership while the provider still reports the PTY live', async () => { + const shutdown = vi.fn(async () => undefined) + const store = { markSshRemotePtyLease: vi.fn() } + const runtime = { setPtyController: vi.fn(), onPtyExit: vi.fn() } + registerSshPtyProvider('ssh-1', { + spawn: vi.fn(), + write: vi.fn(), + resize: vi.fn(), + shutdown, + sendSignal: vi.fn(), + getCwd: vi.fn(), + getInitialCwd: vi.fn(), + clearBuffer: vi.fn(), + acknowledgeDataEvent: vi.fn(), + hasChildProcesses: vi.fn(), + hasPtyAsync: vi.fn(async () => true), + getForegroundProcess: vi.fn(), + serialize: vi.fn(), + revive: vi.fn(), + onData: vi.fn(() => () => {}), + onReplay: vi.fn(() => () => {}), + onExit: vi.fn(() => () => {}), + listProcesses: vi.fn(async () => []), + attach: vi.fn(), + getDefaultShell: vi.fn(), + getProfiles: vi.fn() + } as never) + setPtyOwnership('remote-pty', 'ssh-1') + handlers.clear() + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + const controller = runtime.setPtyController.mock.calls[0]?.[0] as { + kill: (ptyId: string) => boolean + } + + try { + expect(controller.kill('remote-pty')).toBe(true) + await new Promise((resolve) => setImmediate(resolve)) + } finally { + deletePtyOwnership('remote-pty') + } + + expect(shutdown).toHaveBeenCalledWith('remote-pty', { immediate: false }) + expect(store.markSshRemotePtyLease).not.toHaveBeenCalled() + expect(runtime.onPtyExit).not.toHaveBeenCalled() + }) + it('controller kill does not duplicate exits when the provider emits exit during shutdown', async () => { const exitListeners = new Set<(payload: { id: string; code: number }) => void>() const shutdown = vi.fn(async (id: string) => { @@ -3318,6 +3371,33 @@ describe('registerPtyHandlers', () => { expect(runtime.onPtyExit).toHaveBeenCalledWith('remote-pty', -1) }) + it('fails deletion-only stop closed when a detached SSH provider is absent', async () => { + const store = { markSshRemotePtyLease: vi.fn() } + const runtime = { setPtyController: vi.fn(), onPtyExit: vi.fn() } + setPtyOwnership('remote-pty', 'ssh-1') + handlers.clear() + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + const controller = runtime.setPtyController.mock.calls[0]?.[0] as { + stopAndWait: (ptyId: string) => Promise + } + + try { + await expect(controller.stopAndWait('remote-pty')).resolves.toBe(false) + } finally { + deletePtyOwnership('remote-pty') + } + + expect(store.markSshRemotePtyLease).not.toHaveBeenCalled() + expect(runtime.onPtyExit).not.toHaveBeenCalled() + }) + it('preserves an SSH lease when runtime controller kill shutdown fails transiently', async () => { const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) const store = { @@ -3376,7 +3456,7 @@ describe('registerPtyHandlers', () => { 'remote-pty', 'terminated' ) - expect(runtime.onPtyExit).toHaveBeenCalledWith('remote-pty', -1) + expect(runtime.onPtyExit).not.toHaveBeenCalled() }) it('strips ORCA_PANE_KEY/TAB_ID/WORKTREE_ID from SSH spawn env when remote agent hooks are disabled', async () => { diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 4d9e6edc360..5f6cad574e0 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -3354,7 +3354,23 @@ export function registerPtyHandlers( // provider emitted its own exit during shutdown, the exit listener already // delivered runtime + renderer exits — synthesizing again would double-fire. void shutdownProviderAndDetectExit(provider, ptyId, { immediate: false }) - .then((providerExitObserved) => { + .then(async (providerExitObserved) => { + if (!providerExitObserved) { + try { + if (await isProviderPtyLive(provider, ptyId)) { + // Why: graceful shutdown only acknowledges signal delivery. + // Keep ownership until the provider's real exit event arrives. + return + } + } catch (err) { + console.warn( + `[pty] Failed to verify graceful PTY stop ${ptyId}: ${ + err instanceof Error ? err.message : String(err) + }` + ) + return + } + } finishPtyShutdown(ptyId, connectionId, store) if (!providerExitObserved) { runtime?.onPtyExit(ptyId, -1) @@ -3373,11 +3389,8 @@ export function registerPtyHandlers( console.warn( `[pty] Failed to stop PTY ${ptyId}: ${err instanceof Error ? err.message : String(err)}` ) - // Why: callers of controller.kill must observe a kill→exit pair so - // runtime tail buffers close and agents stop treating the pane as - // live. Preserve provider/lease state so a retry can still target - // the remote PTY if it survived the transient failure. - runtime?.onPtyExit(ptyId, -1) + // Why: preserve runtime/provider ownership so deletion can still + // target a PTY whose graceful close failed or remains unverified. }) return true }, @@ -3390,13 +3403,9 @@ export function registerPtyHandlers( provider = connectionId ? getProvider(connectionId) : getProviderForPty(ptyId) } catch { if (connectionId) { - // Why: an absent SSH provider means there is no live target left to - // await, but the relay lease must still be tombstoned. - finishPtyShutdown(ptyId, connectionId, store) - runtime?.onPtyExit(ptyId, -1) - rememberSyntheticKillExit(ptyId) - sendPtyExitToRenderer({ id: ptyId, code: -1 }) - return true + // Why: disconnect unregisters the provider while the relay PTY and + // durable lease intentionally survive. Deletion must fail closed. + return false } return false } diff --git a/src/main/providers/ssh-pty-liveness.test.ts b/src/main/providers/ssh-pty-liveness.test.ts new file mode 100644 index 00000000000..59a00e00057 --- /dev/null +++ b/src/main/providers/ssh-pty-liveness.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it, vi } from 'vitest' +import { JsonRpcErrorCode } from '../ssh/relay-protocol' +import { SshPtyLiveness } from './ssh-pty-liveness' + +function methodNotFound(): Error & { code: number } { + return Object.assign(new Error('Method not found'), { code: JsonRpcErrorCode.MethodNotFound }) +} + +describe('SshPtyLiveness', () => { + it('clears transient legacy overrides after an inventory rejection', async () => { + let rejectInventory = (_error: Error): void => {} + const listIds = vi.fn( + () => + new Promise((_resolve, reject) => { + rejectInventory = reject + }) + ) + const liveness = new SshPtyLiveness({ + probe: vi.fn().mockRejectedValue(methodNotFound()), + listIds + }) + const checking = liveness.hasPty('pty-probe') + await vi.waitFor(() => expect(listIds).toHaveBeenCalledTimes(1)) + + for (let index = 0; index < 100; index += 1) { + liveness.markStopped(`pty-${index}`) + } + rejectInventory(new Error('connection lost')) + await expect(checking).rejects.toThrow('connection lost') + + expect( + (liveness as unknown as { legacyMembershipOverrides: Map }) + .legacyMembershipOverrides.size + ).toBe(0) + }) + + it('bounds override growth and invalidates the in-flight legacy snapshot', async () => { + let resolveInventory = (_ids: string[]): void => {} + const listIds = vi.fn( + () => + new Promise((resolve) => { + resolveInventory = resolve + }) + ) + const liveness = new SshPtyLiveness({ + probe: vi.fn().mockRejectedValue(methodNotFound()), + listIds + }) + const checking = liveness.hasPty('pty-probe') + await vi.waitFor(() => expect(listIds).toHaveBeenCalledTimes(1)) + + for (let index = 0; index < 10_000; index += 1) { + liveness.markStopped(`pty-${index}`) + } + expect( + (liveness as unknown as { legacyMembershipOverrides: Map }) + .legacyMembershipOverrides.size + ).toBe(0) + + resolveInventory([]) + await expect(checking).rejects.toThrow( + 'SSH legacy PTY inventory invalidated during liveness updates' + ) + }) +}) diff --git a/src/main/providers/ssh-pty-liveness.ts b/src/main/providers/ssh-pty-liveness.ts index 0d40f63e4fa..091e2c5ddd4 100644 --- a/src/main/providers/ssh-pty-liveness.ts +++ b/src/main/providers/ssh-pty-liveness.ts @@ -5,11 +5,14 @@ type SshPtyLivenessOptions = { listIds: () => Promise } +const MAX_LEGACY_MEMBERSHIP_OVERRIDES = 256 + export class SshPtyLiveness { private supportsTargetedProbe: boolean | undefined private targetedProbeInFlight: Promise | undefined private legacyIds: Set | undefined private legacyInventoryInFlight: Promise> | undefined + private legacyInventoryAcceptsOverrides = false private legacyInventoryGeneration = 0 private legacyMembershipOverrides = new Map() @@ -20,6 +23,7 @@ export class SshPtyLiveness { this.legacyInventoryGeneration += 1 this.legacyIds = undefined this.legacyInventoryInFlight = undefined + this.legacyInventoryAcceptsOverrides = false this.legacyMembershipOverrides.clear() } @@ -81,9 +85,24 @@ export class SshPtyLiveness { } else { this.legacyIds.delete(id) } - } else if (this.supportsTargetedProbe === false) { + } else if ( + this.supportsTargetedProbe === false && + this.legacyInventoryInFlight && + this.legacyInventoryAcceptsOverrides + ) { // Why: lifecycle changes can settle while the legacy inventory RPC is // in flight; replay them onto its result before any liveness read. + if ( + !this.legacyMembershipOverrides.has(id) && + this.legacyMembershipOverrides.size >= MAX_LEGACY_MEMBERSHIP_OVERRIDES + ) { + // Why: a corrupt/hostile relay must not grow this transient map without + // bound. Invalidate the snapshot so no caller trusts dropped updates. + this.legacyInventoryGeneration += 1 + this.legacyMembershipOverrides.clear() + this.legacyInventoryAcceptsOverrides = false + return + } this.legacyMembershipOverrides.set(id, live) } } @@ -96,9 +115,14 @@ export class SshPtyLiveness { return this.legacyInventoryInFlight } const generation = this.legacyInventoryGeneration - const inventory = this.options.listIds().then((ids) => { - const liveIds = new Set(ids) - if (this.legacyInventoryGeneration === generation) { + this.legacyInventoryAcceptsOverrides = true + const inventory = this.options + .listIds() + .then((ids) => { + if (this.legacyInventoryGeneration !== generation) { + throw new Error('SSH legacy PTY inventory invalidated during liveness updates') + } + const liveIds = new Set(ids) for (const [id, live] of this.legacyMembershipOverrides) { if (live) { liveIds.add(id) @@ -108,15 +132,22 @@ export class SshPtyLiveness { } this.legacyMembershipOverrides.clear() this.legacyIds = liveIds - } - return liveIds - }) + return liveIds + }) + .catch((error) => { + // Why: a failed/invalidated inventory is not a reusable baseline. Its + // transient overrides would otherwise accumulate forever across retries. + this.legacyMembershipOverrides.clear() + throw error + }) this.legacyInventoryInFlight = inventory try { return await inventory } finally { if (this.legacyInventoryInFlight === inventory) { this.legacyInventoryInFlight = undefined + this.legacyInventoryAcceptsOverrides = false + this.legacyMembershipOverrides.clear() } } } diff --git a/src/main/providers/ssh-pty-provider.test.ts b/src/main/providers/ssh-pty-provider.test.ts index 438433af08e..d3ae0dd0ca0 100644 --- a/src/main/providers/ssh-pty-provider.test.ts +++ b/src/main/providers/ssh-pty-provider.test.ts @@ -435,6 +435,35 @@ describe('SshPtyProvider', () => { ) }) + it('keeps graceful legacy liveness authoritative until the relay exit event', async () => { + let notification: ((method: string, params: Record) => void) | undefined + mux.onNotification.mockImplementation((callback) => { + notification = callback + return () => {} + }) + mux.request.mockImplementation(async (method: string) => { + if (method === 'pty.shutdown') { + return undefined + } + if (method === 'pty.hasPty') { + throw Object.assign(new Error('Method not found'), { + code: JsonRpcErrorCode.MethodNotFound + }) + } + if (method === 'pty.listProcesses') { + return [{ id: 'pty-1', cwd: '/work/demo', title: 'shell' }] + } + throw new Error(`Unexpected request: ${method}`) + }) + provider = new SshPtyProvider('conn-1', mux as never) + + await provider.shutdown(scopedPty1, { immediate: false }) + await expect(provider.hasPtyAsync(scopedPty1)).resolves.toBe(true) + + notification?.('pty.exit', { id: 'pty-1', code: 0 }) + await expect(provider.hasPtyAsync(scopedPty1)).resolves.toBe(false) + }) + it('shutdown forwards keepHistory: true over the relay', async () => { await provider.shutdown(scopedPty1, { immediate: true, keepHistory: true }) expect(mux.request).toHaveBeenCalledWith( diff --git a/src/main/providers/ssh-pty-provider.ts b/src/main/providers/ssh-pty-provider.ts index 7069527dfb8..529e4d2596f 100644 --- a/src/main/providers/ssh-pty-provider.ts +++ b/src/main/providers/ssh-pty-provider.ts @@ -271,7 +271,11 @@ export class SshPtyProvider implements IPtyProvider { }, opts.immediate ? { timeoutMs: SSH_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS } : undefined ) - this.ptyLiveness.markStopped(id) + // Why: graceful shutdown only acknowledges SIGTERM delivery; the relay + // can remain alive until its exit event or 5s fallback proves death. + if (opts.immediate) { + this.ptyLiveness.markStopped(id) + } } async sendSignal(id: string, signal: string): Promise { diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 200999b5c72..c69c8f32758 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -22373,6 +22373,37 @@ describe('OrcaRuntimeService', () => { ).resolves.toEqual({ stopped: 0, failedPtyIds: ['ssh:ssh-1@@failed'] }) }) + it('targets lease-only SSH PTYs during worktree teardown', async () => { + const runtime = new OrcaRuntimeService({ + ...store, + getSshRemotePtyLeases: () => [ + { + targetId: 'ssh-1', + ptyId: 'relay-only', + worktreeId: TEST_WORKTREE_ID, + state: 'detached' as const, + createdAt: 1, + updatedAt: 2 + } + ] + }) + const stopAndWait = vi.fn(async () => false) + runtime.setPtyController({ + write: () => true, + kill: () => false, + stopAndWait, + getForegroundProcess: async () => null + }) + + await expect( + runtime.stopTerminalsForWorktree(TEST_WORKTREE_ID, { worktreeTeardown: true }) + ).resolves.toEqual({ + stopped: 0, + failedPtyIds: ['ssh:ssh-1@@relay-only'] + }) + expect(stopAndWait).toHaveBeenCalledWith('ssh:ssh-1@@relay-only') + }) + it('bounds concurrent verified worktree stops', async () => { const runtime = new OrcaRuntimeService(store) const releases: (() => void)[] = [] @@ -29829,6 +29860,77 @@ describe('OrcaRuntimeService', () => { unregisterSshGitProvider('ssh-1') } }) + + it('preserves remote metadata when a lease-only PTY provider is disconnected', async () => { + const remoteRepo = { + ...store.getRepo(TEST_REPO_ID)!, + path: '/remote/repo', + connectionId: 'ssh-1' + } + const remoteWorktree = { + path: '/remote/feature-wt', + head: 'def456', + branch: 'feature/test', + isBare: false, + isMainWorktree: false + } + const remoteWorktreeId = `${remoteRepo.id}::${remoteWorktree.path}` + const removeWorktreeMeta = vi.fn() + const runtimeStore = { + ...store, + getRepos: () => [remoteRepo], + getRepo: (id: string) => (id === remoteRepo.id ? remoteRepo : undefined), + getAllWorktreeMeta: () => ({ [remoteWorktreeId]: makeWorktreeMeta() }), + getWorktreeMeta: (id: string) => (id === remoteWorktreeId ? makeWorktreeMeta() : undefined), + removeWorktreeMeta, + getSshRemotePtyLeases: () => [ + { + targetId: 'ssh-1', + ptyId: 'detached-survivor', + worktreeId: remoteWorktreeId, + state: 'detached' as const, + createdAt: 1, + updatedAt: 2 + } + ] + } + const gitProvider = { + listWorktrees: vi.fn().mockResolvedValue([ + { + path: remoteRepo.path, + head: 'main', + branch: 'main', + isBare: false, + isMainWorktree: true + }, + remoteWorktree + ]), + removeWorktree: vi.fn() + } + const localProvider = createProviderStub(async () => []) + registerSshGitProvider('ssh-1', gitProvider as never) + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { + getLocalProvider: () => localProvider as never + }) + const stopAndWait = vi.fn(async () => false) + runtime.setPtyController({ + write: () => true, + kill: () => false, + stopAndWait, + getForegroundProcess: async () => null + }) + + try { + await expect(runtime.removeManagedWorktree(remoteWorktreeId)).rejects.toThrow( + 'Failed to stop remote worktree terminals: ssh:ssh-1@@detached-survivor' + ) + expect(stopAndWait).toHaveBeenCalledWith('ssh:ssh-1@@detached-survivor') + expect(gitProvider.removeWorktree).not.toHaveBeenCalled() + expect(removeWorktreeMeta).not.toHaveBeenCalled() + } finally { + unregisterSshGitProvider('ssh-1') + } + }) }) describe('stale terminal handle resolution (#7718)', () => { function syncSingleTerminalGraph(runtime: OrcaRuntimeService, ptyId: string): void { diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 67364e353a8..5d8a0236d51 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -203,7 +203,7 @@ import { import { TASK_PROVIDERS } from '../../shared/task-providers' import { FIRST_PANE_ID } from '../../shared/pane-key' import { isTerminalLeafId, makePaneKey, parsePaneKey } from '../../shared/stable-pane-id' -import { parseAppSshPtyId } from '../../shared/ssh-pty-id' +import { parseAppSshPtyId, toAppSshPtyId } from '../../shared/ssh-pty-id' import { isValidHostTerminalTabId, isValidTerminalTabId } from '../../shared/terminal-tab-id' import { buildAgentDraftLaunchPlan, buildAgentStartupPlan } from '../../shared/tui-agent-startup' import { repoIsRemote } from '../../shared/agent-launch-remote' @@ -823,6 +823,7 @@ type RuntimeStore = { getWorkspaceSession?: Store['getWorkspaceSession'] setWorkspaceSession?: Store['setWorkspaceSession'] persistPtyBinding?: Store['persistPtyBinding'] + getSshRemotePtyLeases?: Store['getSshRemotePtyLeases'] getUI?: Store['getUI'] updateUI?: Store['updateUI'] recordFeatureInteraction?: Store['recordFeatureInteraction'] @@ -18913,6 +18914,19 @@ export class OrcaRuntimeService { ptyIds.add(pty.ptyId) } } + if (opts.worktreeTeardown) { + for (const lease of this.store?.getSshRemotePtyLeases?.() ?? []) { + if ( + lease.worktreeId === worktreeId && + lease.state !== 'terminated' && + lease.state !== 'expired' + ) { + // Why: durable leases are the only authoritative owner after a + // restart or disconnect has removed renderer/runtime PTY records. + ptyIds.add(toAppSshPtyId(lease.targetId, lease.ptyId)) + } + } + } const stopPty = async (ptyId: string): Promise => { // Why: worktree removal immediately follows this with provider/registry diff --git a/src/main/runtime/worktree-pty-admission.test.ts b/src/main/runtime/worktree-pty-admission.test.ts index bb83ab9d956..fc7815b62ce 100644 --- a/src/main/runtime/worktree-pty-admission.test.ts +++ b/src/main/runtime/worktree-pty-admission.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest' -import { WorktreePtyAdmission } from './worktree-pty-admission' +import { WORKTREE_PTY_SPAWN_DRAIN_TIMEOUT_MS, WorktreePtyAdmission } from './worktree-pty-admission' describe('WorktreePtyAdmission', () => { it('drains admitted spawns and rejects new ones until teardown settles', async () => { @@ -63,4 +63,24 @@ describe('WorktreePtyAdmission', () => { await first expect(order).toEqual(['first']) }) + + it('fails a stuck drain closed and reopens admission without leaking the waiter', async () => { + vi.useFakeTimers() + try { + const admission = new WorktreePtyAdmission() + const releaseStuckSpawn = admission.beginSpawn('w1') + const teardown = admission.runTeardown('w1', vi.fn()) + const rejection = expect(teardown).rejects.toThrow('Timed out draining PTY spawns: w1') + + await vi.advanceTimersByTimeAsync(WORKTREE_PTY_SPAWN_DRAIN_TIMEOUT_MS) + await rejection + + const releaseNextSpawn = admission.beginSpawn('w1') + releaseNextSpawn() + releaseStuckSpawn() + await expect(admission.runTeardown('w1', async () => 'removed')).resolves.toBe('removed') + } finally { + vi.useRealTimers() + } + }) }) diff --git a/src/main/runtime/worktree-pty-admission.ts b/src/main/runtime/worktree-pty-admission.ts index f3fb2211cb9..b530603c67c 100644 --- a/src/main/runtime/worktree-pty-admission.ts +++ b/src/main/runtime/worktree-pty-admission.ts @@ -4,6 +4,8 @@ type WorktreePtyAdmissionState = { drainWaiters: Set<() => void> } +export const WORKTREE_PTY_SPAWN_DRAIN_TIMEOUT_MS = 30_000 + export class WorktreePtyAdmission { private readonly states = new Map() @@ -50,7 +52,32 @@ export class WorktreePtyAdmission { } state.teardownOwners += 1 if (state.activeSpawns > 0) { - await new Promise((resolve) => state.drainWaiters.add(resolve)) + let resolveDrain = (): void => {} + let timer: NodeJS.Timeout | undefined + const drain = new Promise((resolve) => { + resolveDrain = resolve + state.drainWaiters.add(resolve) + }) + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error(`Timed out draining PTY spawns: ${worktreeId}`)), + WORKTREE_PTY_SPAWN_DRAIN_TIMEOUT_MS + ) + }) + try { + await Promise.race([drain, timeout]) + } catch (error) { + // Why: deletion failed before touching Git. Reopen admission so the + // still-running spawn can finish in the worktree that remains. + state.drainWaiters.delete(resolveDrain) + state.teardownOwners -= 1 + this.deleteIfIdle(worktreeId, state) + throw error + } finally { + if (timer) { + clearTimeout(timer) + } + } } let released = false return () => { diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index c343545d503..00a0983f076 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -1100,6 +1100,96 @@ describe('PtyHandler', () => { expect(mockKill).not.toHaveBeenCalledWith('SIGKILL') }) + it('awaits Windows ConPTY exit after immediate shutdown without a signal argument', async () => { + const originalPlatform = process.platform + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + const mockKill = vi.fn() + const mockDestroy = vi.fn() + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + destroy: mockDestroy, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + }) + try { + await dispatcher.callRequest('pty.spawn', {}) + const shutdown = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + + expect(mockKill).toHaveBeenCalledWith() + expect(handler.activePtyCount).toBe(1) + onExitCb!({ exitCode: 0 }) + await expect(shutdown).resolves.toBeUndefined() + + expect(mockKill).toHaveBeenCalledTimes(1) + expect(mockDestroy).not.toHaveBeenCalled() + expect(handler.activePtyCount).toBe(0) + } finally { + Object.defineProperty(process, 'platform', { configurable: true, value: originalPlatform }) + } + }) + + it('retains a Windows ConPTY owner when immediate exit times out', async () => { + const originalPlatform = process.platform + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const mockKill = vi.fn() + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn() + }) + try { + await dispatcher.callRequest('pty.spawn', {}) + const shutdown = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + const rejection = expect(shutdown).rejects.toThrow( + 'Timed out waiting for Windows PTY teardown: pty-1' + ) + await vi.advanceTimersByTimeAsync(3_000) + + await rejection + expect(mockKill).toHaveBeenCalledTimes(1) + expect(handler.activePtyCount).toBe(1) + } finally { + Object.defineProperty(process, 'platform', { configurable: true, value: originalPlatform }) + } + }) + + it('waits on retry without killing a Windows ConPTY twice', async () => { + const originalPlatform = process.platform + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + const mockKill = vi.fn() + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + }) + try { + await dispatcher.callRequest('pty.spawn', {}) + const first = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + const firstRejection = expect(first).rejects.toThrow( + 'Timed out waiting for Windows PTY teardown: pty-1' + ) + await vi.advanceTimersByTimeAsync(3_000) + await firstRejection + + const retry = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + expect(mockKill).toHaveBeenCalledTimes(1) + onExitCb!({ exitCode: 0 }) + await expect(retry).resolves.toBeUndefined() + expect(mockKill).toHaveBeenCalledTimes(1) + } finally { + Object.defineProperty(process, 'platform', { configurable: true, value: originalPlatform }) + } + }) + it('fails closed when full POSIX PTY session teardown cannot be verified', async () => { const mockKill = vi.fn() mockKillPosixPtySession.mockResolvedValue(false) diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index 95a25e01734..e1b3fcae19b 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -50,6 +50,8 @@ type ManagedPty = { buffered: string /** Timer for SIGKILL fallback after a graceful SIGTERM shutdown. */ killTimer?: ReturnType + /** Windows ConPTY kill has no signal argument and completes asynchronously. */ + windowsImmediateKillIssued?: boolean /** True once disposeManagedPty has run. Prevents double-dispose (onExit + an * explicit shutdown can both fire for the same PTY) and converts post-dispose * entry-point calls into a clean "not found" error instead of a silent no-op @@ -109,13 +111,16 @@ function disposeManagedPty(managed: ManagedPty): void { if (process.platform !== 'win32') { ;(managed.pty as unknown as { kill: (sig?: string) => void }).kill = () => {} } - try { - ;(managed.pty as unknown as { destroy?: () => void }).destroy?.() - } catch { - /* swallow */ + if (!managed.windowsImmediateKillIssued) { + try { + ;(managed.pty as unknown as { destroy?: () => void }).destroy?.() + } catch { + /* swallow */ + } } } const DEFAULT_GRACE_TIME_MS = DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS * 1000 +const WINDOWS_IMMEDIATE_EXIT_TIMEOUT_MS = 3_000 export const REPLAY_BUFFER_MAX = 100 * 1024 const PTY_OUTPUT_BATCH_INTERVAL_MS = 8 const PTY_OUTPUT_DRAIN_CONTINUE_MS = 1 @@ -238,6 +243,7 @@ export class PtyHandler { private pendingOutputByPty = new Map() private lastInputAtByPty = new Map() private interactiveOutputCharsByPty = new Map() + private exitWaitersByPty = new Map void>>() // Why: external observers need to drop per-pane state when a PTY exits. // Today the relay composes multiple consumers (hook-server cache eviction // and plugin-overlay dir cleanup) into a single callback at the call site @@ -414,6 +420,7 @@ export class PtyHandler { this.enqueuePtyOutput(managed.id, data) }) managed.pty.onExit(({ exitCode }: { exitCode: number }) => { + this.resolveExitWaiters(managed.id) if (managed.disposed) { return } @@ -447,6 +454,61 @@ export class PtyHandler { }) } + private waitForExit(id: string): { promise: Promise; cancel: () => void } { + let settled = false + let resolvePromise = (_exited: boolean): void => {} + const waiter = (): void => { + if (settled) { + return + } + settled = true + clearTimeout(timer) + resolvePromise(true) + } + const waiters = this.exitWaitersByPty.get(id) ?? new Set<() => void>() + waiters.add(waiter) + this.exitWaitersByPty.set(id, waiters) + const timer = setTimeout(() => { + if (settled) { + return + } + settled = true + waiters.delete(waiter) + if (waiters.size === 0) { + this.exitWaitersByPty.delete(id) + } + resolvePromise(false) + }, WINDOWS_IMMEDIATE_EXIT_TIMEOUT_MS) + const promise = new Promise((resolve) => { + resolvePromise = resolve + }) + return { + promise, + cancel: () => { + if (settled) { + return + } + settled = true + clearTimeout(timer) + waiters.delete(waiter) + if (waiters.size === 0) { + this.exitWaitersByPty.delete(id) + } + } + } + } + + private resolveExitWaiters(id: string): void { + const waiters = this.exitWaitersByPty.get(id) + if (!waiters) { + return + } + this.exitWaitersByPty.delete(id) + for (const resolve of waiters) { + resolve() + } + } + private notifyExitListener(managed: ManagedPty): void { if (managed.exitListenerNotified) { return @@ -841,18 +903,34 @@ export class PtyHandler { if (immediate) { this.releaseStartupCommand(managed) this.flushPtyOutput(id) + if (process.platform === 'win32') { + // Why: WindowsTerminal rejects signal arguments and ConPTY teardown is + // asynchronous. Git removal must wait for the native exit event. + const exit = this.waitForExit(id) + if (!managed.windowsImmediateKillIssued) { + managed.windowsImmediateKillIssued = true + try { + managed.pty.kill() + } catch (error) { + managed.windowsImmediateKillIssued = false + exit.cancel() + throw error + } + } + if (!(await exit.promise)) { + throw new Error(`Timed out waiting for Windows PTY teardown: ${id}`) + } + return + } const killedSession = await killPosixPtySession( managed.pty.pid, (managed.pty as unknown as { ptsName?: unknown }).ptsName ) - if (!killedSession && process.platform !== 'win32' && !managed.disposed) { + if (!killedSession && !managed.disposed) { // Why: killing only the forkpty leader can strand HUP-resistant job // groups in a deleted cwd. Fail closed so worktree removal is aborted. throw new Error(`Unable to verify full PTY session teardown: ${id}`) } - if (!killedSession && !managed.disposed) { - managed.pty.kill('SIGKILL') - } if (managed.disposed) { return } @@ -1148,6 +1226,7 @@ export class PtyHandler { this.lastInputAtByPty.clear() this.interactiveOutputCharsByPty.clear() for (const [, managed] of this.ptys) { + this.resolveExitWaiters(managed.id) if (managed.killTimer) { clearTimeout(managed.killTimer) managed.killTimer = undefined From 0e904fa8fb78606e85624ec8a5c672d5a2af669d Mon Sep 17 00:00:00 2001 From: bbingz Date: Mon, 13 Jul 2026 15:36:15 +0800 Subject: [PATCH 16/36] test(relay): make POSIX teardown coverage deterministic --- src/relay/pty-handler.test.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index 00a0983f076..4b58d29d3ab 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -1191,6 +1191,8 @@ describe('PtyHandler', () => { }) it('fails closed when full POSIX PTY session teardown cannot be verified', async () => { + const originalPlatform = process.platform + Object.defineProperty(process, 'platform', { configurable: true, value: 'linux' }) const mockKill = vi.fn() mockKillPosixPtySession.mockResolvedValue(false) mockPtySpawn.mockReturnValue({ @@ -1200,13 +1202,17 @@ describe('PtyHandler', () => { onExit: vi.fn() }) - await dispatcher.callRequest('pty.spawn', {}) + try { + await dispatcher.callRequest('pty.spawn', {}) - await expect( - dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) - ).rejects.toThrow('Unable to verify full PTY session teardown: pty-1') - expect(mockKill).not.toHaveBeenCalled() - expect(handler.activePtyCount).toBe(1) + await expect( + dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + ).rejects.toThrow('Unable to verify full PTY session teardown: pty-1') + expect(mockKill).not.toHaveBeenCalled() + expect(handler.activePtyCount).toBe(1) + } finally { + Object.defineProperty(process, 'platform', { configurable: true, value: originalPlatform }) + } }) it('does not signal a recycled root pid after the POSIX session kill succeeds', async () => { From 1569ef729e268c18acb1728e90dcaf20befeddf8 Mon Sep 17 00:00:00 2001 From: bbingz Date: Mon, 13 Jul 2026 15:38:57 +0800 Subject: [PATCH 17/36] test(relay): await Windows immediate teardown --- src/relay/pty-handler.test.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index 6a376a0b54c..a7522581fb8 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -1080,10 +1080,21 @@ describe('PtyHandler', () => { it('on immediate shutdown', async () => { await withWindowsPlatform(async () => { - const mockKill = mockKillablePty() + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + const mockKill = vi.fn() + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + }) await dispatcher.callRequest('pty.spawn', {}) - await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + const shutdown = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) expectBareKills(mockKill, 1) + onExitCb!({ exitCode: 0 }) + await shutdown }) }) From bf1d215717f97a2dbb4339d510108729821100db Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 13 Jul 2026 00:58:27 -0700 Subject: [PATCH 18/36] fix(terminals): scope and verify destructive teardown --- src/main/index.ts | 1 + src/main/ipc/pty.test.ts | 26 +- src/main/ipc/pty.ts | 9 +- src/main/ipc/worktrees.test.ts | 11 +- src/main/ipc/worktrees.ts | 400 ++++++++++-------- src/main/providers/ssh-pty-provider.test.ts | 31 +- src/main/providers/ssh-pty-provider.ts | 2 +- src/main/runtime/orca-runtime.test.ts | 121 +++++- src/main/runtime/orca-runtime.ts | 426 ++++++++++++-------- src/main/runtime/worktree-teardown.test.ts | 24 ++ src/main/runtime/worktree-teardown.ts | 13 +- src/relay/pty-handler.test.ts | 5 +- src/relay/pty-handler.ts | 5 + src/relay/pty-session-kill.test.ts | 194 ++++++--- src/relay/pty-session-kill.ts | 238 ++++++----- 15 files changed, 958 insertions(+), 548 deletions(-) diff --git a/src/main/index.ts b/src/main/index.ts index b76964ae872..fc30fdef463 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -1777,6 +1777,7 @@ app.whenReady().then(async () => { // provider reference eagerly here would freeze the pre-daemon LocalPtyProvider // and defeat the teardown helper's prefix sweep (design §4.3 wire-up). getLocalProvider: () => getLocalPtyProvider(), + awaitLocalPtyStartup: () => localPtyStartupReady, onPtyStopped: clearProviderPtyState, onTerminalAgentStatus: (event) => { agentHookServer.ingestTerminalStatus(event) diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index 7980c751a8d..4214d51e513 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -857,7 +857,7 @@ describe('registerPtyHandlers', () => { }) expect(result).toMatchObject({ id: expect.any(String) }) - expect(runtime.beginWorktreePtySpawn).toHaveBeenCalledWith('wt-renderer') + expect(runtime.beginWorktreePtySpawn).toHaveBeenCalledWith('wt-renderer', undefined) expect(runtime.registerPty).toHaveBeenCalledWith( expect.any(String), 'wt-renderer', @@ -1854,7 +1854,7 @@ describe('registerPtyHandlers', () => { ) expect(spawnOptions.env.ORCA_AGENT_HOOK_PORT).toBe('5678') expect(spawnOptions.env.ORCA_AGENT_HOOK_TOKEN).toBe('agent-token') - expect(runtime.beginWorktreePtySpawn).toHaveBeenCalledWith('wt-runtime') + expect(runtime.beginWorktreePtySpawn).toHaveBeenCalledWith('wt-runtime', undefined) expect(releaseWorktreeSpawn).toHaveBeenCalledTimes(1) }) @@ -2955,7 +2955,7 @@ describe('registerPtyHandlers', () => { it('finishes headless current-relay SSH shutdown before remote removal without inventory', async () => { const order: string[] = [] const request = vi.fn(async (method: string) => { - if (method === 'pty.shutdown') { + if (method === 'pty.shutdownSession') { order.push('shutdown') return undefined } @@ -2998,12 +2998,14 @@ describe('registerPtyHandlers', () => { deletePtyOwnership('ssh:ssh-1@@headless-current') }) - it('bounds headless legacy-relay SSH verification to one inventory before removal', async () => { + it('fails headless legacy-relay teardown closed before inventory or removal', async () => { const order: string[] = [] const request = vi.fn(async (method: string) => { - if (method === 'pty.shutdown') { - order.push('shutdown') - return undefined + if (method === 'pty.shutdownSession') { + order.push('shutdown-miss') + throw Object.assign(new Error('Method not found'), { + code: JsonRpcErrorCode.MethodNotFound + }) } if (method === 'pty.hasPty') { order.push('hasPty-miss') @@ -3044,16 +3046,14 @@ describe('registerPtyHandlers', () => { runtime.stopTerminalsForWorktree('id:wt-headless-legacy', { worktreeTeardown: true }) - ).resolves.toEqual({ stopped: 2 }) - order.push('git-remove') + ).resolves.toEqual({ stopped: 0, failedPtyIds: ptyIds }) }) - expect(order.at(-1)).toBe('git-remove') - expect(order.filter((step) => step === 'shutdown')).toHaveLength(2) - expect(request.mock.calls.filter(([method]) => method === 'pty.hasPty')).toHaveLength(1) + expect(order.filter((step) => step === 'shutdown-miss')).toHaveLength(2) + expect(request.mock.calls.filter(([method]) => method === 'pty.hasPty')).toHaveLength(0) expect( request.mock.calls.filter(([method]) => method === 'pty.listProcesses') - ).toHaveLength(1) + ).toHaveLength(0) for (const ptyId of ptyIds) { deletePtyOwnership(ptyId) } diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 5f6cad574e0..31e23d29ae9 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -3127,7 +3127,7 @@ export function registerPtyHandlers( !admissionHeldRuntimeSpawnArgs.has(args) && args.worktreeId && runtime?.beginWorktreePtySpawn - ? runtime.beginWorktreePtySpawn(args.worktreeId) + ? runtime.beginWorktreePtySpawn(args.worktreeId, args.connectionId) : null const paneSpawnReservation = materializedPaneKey ? reservePaneSpawn(materializedPaneKey) @@ -3612,14 +3612,17 @@ export function registerPtyHandlers( resetPtyRendererDeliveryDebug() }) - const registerWorktreePtySpawnHandler = ( + const registerWorktreePtySpawnHandler = < + T extends { worktreeId?: string; connectionId?: string | null }, + R + >( channel: string, handler: (event: IpcMainInvokeEvent, args: T) => Promise ): void => { ipcMain.handle(channel, async (event, args: T) => { const releaseWorktreeSpawn = typeof args.worktreeId === 'string' && runtime?.beginWorktreePtySpawn - ? runtime.beginWorktreePtySpawn(args.worktreeId) + ? runtime.beginWorktreePtySpawn(args.worktreeId, args.connectionId) : null try { return await handler(event, args) diff --git a/src/main/ipc/worktrees.test.ts b/src/main/ipc/worktrees.test.ts index a0012e7298a..1c1e3e07cb2 100644 --- a/src/main/ipc/worktrees.test.ts +++ b/src/main/ipc/worktrees.test.ts @@ -6749,6 +6749,7 @@ describe('registerWorktreeHandlers', () => { expect(killAllProcessesForWorktreeMock).toHaveBeenCalledWith(worktreeId, { runtime: runtimeStub, localProvider: ptyProvider, + connectionId: null, onPtyStopped: clearProviderPtyStateMock }) expect(killAllProcessesForWorktreeMock.mock.invocationCallOrder[0]).toBeLessThan( @@ -8665,21 +8666,27 @@ describe('registerWorktreeHandlers', () => { const ptyProvider = {} as never const worktreeId = 'repo-1::/workspace/feature-wt' store.getRepo.mockReturnValue(repo) + store.getRepos.mockReturnValue([repo]) getLocalPtyProviderMock.mockReturnValue(ptyProvider) // Why: a removed/disconnected SSH target has no live provider; forgetLocal // must not reach for one. getSshGitProviderMock.mockReturnValue(undefined) - const result = await handlers['worktrees:forgetLocal'](null, { worktreeId }) + const result = await handlers['worktrees:forgetLocal'](null, { + worktreeId, + hostId: 'ssh:ssh-dead' + }) expect(result).toEqual({}) expect(runtimeStub.runWithWorktreePtyTeardown).toHaveBeenCalledWith( worktreeId, - expect.any(Function) + expect.any(Function), + 'ssh-dead' ) expect(killAllProcessesForWorktreeMock).toHaveBeenCalledWith(worktreeId, { runtime: runtimeStub, localProvider: ptyProvider, + connectionId: 'ssh-dead', onPtyStopped: clearProviderPtyStateMock }) expect(runtimeStub.clearOptimisticReconcileToken).toHaveBeenCalledWith(worktreeId) diff --git a/src/main/ipc/worktrees.ts b/src/main/ipc/worktrees.ts index 30cf404b102..f0d4d745fc9 100644 --- a/src/main/ipc/worktrees.ts +++ b/src/main/ipc/worktrees.ts @@ -1395,14 +1395,18 @@ export function registerWorktreeHandlers( const worktreeRemovalsInFlight = new Map() const runWithWorktreePtyTeardown = ( worktreeId: string, - operation: () => Promise + operation: () => Promise, + ptyScopeId?: string | null ): Promise => typeof runtime.runWithWorktreePtyTeardown === 'function' - ? runtime.runWithWorktreePtyTeardown(worktreeId, operation) + ? runtime.runWithWorktreePtyTeardown(worktreeId, operation, ptyScopeId) : operation() - const beginWorktreePtyTeardown = (worktreeId: string): Promise<() => void> => + const beginWorktreePtyTeardown = ( + worktreeId: string, + ptyScopeId?: string | null + ): Promise<() => void> => typeof runtime.beginWorktreePtyTeardown === 'function' - ? runtime.beginWorktreePtyTeardown(worktreeId) + ? runtime.beginWorktreePtyTeardown(worktreeId, ptyScopeId) : Promise.resolve(() => {}) const runWithWorktreeRemovalDedup = ( worktreeId: string, @@ -1414,18 +1418,24 @@ export function registerWorktreeHandlers( : operation() const runWithSuccessfulCleanupTeardown = ( worktreeId: string, + connectionId: string | null | undefined, operation: () => Promise ): Promise => - runWithWorktreePtyTeardown(worktreeId, async () => { - // Why: orphan and stale-registration cleanup still retires an Orca - // workspace; its terminals must not outlive the last addressable owner. - await killAllProcessesForWorktree(worktreeId, { - runtime, - localProvider: getLocalPtyProvider(), - onPtyStopped: clearProviderPtyState - }) - return operation() - }) + runWithWorktreePtyTeardown( + worktreeId, + async () => { + // Why: orphan and stale-registration cleanup still retires an Orca + // workspace; its terminals must not outlive the last addressable owner. + await killAllProcessesForWorktree(worktreeId, { + runtime, + localProvider: getLocalPtyProvider(), + connectionId, + onPtyStopped: clearProviderPtyState + }) + return operation() + }, + connectionId + ) const registerCrossSurfaceWorktreeRemovalHandler = ( channel: string, @@ -1439,7 +1449,7 @@ export function registerWorktreeHandlers( { force: args.force === true, runHooks: args.skipArchive !== true, - ...(repo ? { hostId: getRepoExecutionHostId(repo) } : {}) + ...(repo ? { hostId: args.hostId ?? getRepoExecutionHostId(repo) } : {}) }, () => handler(event, args) ) @@ -1456,7 +1466,7 @@ export function registerWorktreeHandlers( } const inFlightKey = getWorktreeRemovalInFlightKey( args.worktreeId, - getRepoExecutionHostId(repo) + args.hostId ?? getRepoExecutionHostId(repo) ) const optionsKey = getWorktreeRemovalOptionsKey(args) const inFlightRemoval = worktreeRemovalsInFlight.get(inFlightKey) @@ -1477,19 +1487,24 @@ export function registerWorktreeHandlers( 'Cannot delete the project root workspace. Remove the folder project instead.' ) } - return runWithWorktreePtyTeardown(args.worktreeId, async () => { - // Why: folder workspaces share one filesystem root, so there is no Git - // remove step to close shells; sweep PTYs before dropping metadata. - await killAllProcessesForWorktree(args.worktreeId, { - runtime, - localProvider: getLocalPtyProvider(), - onPtyStopped: clearProviderPtyState - }) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - preservedBranchCleanupByWorktreeId.delete(args.worktreeId) - notifyWorktreesChanged(mainWindow, repoId) - return {} - }) + return runWithWorktreePtyTeardown( + args.worktreeId, + async () => { + // Why: folder workspaces share one filesystem root, so there is no Git + // remove step to close shells; sweep PTYs before dropping metadata. + await killAllProcessesForWorktree(args.worktreeId, { + runtime, + localProvider: getLocalPtyProvider(), + connectionId: repo.connectionId ?? null, + onPtyStopped: clearProviderPtyState + }) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + preservedBranchCleanupByWorktreeId.delete(args.worktreeId) + notifyWorktreesChanged(mainWindow, repoId) + return {} + }, + repo.connectionId ?? null + ) } // Why: the renderer-supplied worktreeId contains a filesystem path. @@ -1549,34 +1564,38 @@ export function registerWorktreeHandlers( if (!args.force) { throw new Error(ORPHANED_WORKTREE_DIRECTORY_MESSAGE) } - return runWithSuccessfulCleanupTeardown(args.worktreeId, async () => { - if (repo.connectionId) { - await fsProvider!.deletePath(worktreePath, true) - await cleanupUnusedWorktreePushTargetRemoteSsh( - provider!, - repo.path, - args.worktreeId, - removedPushTarget, - store - ) - } else { - await closeLocalWatcherForRemoval(worktreePath) - await removeLocalWorktreePath(worktreePath, localWorktreeGitOptions) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - args.worktreeId, - removedPushTarget, - store, - localWorktreeGitOptions - ) - invalidateAuthorizedRootsCache() + return runWithSuccessfulCleanupTeardown( + args.worktreeId, + repo.connectionId ?? null, + async () => { + if (repo.connectionId) { + await fsProvider!.deletePath(worktreePath, true) + await cleanupUnusedWorktreePushTargetRemoteSsh( + provider!, + repo.path, + args.worktreeId, + removedPushTarget, + store + ) + } else { + await closeLocalWatcherForRemoval(worktreePath) + await removeLocalWorktreePath(worktreePath, localWorktreeGitOptions) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + args.worktreeId, + removedPushTarget, + store, + localWorktreeGitOptions + ) + invalidateAuthorizedRootsCache() + } + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + preservedBranchCleanupByWorktreeId.delete(args.worktreeId) + notifyWorktreesChanged(mainWindow, repoId) + return {} } - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - preservedBranchCleanupByWorktreeId.delete(args.worktreeId) - notifyWorktreesChanged(mainWindow, repoId) - return {} - }) + ) } if (!repo.connectionId) { const access = getLocalWorktreePathAccess(localWorktreeGitOptions) @@ -1599,23 +1618,27 @@ export function registerWorktreeHandlers( if (!args.force) { throw new Error(ORPHANED_WORKTREE_DIRECTORY_MESSAGE) } - return runWithSuccessfulCleanupTeardown(args.worktreeId, async () => { - await closeLocalWatcherForRemoval(worktreePath) - await removeLocalWorktreePath(worktreePath, localWorktreeGitOptions) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - args.worktreeId, - removedPushTarget, - store, - localWorktreeGitOptions - ) - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - preservedBranchCleanupByWorktreeId.delete(args.worktreeId) - invalidateAuthorizedRootsCache() - notifyWorktreesChanged(mainWindow, repoId) - return {} - }) + return runWithSuccessfulCleanupTeardown( + args.worktreeId, + repo.connectionId ?? null, + async () => { + await closeLocalWatcherForRemoval(worktreePath) + await removeLocalWorktreePath(worktreePath, localWorktreeGitOptions) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + args.worktreeId, + removedPushTarget, + store, + localWorktreeGitOptions + ) + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + preservedBranchCleanupByWorktreeId.delete(args.worktreeId) + invalidateAuthorizedRootsCache() + notifyWorktreesChanged(mainWindow, repoId) + return {} + } + ) } } if (await isAlreadyRemovedWorktreePath(repo, worktreePath, localWorktreeGitOptions)) { @@ -1627,31 +1650,35 @@ export function registerWorktreeHandlers( // Why: a manually deleted worktree is already gone from Git and disk. // The sidebar delete action has persisted metadata proving this was // an Orca-known row, so no force confirmation is needed. - return runWithSuccessfulCleanupTeardown(args.worktreeId, async () => { - if (repo.connectionId) { - await cleanupUnusedWorktreePushTargetRemoteSsh( - provider!, - repo.path, - args.worktreeId, - removedPushTarget, - store - ) - } else { - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - args.worktreeId, - removedPushTarget, - store, - localWorktreeGitOptions - ) - invalidateAuthorizedRootsCache() + return runWithSuccessfulCleanupTeardown( + args.worktreeId, + repo.connectionId ?? null, + async () => { + if (repo.connectionId) { + await cleanupUnusedWorktreePushTargetRemoteSsh( + provider!, + repo.path, + args.worktreeId, + removedPushTarget, + store + ) + } else { + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + args.worktreeId, + removedPushTarget, + store, + localWorktreeGitOptions + ) + invalidateAuthorizedRootsCache() + } + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + preservedBranchCleanupByWorktreeId.delete(args.worktreeId) + notifyWorktreesChanged(mainWindow, repoId) + return {} } - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - preservedBranchCleanupByWorktreeId.delete(args.worktreeId) - notifyWorktreesChanged(mainWindow, repoId) - return {} - }) + ) } throw new Error(`Refusing to delete unregistered worktree path: ${worktreePath}`) } @@ -1679,33 +1706,38 @@ export function registerWorktreeHandlers( removedMeta && (await isAlreadyRemovedWorktreePath(repo, canonicalWorktreePath, localWorktreeGitOptions)) ) { - return runWithSuccessfulCleanupTeardown(args.worktreeId, async () => { - const removalResult = await removeStaleLocalWorktreeRegistrationAfterFilesystemRemoval({ - canonicalWorktreePath, - repoPath: repo.path, - localWorktreeGitOptions, - registeredWorktree, - deleteBranch - }) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - args.worktreeId, - removedPushTarget, - store, - localWorktreeGitOptions - ) - rememberPreservedBranchCleanupTarget( - args.worktreeId, - removalResult, - registeredWorktree.head, - removedPushTarget - ) - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - invalidateAuthorizedRootsCache() - notifyWorktreesChanged(mainWindow, repoId) - return removalResult ?? {} - }) + return runWithSuccessfulCleanupTeardown( + args.worktreeId, + repo.connectionId ?? null, + async () => { + const removalResult = + await removeStaleLocalWorktreeRegistrationAfterFilesystemRemoval({ + canonicalWorktreePath, + repoPath: repo.path, + localWorktreeGitOptions, + registeredWorktree, + deleteBranch + }) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + args.worktreeId, + removedPushTarget, + store, + localWorktreeGitOptions + ) + rememberPreservedBranchCleanupTarget( + args.worktreeId, + removalResult, + registeredWorktree.head, + removedPushTarget + ) + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + invalidateAuthorizedRootsCache() + notifyWorktreesChanged(mainWindow, repoId) + return removalResult ?? {} + } + ) } // Run archive hook before removal so teardown scripts still see the worktree directory. @@ -1741,37 +1773,42 @@ export function registerWorktreeHandlers( } const remoteRemoveOptions = !deleteBranch ? { deleteBranch } : {} - return runWithWorktreePtyTeardown(args.worktreeId, async () => { - await killAllProcessesForWorktree(args.worktreeId, { - runtime, - localProvider: getLocalPtyProvider(), - onPtyStopped: clearProviderPtyState - }) - const rawRemovalResult = await (Object.keys(remoteRemoveOptions).length > 0 - ? provider!.removeWorktree(canonicalWorktreePath, args.force, remoteRemoveOptions) - : provider!.removeWorktree(canonicalWorktreePath, args.force)) - const removalResult = preserveBranchHeadFallback( - rawRemovalResult, - registeredWorktree.head - ) - await cleanupUnusedWorktreePushTargetRemoteSsh( - provider!, - repo.path, - args.worktreeId, - removedPushTarget, - store - ) - rememberPreservedBranchCleanupTarget( - args.worktreeId, - removalResult, - registeredWorktree.head, - removedPushTarget - ) - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - notifyWorktreesChanged(mainWindow, repoId) - return removalResult ?? {} - }) + return runWithWorktreePtyTeardown( + args.worktreeId, + async () => { + await killAllProcessesForWorktree(args.worktreeId, { + runtime, + localProvider: getLocalPtyProvider(), + connectionId: repo.connectionId ?? null, + onPtyStopped: clearProviderPtyState + }) + const rawRemovalResult = await (Object.keys(remoteRemoveOptions).length > 0 + ? provider!.removeWorktree(canonicalWorktreePath, args.force, remoteRemoveOptions) + : provider!.removeWorktree(canonicalWorktreePath, args.force)) + const removalResult = preserveBranchHeadFallback( + rawRemovalResult, + registeredWorktree.head + ) + await cleanupUnusedWorktreePushTargetRemoteSsh( + provider!, + repo.path, + args.worktreeId, + removedPushTarget, + store + ) + rememberPreservedBranchCleanupTarget( + args.worktreeId, + removalResult, + registeredWorktree.head, + removedPushTarget + ) + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + notifyWorktreesChanged(mainWindow, repoId) + return removalResult ?? {} + }, + repo.connectionId ?? null + ) } const refreshedWorktrees = hasLocalWorktreeGitOptions @@ -1823,7 +1860,10 @@ export function registerWorktreeHandlers( // continue to the fenced teardown before orphan recovery mutates state. } - const releaseWorktreePtyTeardown = await beginWorktreePtyTeardown(args.worktreeId) + const releaseWorktreePtyTeardown = await beginWorktreePtyTeardown( + args.worktreeId, + repo.connectionId ?? null + ) let removalResult: RemoveWorktreeResult | undefined try { await closeLocalWatcherForRemoval(canonicalWorktreePath) @@ -1831,6 +1871,7 @@ export function registerWorktreeHandlers( await killAllProcessesForWorktree(args.worktreeId, { runtime, localProvider: getLocalPtyProvider(), + connectionId: repo.connectionId ?? null, onPtyStopped: clearProviderPtyState }).then((r) => { const total = r.runtimeStopped + r.providerStopped + r.registryStopped @@ -1974,7 +2015,7 @@ export function registerWorktreeHandlers( force: false, runHooks: false, mode: 'forget', - ...(repo ? { hostId: getRepoExecutionHostId(repo) } : {}) + ...(repo ? { hostId: args.hostId ?? getRepoExecutionHostId(repo) } : {}) }, () => handler(event, args) ) @@ -2003,7 +2044,7 @@ export function registerWorktreeHandlers( // mutate metadata. A forget takes no force/skipArchive options. const inFlightKey = getWorktreeRemovalInFlightKey( args.worktreeId, - getRepoExecutionHostId(repo) + args.hostId ?? getRepoExecutionHostId(repo) ) const optionsKey = 'forget-local' const inFlight = worktreeRemovalsInFlight.get(inFlightKey) @@ -2014,27 +2055,32 @@ export function registerWorktreeHandlers( throw new Error(`Worktree deletion already in progress: ${args.worktreeId}`) } - const forget = runWithWorktreePtyTeardown(args.worktreeId, async () => { - if (isFolderRepo(repo) && args.worktreeId === getFolderWorkspaceRootId(repo)) { - throw new Error( - 'Cannot delete the project root workspace. Remove the folder project instead.' - ) - } + const forget = runWithWorktreePtyTeardown( + args.worktreeId, + async () => { + if (isFolderRepo(repo) && args.worktreeId === getFolderWorkspaceRootId(repo)) { + throw new Error( + 'Cannot delete the project root workspace. Remove the folder project instead.' + ) + } - // Why: forgetting metadata releases Orca's last addressable owner. An - // unverified terminal must keep that owner intact so cleanup can retry. - await killAllProcessesForWorktree(args.worktreeId, { - runtime, - localProvider: getLocalPtyProvider(), - onPtyStopped: clearProviderPtyState - }) + // Why: forgetting metadata releases Orca's last addressable owner. An + // unverified terminal must keep that owner intact so cleanup can retry. + await killAllProcessesForWorktree(args.worktreeId, { + runtime, + localProvider: getLocalPtyProvider(), + connectionId: repo.connectionId ?? null, + onPtyStopped: clearProviderPtyState + }) - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - preservedBranchCleanupByWorktreeId.delete(args.worktreeId) - notifyWorktreesChanged(mainWindow, repoId) - return {} - }) + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + preservedBranchCleanupByWorktreeId.delete(args.worktreeId) + notifyWorktreesChanged(mainWindow, repoId) + return {} + }, + repo.connectionId ?? null + ) worktreeRemovalsInFlight.set(inFlightKey, { optionsKey, promise: forget }) try { return await forget diff --git a/src/main/providers/ssh-pty-provider.test.ts b/src/main/providers/ssh-pty-provider.test.ts index d3ae0dd0ca0..032f0bf9de5 100644 --- a/src/main/providers/ssh-pty-provider.test.ts +++ b/src/main/providers/ssh-pty-provider.test.ts @@ -422,10 +422,10 @@ describe('SshPtyProvider', () => { expect(mux.notify).toHaveBeenCalledWith('pty.resize', { id: 'pty-1', cols: 120, rows: 40 }) }) - it('shutdown sends pty.shutdown request', async () => { + it('immediate shutdown requires the full-session relay method', async () => { await provider.shutdown(scopedPty1, { immediate: true }) expect(mux.request).toHaveBeenCalledWith( - 'pty.shutdown', + 'pty.shutdownSession', { id: 'pty-1', immediate: true, @@ -467,7 +467,7 @@ describe('SshPtyProvider', () => { it('shutdown forwards keepHistory: true over the relay', async () => { await provider.shutdown(scopedPty1, { immediate: true, keepHistory: true }) expect(mux.request).toHaveBeenCalledWith( - 'pty.shutdown', + 'pty.shutdownSession', { id: 'pty-1', immediate: true, @@ -571,7 +571,7 @@ describe('SshPtyProvider', () => { if (method === 'pty.spawn') { return { id: 'pty-3' } } - if (method === 'pty.shutdown' || method === 'pty.attach') { + if (method === 'pty.shutdownSession' || method === 'pty.attach') { return undefined } throw new Error(`Unexpected request: ${method}`) @@ -624,7 +624,7 @@ describe('SshPtyProvider', () => { if (method === 'pty.listProcesses') { return inventory } - if (method === 'pty.shutdown') { + if (method === 'pty.shutdownSession') { return undefined } throw new Error(`Unexpected request: ${method}`) @@ -644,6 +644,27 @@ describe('SshPtyProvider', () => { ).toHaveLength(1) }) + it('fails closed when an older relay lacks full-session shutdown', async () => { + mux.request.mockImplementation(async (method: string) => { + if (method === 'pty.shutdownSession') { + throw Object.assign(new Error('Method not found'), { + code: JsonRpcErrorCode.MethodNotFound + }) + } + if (method === 'pty.listProcesses') { + return [] + } + throw new Error(`Unexpected request: ${method}`) + }) + + await expect(provider.shutdown(scopedPty1, { immediate: true })).rejects.toThrow( + 'Method not found' + ) + expect( + mux.request.mock.calls.filter(([method]) => method === 'pty.listProcesses') + ).toHaveLength(0) + }) + it('getDefaultShell returns shell path', async () => { mux.request.mockResolvedValue('/bin/bash') const result = await provider.getDefaultShell() diff --git a/src/main/providers/ssh-pty-provider.ts b/src/main/providers/ssh-pty-provider.ts index 529e4d2596f..85b25adac70 100644 --- a/src/main/providers/ssh-pty-provider.ts +++ b/src/main/providers/ssh-pty-provider.ts @@ -263,7 +263,7 @@ export class SshPtyProvider implements IPtyProvider { async shutdown(id: string, opts: { immediate?: boolean; keepHistory?: boolean }): Promise { await this.mux.request( - 'pty.shutdown', + opts.immediate ? 'pty.shutdownSession' : 'pty.shutdown', { id: this.toRelayPtyId(id), immediate: opts.immediate ?? false, diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index c69c8f32758..e02eb1363d2 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -22215,7 +22215,7 @@ describe('OrcaRuntimeService', () => { }) it('uses authoritative runtime PTYs for teardown while the renderer graph is reloading', async () => { - const stopAndWait = vi.fn(async () => true) + const stopAndWait = vi.fn(async (_ptyId: string) => true) const runtime = new OrcaRuntimeService(store) runtime.setPtyController({ write: () => true, @@ -22404,6 +22404,87 @@ describe('OrcaRuntimeService', () => { expect(stopAndWait).toHaveBeenCalledWith('ssh:ssh-1@@relay-only') }) + it('filters runtime PTYs and durable leases to the deleting SSH host', async () => { + const runtime = new OrcaRuntimeService({ + ...store, + getSshRemotePtyLeases: () => [ + { + targetId: 'ssh-a', + ptyId: 'lease-a', + worktreeId: TEST_WORKTREE_ID, + state: 'detached' as const, + createdAt: 1, + updatedAt: 2 + }, + { + targetId: 'ssh-b', + ptyId: 'lease-b', + worktreeId: TEST_WORKTREE_ID, + state: 'detached' as const, + createdAt: 1, + updatedAt: 2 + } + ] + }) + const stopAndWait = vi.fn(async (_ptyId: string) => true) + runtime.setPtyController({ + write: () => true, + kill: () => false, + stopAndWait, + getForegroundProcess: async () => null + }) + runtime.registerPty('ssh:ssh-a@@runtime-a', TEST_WORKTREE_ID, 'ssh-a') + runtime.registerPty('ssh:ssh-b@@runtime-b', TEST_WORKTREE_ID, 'ssh-b') + + await expect( + runtime.stopTerminalsForWorktree(TEST_WORKTREE_ID, { + worktreeTeardown: true, + connectionId: 'ssh-a' + }) + ).resolves.toEqual({ stopped: 2 }) + expect(stopAndWait.mock.calls.map(([ptyId]) => ptyId).sort()).toEqual([ + 'ssh:ssh-a@@lease-a', + 'ssh:ssh-a@@runtime-a' + ]) + }) + + it('scopes spawn admission independently for identical worktree ids on two SSH hosts', async () => { + const runtime = new OrcaRuntimeService(store) + const releaseHostA = runtime.beginWorktreePtySpawn(TEST_WORKTREE_ID, 'ssh-a') + const hostBOperation = vi.fn(async () => 'host-b-removed') + + await expect( + runtime.runWithWorktreePtyTeardown(TEST_WORKTREE_ID, hostBOperation, 'ssh-b') + ).resolves.toBe('host-b-removed') + expect(hostBOperation).toHaveBeenCalledTimes(1) + + let hostARan = false + const hostATeardown = runtime.runWithWorktreePtyTeardown( + TEST_WORKTREE_ID, + async () => { + hostARan = true + }, + 'ssh-a' + ) + await Promise.resolve() + expect(hostARan).toBe(false) + releaseHostA() + await hostATeardown + expect(hostARan).toBe(true) + }) + + it('does not collide local admission with an SSH connection named local', async () => { + const runtime = new OrcaRuntimeService(store) + const releaseLocal = runtime.beginWorktreePtySpawn(TEST_WORKTREE_ID) + const sshOperation = vi.fn(async () => 'ssh-removed') + + await expect( + runtime.runWithWorktreePtyTeardown(TEST_WORKTREE_ID, sshOperation, 'local') + ).resolves.toBe('ssh-removed') + expect(sshOperation).toHaveBeenCalledTimes(1) + releaseLocal() + }) + it('bounds concurrent verified worktree stops', async () => { const runtime = new OrcaRuntimeService(store) const releases: (() => void)[] = [] @@ -29706,11 +29787,39 @@ describe('OrcaRuntimeService', () => { expect(preDaemonProvider.shutdown).not.toHaveBeenCalled() }) - it('drains headless and late SSH spawns before remote worktree removal', async () => { + it('waits for delayed daemon startup before CLI removal captures the provider', async () => { + const startup = deferred() + const fallbackProvider = createProviderStub(async () => []) + const daemonPtyId = `${TEST_WORKTREE_ID}@@daemon-startup` + const daemonProvider = createProviderStub(async () => [ + { id: daemonPtyId, cwd: TEST_WORKTREE_PATH, title: 'shell' } + ]) + let currentProvider = fallbackProvider + const runtime = new OrcaRuntimeService(store, undefined, { + getLocalProvider: () => currentProvider as never, + awaitLocalPtyStartup: () => startup.promise + }) + + const removal = runtime.removeManagedWorktree(TEST_WORKTREE_ID) + await Promise.resolve() + expect(removeWorktree).not.toHaveBeenCalled() + expect(fallbackProvider.listProcesses).not.toHaveBeenCalled() + + currentProvider = daemonProvider + startup.resolve() + await removal + + expect(daemonProvider.shutdown).toHaveBeenCalledWith(daemonPtyId, { immediate: true }) + expect(fallbackProvider.shutdown).not.toHaveBeenCalled() + expect(removeWorktree).toHaveBeenCalledTimes(1) + }) + + it('drains headless and late runtime-backed SSH spawns before remote removal', async () => { const remoteRepo = { ...store.getRepo(TEST_REPO_ID)!, path: '/remote/repo', - connectionId: 'ssh-1' + connectionId: 'ssh-1', + executionHostId: 'runtime:runtime-a' as const } const remoteWorktree = { path: '/remote/feature-wt', @@ -29729,7 +29838,7 @@ describe('OrcaRuntimeService', () => { getAllWorktreeMeta: () => ({ [remoteWorktreeId]: makeWorktreeMeta() }), getWorktreeMeta: (id: string) => (id === remoteWorktreeId ? makeWorktreeMeta() : undefined), removeWorktreeMeta: () => { - expect(() => runtime.beginWorktreePtySpawn(remoteWorktreeId)).toThrow( + expect(() => runtime.beginWorktreePtySpawn(remoteWorktreeId, 'ssh-1')).toThrow( 'Worktree teardown is in progress' ) rejectedDuringRemoteMetadataCleanup = true @@ -29775,7 +29884,7 @@ describe('OrcaRuntimeService', () => { getForegroundProcess: async () => null }) runtime.registerPty('ssh:ssh-1@@existing', remoteWorktreeId, 'ssh-1') - const releaseLateSpawn = runtime.beginWorktreePtySpawn(remoteWorktreeId) + const releaseLateSpawn = runtime.beginWorktreePtySpawn(remoteWorktreeId, 'ssh-1') try { const removal = runtime.removeManagedWorktree(remoteWorktreeId) @@ -29790,7 +29899,7 @@ describe('OrcaRuntimeService', () => { releaseFirstStop() await removal - const releasePostRemovalSpawn = runtime.beginWorktreePtySpawn(remoteWorktreeId) + const releasePostRemovalSpawn = runtime.beginWorktreePtySpawn(remoteWorktreeId, 'ssh-1') releasePostRemovalSpawn() expect(callOrder).toEqual([ diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 5d8a0236d51..868ab7846a7 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -2488,6 +2488,7 @@ export class OrcaRuntimeService { private readonly worktreePtyAdmission = new WorktreePtyAdmission() private preservedBranchCleanupByWorktreeId = new Map() private readonly getLocalProviderFn: (() => IPtyProvider) | null + private readonly awaitLocalPtyStartup: (() => Promise) | null private readonly onPtyStopped: ((ptyId: string) => void) | null private readonly onTerminalAgentStatus: ((event: RuntimeTerminalAgentStatusEvent) => void) | null private readonly onTerminalSideEffects: ((batch: TerminalSideEffectBatch) => void) | null @@ -2513,6 +2514,7 @@ export class OrcaRuntimeService { stats?: StatsCollector, deps?: { getLocalProvider?: () => IPtyProvider + awaitLocalPtyStartup?: () => Promise onPtyStopped?: (ptyId: string) => void onTerminalAgentStatus?: (event: RuntimeTerminalAgentStatusEvent) => void onTerminalSideEffects?: (batch: TerminalSideEffectBatch) => void @@ -2549,6 +2551,7 @@ export class OrcaRuntimeService { // lazily via thunk so teardown always sees the currently-installed // provider (design §4.3 wire-up). this.getLocalProviderFn = deps?.getLocalProvider ?? null + this.awaitLocalPtyStartup = deps?.awaitLocalPtyStartup ?? null this.onPtyStopped = deps?.onPtyStopped ?? null this.onTerminalAgentStatus = deps?.onTerminalAgentStatus ?? null this.buildAgentHookPtyEnv = deps?.buildAgentHookPtyEnv ?? null @@ -16935,23 +16938,28 @@ export class OrcaRuntimeService { 'Cannot delete the project root workspace. Remove the folder project instead.' ) } - return this.runWithWorktreePtyTeardown(removalTarget.id, async () => { - const localProvider = this.getLocalProvider() - if (localProvider) { - // Why: folder workspace deletion has no Git removal phase where PTYs - // would otherwise be swept; tear them down before hiding the workspace. - await killAllProcessesForWorktree(removalTarget.id, { - runtime: this, - localProvider, - onPtyStopped: this.onPtyStopped ?? undefined - }) - } - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) - this.invalidateResolvedWorktreeCache() - this.notifyWorktreesChanged(repo.id) - return {} - }) + return this.runWithWorktreePtyTeardown( + removalTarget.id, + async () => { + const localProvider = this.getLocalProvider() + if (localProvider) { + // Why: folder workspace deletion has no Git removal phase where PTYs + // would otherwise be swept; tear them down before hiding the workspace. + await killAllProcessesForWorktree(removalTarget.id, { + runtime: this, + localProvider, + connectionId: repo.connectionId ?? null, + onPtyStopped: this.onPtyStopped ?? undefined + }) + } + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) + this.invalidateResolvedWorktreeCache() + this.notifyWorktreesChanged(repo.id) + return {} + }, + repo.connectionId ?? null + ) } const provider = repo.connectionId ? requireSshGitProvider(repo.connectionId) : null const fsProvider = repo.connectionId ? getSshFilesystemProvider(repo.connectionId) : null @@ -17008,34 +17016,38 @@ export class OrcaRuntimeService { if (!force) { throw new Error(ORPHANED_WORKTREE_DIRECTORY_MESSAGE) } - return this.runWithSuccessfulCleanupTeardown(removalTarget.id, async () => { - if (repo.connectionId) { - await fsProvider!.deletePath(removalTarget.path, true) - await cleanupUnusedWorktreePushTargetRemoteSsh( - provider!, - repo.path, - removalTarget.id, - removedPushTarget, - store - ) - } else { - await removeLocalWorktreePath(removalTarget.path, localWorktreeGitOptions) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - removalTarget.id, - removedPushTarget, - store, - localWorktreeGitOptions - ) + return this.runWithSuccessfulCleanupTeardown( + removalTarget.id, + repo.connectionId ?? null, + async () => { + if (repo.connectionId) { + await fsProvider!.deletePath(removalTarget.path, true) + await cleanupUnusedWorktreePushTargetRemoteSsh( + provider!, + repo.path, + removalTarget.id, + removedPushTarget, + store + ) + } else { + await removeLocalWorktreePath(removalTarget.path, localWorktreeGitOptions) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + removalTarget.id, + removedPushTarget, + store, + localWorktreeGitOptions + ) + } + this.clearOptimisticReconcileToken(removalTarget.id) + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) + this.invalidateResolvedWorktreeCache() + invalidateAuthorizedRootsCache() + this.notifyWorktreesChanged(repo.id) + return {} } - this.clearOptimisticReconcileToken(removalTarget.id) - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) - this.invalidateResolvedWorktreeCache() - invalidateAuthorizedRootsCache() - this.notifyWorktreesChanged(repo.id) - return {} - }) + ) } if (!repo.connectionId) { const access = getLocalWorktreePathAccess(localWorktreeGitOptions) @@ -17058,26 +17070,30 @@ export class OrcaRuntimeService { if (!force) { throw new Error(ORPHANED_WORKTREE_DIRECTORY_MESSAGE) } - return this.runWithSuccessfulCleanupTeardown(removalTarget.id, async () => { - await closeLocalWatcherForWorktreePath(removalTarget.path).catch((err) => { - console.warn(`[filesystem-watcher] failed to close ${removalTarget.path}:`, err) - }) - await removeLocalWorktreePath(removalTarget.path, localWorktreeGitOptions) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - removalTarget.id, - removedPushTarget, - store, - localWorktreeGitOptions - ) - this.clearOptimisticReconcileToken(removalTarget.id) - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) - this.invalidateResolvedWorktreeCache() - invalidateAuthorizedRootsCache() - this.notifyWorktreesChanged(repo.id) - return {} - }) + return this.runWithSuccessfulCleanupTeardown( + removalTarget.id, + repo.connectionId ?? null, + async () => { + await closeLocalWatcherForWorktreePath(removalTarget.path).catch((err) => { + console.warn(`[filesystem-watcher] failed to close ${removalTarget.path}:`, err) + }) + await removeLocalWorktreePath(removalTarget.path, localWorktreeGitOptions) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + removalTarget.id, + removedPushTarget, + store, + localWorktreeGitOptions + ) + this.clearOptimisticReconcileToken(removalTarget.id) + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) + this.invalidateResolvedWorktreeCache() + invalidateAuthorizedRootsCache() + this.notifyWorktreesChanged(repo.id) + return {} + } + ) } } if (await isRuntimeWorktreePathMissing(repo, removalTarget.path, localWorktreeGitOptions)) { @@ -17089,30 +17105,34 @@ export class OrcaRuntimeService { // Why: a manually deleted worktree is already gone from Git and disk. // Finish runtime metadata cleanup without requiring force or touching // any unregistered path that still exists. - return this.runWithSuccessfulCleanupTeardown(removalTarget.id, async () => { - await (repo.connectionId - ? cleanupUnusedWorktreePushTargetRemoteSsh( - provider!, - repo.path, - removalTarget.id, - removedPushTarget, - store - ) - : cleanupUnusedWorktreePushTargetRemote( - repo.path, - removalTarget.id, - removedPushTarget, - store, - localWorktreeGitOptions - )) - this.clearOptimisticReconcileToken(removalTarget.id) - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) - this.invalidateResolvedWorktreeCache() - invalidateAuthorizedRootsCache() - this.notifyWorktreesChanged(repo.id) - return {} - }) + return this.runWithSuccessfulCleanupTeardown( + removalTarget.id, + repo.connectionId ?? null, + async () => { + await (repo.connectionId + ? cleanupUnusedWorktreePushTargetRemoteSsh( + provider!, + repo.path, + removalTarget.id, + removedPushTarget, + store + ) + : cleanupUnusedWorktreePushTargetRemote( + repo.path, + removalTarget.id, + removedPushTarget, + store, + localWorktreeGitOptions + )) + this.clearOptimisticReconcileToken(removalTarget.id) + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) + this.invalidateResolvedWorktreeCache() + invalidateAuthorizedRootsCache() + this.notifyWorktreesChanged(repo.id) + return {} + } + ) } throw new Error(`Refusing to delete unregistered worktree path: ${removalTarget.path}`) } @@ -17137,73 +17157,82 @@ export class OrcaRuntimeService { removedMeta && (await isRuntimeWorktreePathMissing(repo, canonicalWorktreePath, localWorktreeGitOptions)) ) { - return this.runWithSuccessfulCleanupTeardown(removalTarget.id, async () => { - const removalResult = await removeStaleLocalWorktreeRegistrationAfterFilesystemRemoval({ - canonicalWorktreePath, - repoPath: repo.path, - localWorktreeGitOptions, - registeredWorktree, - deleteBranch - }) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - removalTarget.id, - removedPushTarget, - store, - localWorktreeGitOptions - ) - this.rememberPreservedBranchCleanupTarget( - removalTarget.id, - removalResult, - registeredWorktree.head, - removedPushTarget - ) - this.clearOptimisticReconcileToken(removalTarget.id) - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.invalidateResolvedWorktreeCache() - invalidateAuthorizedRootsCache() - this.notifyWorktreesChanged(repo.id) - return removalResult ?? {} - }) + return this.runWithSuccessfulCleanupTeardown( + removalTarget.id, + repo.connectionId ?? null, + async () => { + const removalResult = await removeStaleLocalWorktreeRegistrationAfterFilesystemRemoval({ + canonicalWorktreePath, + repoPath: repo.path, + localWorktreeGitOptions, + registeredWorktree, + deleteBranch + }) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + removalTarget.id, + removedPushTarget, + store, + localWorktreeGitOptions + ) + this.rememberPreservedBranchCleanupTarget( + removalTarget.id, + removalResult, + registeredWorktree.head, + removedPushTarget + ) + this.clearOptimisticReconcileToken(removalTarget.id) + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.invalidateResolvedWorktreeCache() + invalidateAuthorizedRootsCache() + this.notifyWorktreesChanged(repo.id) + return removalResult ?? {} + } + ) } if (repo.connectionId) { const remoteRemoveOptions = !deleteBranch ? { deleteBranch } : {} - return this.runWithWorktreePtyTeardown(removalTarget.id, async () => { - const localProvider = this.getLocalProvider() - if (localProvider) { - await killAllProcessesForWorktree(removalTarget.id, { - runtime: this, - localProvider, - onPtyStopped: this.onPtyStopped ?? undefined - }) - } - const rawRemovalResult = await (Object.keys(remoteRemoveOptions).length > 0 - ? provider!.removeWorktree(canonicalWorktreePath, force, remoteRemoveOptions) - : provider!.removeWorktree(canonicalWorktreePath, force)) - const removalResult = this.preserveBranchHeadFallback( - rawRemovalResult, - registeredWorktree.head - ) - await cleanupUnusedWorktreePushTargetRemoteSsh( - provider!, - repo.path, - removalTarget.id, - removedPushTarget, - store - ) - this.rememberPreservedBranchCleanupTarget( - removalTarget.id, - removalResult, - registeredWorktree.head, - removedPushTarget - ) - this.clearOptimisticReconcileToken(removalTarget.id) - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.invalidateResolvedWorktreeCache() - invalidateAuthorizedRootsCache() - this.notifyWorktreesChanged(repo.id) - return removalResult ?? {} - }) + return this.runWithWorktreePtyTeardown( + removalTarget.id, + async () => { + const localProvider = this.getLocalProvider() + if (localProvider) { + await killAllProcessesForWorktree(removalTarget.id, { + runtime: this, + localProvider, + connectionId: repo.connectionId ?? null, + onPtyStopped: this.onPtyStopped ?? undefined + }) + } + const rawRemovalResult = await (Object.keys(remoteRemoveOptions).length > 0 + ? provider!.removeWorktree(canonicalWorktreePath, force, remoteRemoveOptions) + : provider!.removeWorktree(canonicalWorktreePath, force)) + const removalResult = this.preserveBranchHeadFallback( + rawRemovalResult, + registeredWorktree.head + ) + await cleanupUnusedWorktreePushTargetRemoteSsh( + provider!, + repo.path, + removalTarget.id, + removedPushTarget, + store + ) + this.rememberPreservedBranchCleanupTarget( + removalTarget.id, + removalResult, + registeredWorktree.head, + removedPushTarget + ) + this.clearOptimisticReconcileToken(removalTarget.id) + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.invalidateResolvedWorktreeCache() + invalidateAuthorizedRootsCache() + this.notifyWorktreesChanged(repo.id) + return removalResult ?? {} + }, + repo.connectionId ?? null + ) } const hooks = getEffectiveHooks(repo) @@ -17261,7 +17290,10 @@ export class OrcaRuntimeService { // continue to the fenced teardown before orphan recovery mutates state. } - const releaseWorktreePtyTeardown = await this.beginWorktreePtyTeardown(removalTarget.id) + const releaseWorktreePtyTeardown = await this.beginWorktreePtyTeardown( + removalTarget.id, + repo.connectionId ?? null + ) let removalResult: RemoveWorktreeResult | undefined try { const localProvider = this.getLocalProvider() @@ -17274,6 +17306,7 @@ export class OrcaRuntimeService { await killAllProcessesForWorktree(removalTarget.id, { runtime: this, localProvider, + connectionId: repo.connectionId ?? null, onPtyStopped: this.onPtyStopped ?? undefined }).then((r) => { const total = r.runtimeStopped + r.providerStopped + r.registryStopped @@ -17525,7 +17558,7 @@ export class OrcaRuntimeService { throw new Error('runtime_unavailable') } const workspace = await this.resolveTerminalWorkspaceLaunchScope(worktreeSelector) - return await this.runWithWorktreePtySpawn(workspace.id, async () => { + return await this.runWithWorktreePtySpawn(workspace.id, workspace.connectionId, async () => { const launchOpts = await this.resolveAgentTerminalCreateOptions(workspace, opts) const cwd = this.resolveWorkspaceTerminalStartupCwd(workspace, launchOpts.cwd) ?? workspace.path @@ -18644,7 +18677,7 @@ export class OrcaRuntimeService { } const direction = opts.direction ?? 'horizontal' const workspace = await this.resolveTerminalWorkspaceLaunchScope(`id:${pty.worktreeId}`) - return await this.runWithWorktreePtySpawn(workspace.id, async () => { + return await this.runWithWorktreePtySpawn(workspace.id, workspace.connectionId, async () => { const leafId = randomUUID() const preAllocatedHandle = this.createPreAllocatedTerminalHandle() const paneKey = makePaneKey(parentTabId, leafId) @@ -18799,15 +18832,18 @@ export class OrcaRuntimeService { }) } - beginWorktreePtySpawn(worktreeId: string): () => void { - return this.worktreePtyAdmission.beginSpawn(worktreeId) + beginWorktreePtySpawn(worktreeId: string, connectionId?: string | null): () => void { + return this.worktreePtyAdmission.beginSpawn( + this.getWorktreePtyScopeKey(worktreeId, connectionId) + ) } private async runWithWorktreePtySpawn( worktreeId: string, + connectionId: string | null | undefined, operation: () => Promise ): Promise { - const release = this.beginWorktreePtySpawn(worktreeId) + const release = this.beginWorktreePtySpawn(worktreeId, connectionId) try { return await operation() } finally { @@ -18824,33 +18860,62 @@ export class OrcaRuntimeService { return await spawn.call(controller, opts) } - async beginWorktreePtyTeardown(worktreeId: string): Promise<() => void> { - return this.worktreePtyAdmission.closeForTeardown(worktreeId) + async beginWorktreePtyTeardown( + worktreeId: string, + connectionId?: string | null + ): Promise<() => void> { + // Why: an existing daemon can own PTYs before the renderer/runtime graph + // hydrates. Do not snapshot the fallback provider while startup is swapping it. + if (connectionId == null) { + await this.awaitLocalPtyStartup?.() + } + return this.worktreePtyAdmission.closeForTeardown( + this.getWorktreePtyScopeKey(worktreeId, connectionId) + ) } - async runWithWorktreePtyTeardown(worktreeId: string, operation: () => Promise): Promise { + async runWithWorktreePtyTeardown( + worktreeId: string, + operation: () => Promise, + connectionId?: string | null + ): Promise { // Why: closing admission before snapshots makes deletion own every spawn // already in flight and keeps new desktop/mobile/SSH spawns out until Git settles. - return this.worktreePtyAdmission.runTeardown(worktreeId, operation) + const release = await this.beginWorktreePtyTeardown(worktreeId, connectionId) + try { + return await operation() + } finally { + release() + } + } + + private getWorktreePtyScopeKey(worktreeId: string, connectionId?: string | null): string { + return JSON.stringify([connectionId ?? null, worktreeId]) } private runWithSuccessfulCleanupTeardown( worktreeId: string, + connectionId: string | null | undefined, operation: () => Promise ): Promise { - return this.runWithWorktreePtyTeardown(worktreeId, async () => { - // Why: orphan and stale-registration cleanup still retires an Orca - // workspace; its terminals must not outlive the last addressable owner. - const localProvider = this.getLocalProvider() - if (localProvider) { - await killAllProcessesForWorktree(worktreeId, { - runtime: this, - localProvider, - onPtyStopped: this.onPtyStopped ?? undefined - }) - } - return operation() - }) + return this.runWithWorktreePtyTeardown( + worktreeId, + async () => { + // Why: orphan and stale-registration cleanup still retires an Orca + // workspace; its terminals must not outlive the last addressable owner. + const localProvider = this.getLocalProvider() + if (localProvider) { + await killAllProcessesForWorktree(worktreeId, { + runtime: this, + localProvider, + connectionId, + onPtyStopped: this.onPtyStopped ?? undefined + }) + } + return operation() + }, + connectionId + ) } async runWithWorktreeRemovalDedup( @@ -18886,7 +18951,7 @@ export class OrcaRuntimeService { async stopTerminalsForWorktree( worktreeSelector: string, - opts: { worktreeTeardown?: boolean } = {} + opts: { worktreeTeardown?: boolean; connectionId?: string | null } = {} ): Promise<{ stopped: number; failedPtyIds?: string[] }> { // Why: generic terminal.stop must not act on a stale renderer graph, while // deletion runs under admission and owns main's authoritative PTY records. @@ -18910,14 +18975,19 @@ export class OrcaRuntimeService { } } for (const pty of this.ptysById.values()) { - if (pty.worktreeId === worktreeId && pty.connected) { + if ( + pty.worktreeId === worktreeId && + pty.connected && + (opts.connectionId === undefined || pty.connectionId === opts.connectionId) + ) { ptyIds.add(pty.ptyId) } } - if (opts.worktreeTeardown) { + if (opts.worktreeTeardown && opts.connectionId !== null) { for (const lease of this.store?.getSshRemotePtyLeases?.() ?? []) { if ( lease.worktreeId === worktreeId && + (opts.connectionId === undefined || lease.targetId === opts.connectionId) && lease.state !== 'terminated' && lease.state !== 'expired' ) { diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index 0f2fccfd3f6..ad6f60d36b3 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -193,6 +193,30 @@ describe('killAllProcessesForWorktree', () => { expect(localProvider.listProcesses).toHaveBeenCalledTimes(1) }) + it('does not sweep a colliding local-provider worktree during SSH teardown', async () => { + const stopTerminalsForWorktree = vi.fn().mockResolvedValue({ stopped: 1 }) + const runtime = { + stopTerminalsForWorktree + } as unknown as Parameters[1]['runtime'] + const localProvider = createProviderStub(async () => [ + { id: 'w1@@local-witness', cwd: '/tmp/w1', title: 'shell' } + ]) + + await expect( + killAllProcessesForWorktree('w1', { + runtime, + localProvider, + connectionId: 'ssh-1' + }) + ).resolves.toEqual({ runtimeStopped: 1, providerStopped: 0, registryStopped: 0 }) + expect(stopTerminalsForWorktree).toHaveBeenCalledWith('w1', { + worktreeTeardown: true, + connectionId: 'ssh-1' + }) + expect(localProvider.listProcesses).not.toHaveBeenCalled() + expect(localProvider.shutdown).not.toHaveBeenCalled() + }) + it('fails closed when a local runtime PTY remains after fallback shutdown rejects', async () => { const runtime = { stopTerminalsForWorktree: vi.fn().mockResolvedValue({ diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index 57ba4676455..13ca102c378 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -7,6 +7,7 @@ import { mapPtyStopsWithConcurrency } from './pty-stop-concurrency' export type WorktreeTeardownDeps = { runtime?: OrcaRuntimeService localProvider: IPtyProvider + connectionId?: string | null onPtyStopped?: (ptyId: string) => void } @@ -51,7 +52,8 @@ export async function killAllProcessesForWorktree( if (deps.runtime) { const r = await deps.runtime.stopTerminalsForWorktree(worktreeId, { - worktreeTeardown: true + worktreeTeardown: true, + ...(deps.connectionId !== undefined ? { connectionId: deps.connectionId } : {}) }) result.runtimeStopped = r.stopped failedPtyIds = 'failedPtyIds' in r ? (r.failedPtyIds ?? []) : [] @@ -59,12 +61,9 @@ export async function killAllProcessesForWorktree( const failedRemotePtyIds = failedPtyIds.filter((ptyId) => parseAppSshPtyId(ptyId) !== null) const failedLocalPtyIds = failedPtyIds.filter((ptyId) => parseAppSshPtyId(ptyId) === null) - const fallbackResult = await sweepLocalProvider( - worktreeId, - deps.localProvider, - failedLocalPtyIds, - deps.onPtyStopped - ) + const fallbackResult = deps.connectionId + ? { providerStopped: 0, registryStopped: 0 } + : await sweepLocalProvider(worktreeId, deps.localProvider, failedLocalPtyIds, deps.onPtyStopped) result.providerStopped = fallbackResult.providerStopped result.registryStopped = fallbackResult.registryStopped diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index 00a0983f076..25cefc085f4 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -127,6 +127,7 @@ describe('PtyHandler', () => { expect(methods).toContain('pty.spawn') expect(methods).toContain('pty.attach') expect(methods).toContain('pty.shutdown') + expect(methods).toContain('pty.shutdownSession') expect(methods).toContain('pty.sendSignal') expect(methods).toContain('pty.getCwd') expect(methods).toContain('pty.getInitialCwd') @@ -1085,7 +1086,7 @@ describe('PtyHandler', () => { expect(handler.activePtyCount).toBe(0) }) - it('kills the full POSIX PTY session when shutdown is immediate', async () => { + it('kills the full POSIX PTY session through the capability method', async () => { const mockKill = vi.fn() mockPtySpawn.mockReturnValue({ ...mockPtyInstance, @@ -1095,7 +1096,7 @@ describe('PtyHandler', () => { }) await dispatcher.callRequest('pty.spawn', {}) - await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + await dispatcher.callRequest('pty.shutdownSession', { id: 'pty-1' }) expect(mockKillPosixPtySession).toHaveBeenCalledWith(process.pid, undefined) expect(mockKill).not.toHaveBeenCalledWith('SIGKILL') }) diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index e1b3fcae19b..a373479f93b 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -533,6 +533,11 @@ export class PtyHandler { this.dispatcher.onRequest('pty.spawn', (p, context) => this.spawn(p, context)) this.dispatcher.onRequest('pty.attach', (p) => this.attach(p)) this.dispatcher.onRequest('pty.shutdown', (p) => this.shutdown(p)) + // Why: method presence proves the relay can kill and verify the whole PTY + // process tree. Older relays only removed the forkpty leader/map entry. + this.dispatcher.onRequest('pty.shutdownSession', (p) => + this.shutdown({ ...p, immediate: true }) + ) this.dispatcher.onRequest('pty.sendSignal', (p) => this.sendSignal(p)) this.dispatcher.onRequest('pty.getCwd', (p) => this.getCwd(p)) this.dispatcher.onRequest('pty.getInitialCwd', (p) => this.getInitialCwd(p)) diff --git a/src/relay/pty-session-kill.test.ts b/src/relay/pty-session-kill.test.ts index 56dc133cd25..710af546b64 100644 --- a/src/relay/pty-session-kill.test.ts +++ b/src/relay/pty-session-kill.test.ts @@ -5,44 +5,85 @@ import { PTY_SESSION_VERIFY_TIMEOUT_MS } from './pty-session-kill' +function emptySelection(): Error & { code: number; stdout: string } { + return Object.assign(new Error('no processes'), { code: 1, stdout: '' }) +} + describe('killPosixPtySession', () => { - it('targets the full POSIX session with a bounded argument list', async () => { - const run = vi.fn().mockResolvedValueOnce(undefined).mockResolvedValueOnce({ stdout: 'Z\n' }) + it('freezes and kills a Linux descendant that escaped into a new session', async () => { + const run = vi.fn(async (file: string, args: string[]) => { + if (file === '/bin/ps' && args.includes('sid=')) { + return { stdout: '4242 pts/7\n' } + } + if (file === '/usr/bin/pgrep') { + const parents = new Set(args[1]?.split(',')) + if (parents.has('4242')) { + return { stdout: '4243\n4244\n' } + } + if (parents.has('4244')) { + return { stdout: '4245\n' } + } + throw emptySelection() + } + if (file === '/bin/ps' && args.some((arg) => arg.includes('4245'))) { + return { stdout: '4242 Z\n4243 Z\n4244 Z\n4245 Z\n' } + } + throw new Error(`Unexpected command: ${file} ${args.join(' ')}`) + }) + const killProcess = vi.fn() - await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run)).resolves.toBe(true) + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + true + ) - expect(run).toHaveBeenCalledWith('pkill', ['-KILL', '-s', '4242', '.*'], { - timeout: PTY_SESSION_COMMAND_TIMEOUT_MS - }) - expect(run).toHaveBeenCalledWith('/bin/ps', ['-s', '4242', '-o', 'stat='], { - timeout: PTY_SESSION_VERIFY_TIMEOUT_MS - }) + expect(killProcess.mock.calls).toEqual([ + [4242, 'SIGSTOP'], + [4243, 'SIGSTOP'], + [4244, 'SIGSTOP'], + [4245, 'SIGSTOP'], + [4245, 'SIGKILL'], + [4244, 'SIGKILL'], + [4243, 'SIGKILL'], + [4242, 'SIGKILL'] + ]) + expect(run).not.toHaveBeenCalledWith( + '/bin/ps', + expect.arrayContaining(['-e']), + expect.anything() + ) + expect(run.mock.calls.filter(([file]) => file === '/usr/bin/pgrep')).toHaveLength(3) }) - it('targets Darwin forkpty jobs through their controlling TTY', async () => { - const run = vi - .fn() - .mockResolvedValueOnce({ stdout: '4242 1\n4243 4242\n4244 4243\n' }) - .mockResolvedValueOnce({ stdout: '4242 Z\n4243 Z\n4244 Z\n' }) + it('validates Darwin root ownership before targeted descendant teardown', async () => { + const run = vi.fn(async (file: string, args: string[]) => { + if (file === '/bin/ps' && args.includes('sess=')) { + return { stdout: '4242 ttys042\n' } + } + if (file === '/usr/bin/pgrep') { + if (args[1] === '4242') { + return { stdout: '4243\n' } + } + throw emptySelection() + } + if (file === '/bin/ps' && args.includes('pid=')) { + throw emptySelection() + } + throw new Error(`Unexpected command: ${file} ${args.join(' ')}`) + }) const killProcess = vi.fn() await expect( killPosixPtySession(4242, '/dev/ttys042', 'darwin', run, killProcess) ).resolves.toBe(true) - - expect(run).toHaveBeenCalledWith('/bin/ps', ['-t', 'ttys042', '-o', 'pid=', '-o', 'ppid='], { + expect(run).toHaveBeenCalledWith('/bin/ps', ['-p', '4242', '-o', 'sess=', '-o', 'tty='], { timeout: PTY_SESSION_COMMAND_TIMEOUT_MS }) expect(killProcess.mock.calls).toEqual([ - [4244, 'SIGKILL'], + [4242, 'SIGSTOP'], + [4243, 'SIGSTOP'], [4243, 'SIGKILL'], [4242, 'SIGKILL'] ]) - expect(run).toHaveBeenLastCalledWith( - '/bin/ps', - ['-t', 'ttys042', '-o', 'pid=', '-o', 'stat='], - { timeout: PTY_SESSION_VERIFY_TIMEOUT_MS } - ) }) it('leaves Windows ConPTY teardown to node-pty', async () => { @@ -52,60 +93,101 @@ describe('killPosixPtySession', () => { expect(run).not.toHaveBeenCalled() }) - it('falls back cleanly when pkill is unavailable', async () => { - const run = vi.fn().mockRejectedValue(new Error('pkill failed')) - - await expect(killPosixPtySession(4242, '/dev/ttys042', 'darwin', run)).resolves.toBe(false) - }) - - it('rejects an absent or ambiguous Darwin PTY name', async () => { - const run = vi.fn() + it('does not signal a root whose session identity no longer matches', async () => { + const run = vi.fn().mockResolvedValue({ stdout: '9999 pts/7\n' }) + const killProcess = vi.fn() - await expect(killPosixPtySession(4242, '/dev/ttys1,ttys2', 'darwin', run)).resolves.toBe(false) - expect(run).not.toHaveBeenCalled() + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + false + ) + expect(killProcess).not.toHaveBeenCalled() }) - it('does not signal a Darwin TTY that no longer owns the root pid', async () => { - const run = vi.fn().mockResolvedValue({ stdout: '9999 1\n' }) + it('does not signal a reused Linux session leader on a different TTY', async () => { + const run = vi.fn().mockResolvedValue({ stdout: '4242 pts/99\n' }) const killProcess = vi.fn() - await expect( - killPosixPtySession(4242, '/dev/ttys042', 'darwin', run, killProcess) - ).resolves.toBe(false) + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + false + ) expect(killProcess).not.toHaveBeenCalled() }) - it('fails closed when Linux session members remain runnable after SIGKILL', async () => { - const run = vi.fn().mockResolvedValueOnce(undefined).mockResolvedValueOnce({ stdout: 'D\n' }) + it('resumes the frozen root when a discovered child exits before it can be frozen', async () => { + const run = vi + .fn() + .mockResolvedValueOnce({ stdout: '4242 pts/7\n' }) + .mockResolvedValueOnce({ stdout: '4243\n' }) + const killProcess = vi.fn((pid: number, signal: NodeJS.Signals) => { + if (pid === 4243 && signal === 'SIGSTOP') { + throw Object.assign(new Error('gone'), { code: 'ESRCH' }) + } + }) - await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run)).resolves.toBe(false) + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + false + ) + expect(killProcess.mock.calls).toEqual([ + [4242, 'SIGSTOP'], + [4243, 'SIGSTOP'], + [4242, 'SIGCONT'] + ]) }) - it('fails closed when Darwin signals do not remove the captured processes', async () => { + it('resumes frozen processes when descendant discovery fails', async () => { const run = vi .fn() - .mockResolvedValueOnce({ stdout: '4242 1\n4243 4242\n' }) - .mockResolvedValueOnce({ stdout: '4242 S\n4243 S\n' }) + .mockResolvedValueOnce({ stdout: '4242 pts/7\n' }) + .mockRejectedValueOnce(new Error('pgrep unavailable')) const killProcess = vi.fn() - await expect( - killPosixPtySession(4242, '/dev/ttys042', 'darwin', run, killProcess) - ).resolves.toBe(false) - expect(killProcess).toHaveBeenCalledTimes(2) + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + false + ) + expect(killProcess.mock.calls).toEqual([ + [4242, 'SIGSTOP'], + [4242, 'SIGCONT'] + ]) }) - it('accepts an exact empty-process ps exit after signalling', async () => { - const emptySelection = Object.assign(new Error('no processes'), { code: 1, stdout: '' }) - const run = vi.fn().mockResolvedValueOnce(undefined).mockRejectedValueOnce(emptySelection) + it('fails closed when a captured process remains runnable after SIGKILL', async () => { + const run = vi.fn(async (file: string) => { + if (file === '/usr/bin/pgrep') { + throw emptySelection() + } + if (run.mock.calls.length === 1) { + return { stdout: '4242 pts/7\n' } + } + return { stdout: '4242 D\n' } + }) + const killProcess = vi.fn() - await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run)).resolves.toBe(true) + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + false + ) }) - it('verifies an already-empty Linux session after pkill finds no match', async () => { - const emptySelection = Object.assign(new Error('no processes'), { code: 1, stdout: '' }) - const run = vi.fn().mockRejectedValueOnce(emptySelection).mockRejectedValueOnce(emptySelection) + it('uses bounded targeted command timeouts for ownership, children, and verification', async () => { + const run = vi.fn(async (file: string, _args?: string[], _options?: { timeout: number }) => { + if (file === '/usr/bin/pgrep') { + throw emptySelection() + } + if (run.mock.calls.length === 1) { + return { stdout: '4242 pts/7\n' } + } + throw emptySelection() + }) - await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run)).resolves.toBe(true) - expect(run).toHaveBeenCalledTimes(2) + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, vi.fn())).resolves.toBe(true) + expect(run).toHaveBeenNthCalledWith(1, '/bin/ps', ['-p', '4242', '-o', 'sid=', '-o', 'tty='], { + timeout: PTY_SESSION_COMMAND_TIMEOUT_MS + }) + expect(run).toHaveBeenNthCalledWith(2, '/usr/bin/pgrep', ['-P', '4242'], { + timeout: expect.any(Number) + }) + expect(run.mock.calls[1]?.[2]?.timeout).toBeLessThanOrEqual(PTY_SESSION_COMMAND_TIMEOUT_MS) + expect(run).toHaveBeenNthCalledWith(3, '/bin/ps', ['-p', '4242', '-o', 'pid=', '-o', 'stat='], { + timeout: PTY_SESSION_VERIFY_TIMEOUT_MS + }) }) }) diff --git a/src/relay/pty-session-kill.ts b/src/relay/pty-session-kill.ts index 2aa551b0b19..1744513d7c7 100644 --- a/src/relay/pty-session-kill.ts +++ b/src/relay/pty-session-kill.ts @@ -7,6 +7,9 @@ type KillProcess = (pid: number, signal: NodeJS.Signals) => void const execFile = promisify(execFileCallback) as ExecFile export const PTY_SESSION_COMMAND_TIMEOUT_MS = 2500 export const PTY_SESSION_VERIFY_TIMEOUT_MS = 500 +export const MAX_PTY_PROCESS_TREE_SIZE = 1024 +const VERIFY_PID_BATCH_SIZE = 64 +const PARENT_PID_BATCH_SIZE = 64 export async function killPosixPtySession( pid: number, @@ -15,141 +18,180 @@ export async function killPosixPtySession( run: ExecFile = execFile, killProcess: KillProcess = process.kill ): Promise { - if (platform === 'win32' || !Number.isSafeInteger(pid) || pid <= 0) { + if ((platform !== 'linux' && platform !== 'darwin') || !Number.isSafeInteger(pid) || pid <= 0) { return false } + const stopped = new Set() try { - if (platform === 'linux') { - // Why: forkpty makes the Linux shell a session leader. SID targeting - // includes background job groups without a broad process inventory. - try { - await run('pkill', ['-KILL', '-s', String(pid), '.*'], { - timeout: PTY_SESSION_COMMAND_TIMEOUT_MS - }) - } catch (error) { - if (!isEmptyProcessSelection(error)) { - throw error - } - } - return await verifyLinuxSessionStopped(pid, run) + if (!(await rootStillOwnsPty(pid, ptsName, platform, run))) { + return false } - if (platform === 'darwin') { - return await killDarwinPtyProcesses(pid, ptsName, run, killProcess) + if (!stopProcess(pid, stopped, killProcess)) { + return false } - return false + const processTree = await freezeProcessTree(pid, stopped, run, killProcess) + if (!processTree) { + resumeProcesses(stopped, killProcess) + return false + } + for (const candidate of processTree.toReversed()) { + if (!signalProcess(candidate, 'SIGKILL', killProcess)) { + resumeProcesses(stopped, killProcess) + return false + } + stopped.delete(candidate) + } + return await verifyProcessesStopped(processTree, platform, run) } catch { - // Missing platform tools and already-empty sessions fall back to node-pty. + resumeProcesses(stopped, killProcess) return false } } -async function killDarwinPtyProcesses( - rootPid: number, +async function rootStillOwnsPty( + pid: number, ptsName: unknown, - run: ExecFile, - killProcess: KillProcess + platform: NodeJS.Platform, + run: ExecFile ): Promise { - if (typeof ptsName !== 'string') { - return false - } - const tty = ptsName.startsWith('/dev/') ? ptsName.slice('/dev/'.length) : ptsName - if (!/^[A-Za-z0-9._-]+$/.test(tty)) { - return false - } - // Why: Darwin pkill has no SID selector and its TTY filter does not match - // forkpty children. A targeted ps query avoids a system-wide inventory. - const result = (await run('/bin/ps', ['-t', tty, '-o', 'pid=', '-o', 'ppid='], { + const sessionColumn = platform === 'darwin' ? 'sess=' : 'sid=' + const result = (await run('/bin/ps', ['-p', String(pid), '-o', sessionColumn, '-o', 'tty='], { timeout: PTY_SESSION_COMMAND_TIMEOUT_MS - })) as { - stdout?: string | Buffer - } - const rows = String(result.stdout ?? '') + })) as { stdout?: string | Buffer } + const [sidText, tty = ''] = String(result.stdout ?? '') .trim() - .split('\n') - .map((line) => line.trim().split(/\s+/).map(Number)) - .filter( - (row): row is [number, number] => - row.length === 2 && - row.every((candidate) => Number.isSafeInteger(candidate) && candidate >= 0) && - row[0] > 0 - ) - if (!rows.some(([candidate]) => candidate === rootPid)) { + .split(/\s+/) + if (Number(sidText) !== pid) { return false } - const parentByPid = new Map(rows) - const depth = (pid: number): number => { - let current = pid - let result = 0 - const visited = new Set() - while (current !== rootPid && !visited.has(current)) { - visited.add(current) - const parent = parentByPid.get(current) - if (!parent || !parentByPid.has(parent)) { - break - } - current = parent - result += 1 - } - return result + if (typeof ptsName !== 'string') { + // Why: some node-pty builds omit the private ptsName field. The + // session-leader check plus a real controlling TTY still proves ownership. + return tty.length > 0 && tty !== '??' && tty !== '?' } - // Why: descendants must be signalled before their parents so the snapshot - // cannot be invalidated by reparenting while teardown is in progress. - const pids = rows - .map(([candidate]) => candidate) - .sort((a, b) => { - if (a === rootPid) { - return 1 - } - if (b === rootPid) { - return -1 + const expectedTty = ptsName.startsWith('/dev/') ? ptsName.slice('/dev/'.length) : ptsName + return /^[A-Za-z0-9._/-]+$/.test(expectedTty) && tty === expectedTty +} + +async function freezeProcessTree( + rootPid: number, + stopped: Set, + run: ExecFile, + killProcess: KillProcess +): Promise { + const processTree = [rootPid] + const seen = new Set(processTree) + let frontier = [rootPid] + const discoveryDeadline = Date.now() + PTY_SESSION_COMMAND_TIMEOUT_MS + while (frontier.length > 0) { + const nextFrontier: number[] = [] + for (let index = 0; index < frontier.length; index += PARENT_PID_BATCH_SIZE) { + const remainingMs = discoveryDeadline - Date.now() + if (remainingMs <= 0) { + return null } - return depth(b) - depth(a) - }) - for (const candidate of pids) { - try { - killProcess(candidate, 'SIGKILL') - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ESRCH') { - return false + const parents = frontier.slice(index, index + PARENT_PID_BATCH_SIZE) + for (const child of await listChildren(parents, run, remainingMs)) { + if (seen.has(child)) { + continue + } + if (seen.size >= MAX_PTY_PROCESS_TREE_SIZE) { + return null + } + seen.add(child) + if (!stopProcess(child, stopped, killProcess)) { + return null + } + processTree.push(child) + nextFrontier.push(child) } } + frontier = nextFrontier } - return await verifyDarwinTtyStopped(tty, run) + return processTree } -async function verifyLinuxSessionStopped(pid: number, run: ExecFile): Promise { +async function listChildren( + parentPids: readonly number[], + run: ExecFile, + timeout: number +): Promise { try { - const result = (await run('/bin/ps', ['-s', String(pid), '-o', 'stat='], { - timeout: PTY_SESSION_VERIFY_TIMEOUT_MS + const result = (await run('/usr/bin/pgrep', ['-P', parentPids.join(',')], { + timeout })) as { stdout?: string | Buffer } - return hasOnlyExitedProcesses(result.stdout) + return String(result.stdout ?? '') + .trim() + .split(/\s+/) + .map(Number) + .filter((candidate) => Number.isSafeInteger(candidate) && candidate > 0) } catch (error) { - return isEmptyProcessSelection(error) + if (isEmptyProcessSelection(error)) { + return [] + } + throw error } } -async function verifyDarwinTtyStopped(tty: string, run: ExecFile): Promise { +function stopProcess(pid: number, stopped: Set, killProcess: KillProcess): boolean { try { - const result = (await run('/bin/ps', ['-t', tty, '-o', 'pid=', '-o', 'stat='], { - timeout: PTY_SESSION_VERIFY_TIMEOUT_MS - })) as { stdout?: string | Buffer } - return hasOnlyExitedProcesses(result.stdout, true) + killProcess(pid, 'SIGSTOP') + stopped.add(pid) + return true + } catch { + return false + } +} + +function signalProcess(pid: number, signal: NodeJS.Signals, killProcess: KillProcess): boolean { + try { + killProcess(pid, signal) + return true } catch (error) { - return isEmptyProcessSelection(error) + return (error as NodeJS.ErrnoException).code === 'ESRCH' + } +} + +function resumeProcesses(stopped: Set, killProcess: KillProcess): void { + for (const pid of stopped) { + signalProcess(pid, 'SIGCONT', killProcess) + } + stopped.clear() +} + +async function verifyProcessesStopped( + pids: number[], + platform: NodeJS.Platform, + run: ExecFile +): Promise { + const batchSize = platform === 'darwin' ? 1 : VERIFY_PID_BATCH_SIZE + for (let index = 0; index < pids.length; index += batchSize) { + const batch = pids.slice(index, index + batchSize) + try { + const pidSelector = platform === 'darwin' ? String(batch[0]) : batch.join(',') + const result = (await run('/bin/ps', ['-p', pidSelector, '-o', 'pid=', '-o', 'stat='], { + timeout: PTY_SESSION_VERIFY_TIMEOUT_MS + })) as { stdout?: string | Buffer } + if (!hasOnlyExitedProcesses(result.stdout)) { + return false + } + } catch (error) { + if (!isEmptyProcessSelection(error)) { + return false + } + } } + return true } -function hasOnlyExitedProcesses(stdout: string | Buffer | undefined, pidColumn = false): boolean { - const lines = String(stdout ?? '') +function hasOnlyExitedProcesses(stdout: string | Buffer | undefined): boolean { + return String(stdout ?? '') .trim() .split('\n') .map((line) => line.trim()) .filter(Boolean) - return lines.every((line) => { - const stat = pidColumn ? line.split(/\s+/).at(-1) : line - return stat?.startsWith('Z') === true - }) + .every((line) => line.split(/\s+/).at(-1)?.startsWith('Z') === true) } function isEmptyProcessSelection(error: unknown): boolean { @@ -157,7 +199,7 @@ function isEmptyProcessSelection(error: unknown): boolean { return false } const processError = error as { code?: unknown; stdout?: unknown } - // Why: BSD/procps ps exits 1 for an empty selector. Only that exact empty - // result proves absence; ENOENT, timeout, and malformed output fail closed. + // Why: BSD/procps ps/pgrep exit 1 for an empty selector. Only that exact + // empty result proves absence; ENOENT, timeout, and malformed output fail closed. return processError.code === 1 && String(processError.stdout ?? '').trim() === '' } From a28027208a6fbb921723f2acd9bed5d80162ad81 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 13 Jul 2026 01:25:35 -0700 Subject: [PATCH 19/36] fix(relay): support Node 18 session teardown --- src/relay/pty-session-kill.test.ts | 19 ++++++++++++++++--- src/relay/pty-session-kill.ts | 4 +++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/relay/pty-session-kill.test.ts b/src/relay/pty-session-kill.test.ts index 710af546b64..6b70e42e6fc 100644 --- a/src/relay/pty-session-kill.test.ts +++ b/src/relay/pty-session-kill.test.ts @@ -11,6 +11,11 @@ function emptySelection(): Error & { code: number; stdout: string } { describe('killPosixPtySession', () => { it('freezes and kills a Linux descendant that escaped into a new session', async () => { + const toReversedDescriptor = Object.getOwnPropertyDescriptor(Array.prototype, 'toReversed') + Object.defineProperty(Array.prototype, 'toReversed', { + configurable: true, + value: undefined + }) const run = vi.fn(async (file: string, args: string[]) => { if (file === '/bin/ps' && args.includes('sid=')) { return { stdout: '4242 pts/7\n' } @@ -32,9 +37,17 @@ describe('killPosixPtySession', () => { }) const killProcess = vi.fn() - await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( - true - ) + try { + await expect( + killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess) + ).resolves.toBe(true) + } finally { + if (toReversedDescriptor) { + Object.defineProperty(Array.prototype, 'toReversed', toReversedDescriptor) + } else { + Reflect.deleteProperty(Array.prototype, 'toReversed') + } + } expect(killProcess.mock.calls).toEqual([ [4242, 'SIGSTOP'], diff --git a/src/relay/pty-session-kill.ts b/src/relay/pty-session-kill.ts index 1744513d7c7..056c35e1201 100644 --- a/src/relay/pty-session-kill.ts +++ b/src/relay/pty-session-kill.ts @@ -35,7 +35,9 @@ export async function killPosixPtySession( resumeProcesses(stopped, killProcess) return false } - for (const candidate of processTree.toReversed()) { + // Why: remote relays support Node 18, which predates Array#toReversed. + for (let index = processTree.length - 1; index >= 0; index -= 1) { + const candidate = processTree[index]! if (!signalProcess(candidate, 'SIGKILL', killProcess)) { resumeProcesses(stopped, killProcess) return false From 5c81d3c2b6b07d6fad63dadec3ff7506a736beeb Mon Sep 17 00:00:00 2001 From: bbingz Date: Mon, 13 Jul 2026 16:28:22 +0800 Subject: [PATCH 20/36] fix(daemon): tolerate legacy teardown exit race --- src/main/daemon/session.test.ts | 13 +++++++++++++ src/main/daemon/session.ts | 14 +++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/main/daemon/session.test.ts b/src/main/daemon/session.test.ts index ade8b18fe15..530c7f6f511 100644 --- a/src/main/daemon/session.test.ts +++ b/src/main/daemon/session.test.ts @@ -365,6 +365,19 @@ describe('Session', () => { expect(subprocess.written).toEqual([]) }) + it('still disposes when the legacy force-kill fallback races with process exit', async () => { + createSession() + subprocess.forceKill = vi.fn(() => { + throw new Error('process already exited') + }) + subprocess.dispose = vi.fn() + + await expect(session.forceKillAndDisposeSubprocessAndWait()).resolves.toBeUndefined() + + expect(subprocess.dispose).toHaveBeenCalledOnce() + expect(session.state).toBe('exited') + }) + it('transitions to timed_out after 15 seconds', () => { createSession({ shellReadySupported: true }) session.write('waiting input') diff --git a/src/main/daemon/session.ts b/src/main/daemon/session.ts index 3aa64b097ec..160b344c0bf 100644 --- a/src/main/daemon/session.ts +++ b/src/main/daemon/session.ts @@ -497,9 +497,17 @@ export class Session { } async forceKillAndDisposeSubprocessAndWait(): Promise { - const stopped = this.subprocess.forceKillAndWait - ? await this.subprocess.forceKillAndWait() - : (this.subprocess.forceKill(), true) + let stopped: boolean + if (this.subprocess.forceKillAndWait) { + stopped = await this.subprocess.forceKillAndWait() + } else { + try { + this.subprocess.forceKill() + } catch { + // Why: the child can exit between liveness inspection and the legacy kill call. + } + stopped = true + } if (!stopped) { // Why: disposing here would erase the only retryable owner while its // process may still hold the worktree. The daemon request must fail. From 7666c0eb42bbea17d8b4edc8d4c2280ccccb432b Mon Sep 17 00:00:00 2001 From: bbingz Date: Mon, 13 Jul 2026 16:41:06 +0800 Subject: [PATCH 21/36] fix(windows): allow ConPTY teardown verification to settle --- src/main/daemon/pty-subprocess.test.ts | 22 +++++++++++++++++++ src/main/daemon/pty-subprocess.ts | 5 ++++- src/main/providers/local-pty-provider.test.ts | 2 +- src/main/providers/local-pty-provider.ts | 4 +++- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/main/daemon/pty-subprocess.test.ts b/src/main/daemon/pty-subprocess.test.ts index cf2695348b1..659a84c52aa 100644 --- a/src/main/daemon/pty-subprocess.test.ts +++ b/src/main/daemon/pty-subprocess.test.ts @@ -2658,6 +2658,28 @@ describe('createPtySubprocess', () => { } }) + it('waits through node-pty Windows process enumeration before verifying exit', async () => { + vi.useFakeTimers() + const proc = mockPtyProcess(123456) + spawnMock.mockReturnValue(proc) + const origPlatform = Object.getOwnPropertyDescriptor(process, 'platform') + Object.defineProperty(process, 'platform', { value: 'win32' }) + try { + const handle = createPtySubprocess({ sessionId: 'test', cols: 80, rows: 24 }) + handle.kill() + + const stopped = handle.forceKillAndWait!() + await vi.advanceTimersByTimeAsync(5100) + proc._simulateExit(0) + + await expect(stopped).resolves.toBe(true) + expect(proc.kill).toHaveBeenCalledOnce() + } finally { + vi.useRealTimers() + restorePlatform(origPlatform) + } + }) + it('dispose() on Windows skips destroy after forceKill falls back to node-pty kill()', () => { const proc = mockPtyProcess(123456) as ReturnType & { destroy: ReturnType diff --git a/src/main/daemon/pty-subprocess.ts b/src/main/daemon/pty-subprocess.ts index 0d4fa1841d7..0294b0a43e5 100644 --- a/src/main/daemon/pty-subprocess.ts +++ b/src/main/daemon/pty-subprocess.ts @@ -82,6 +82,9 @@ const PTY_SPAWN_HEALTH_TIMEOUT_MS = 4_000 // fallback (losing daemon persistence) until a manual restart. const PTY_SPAWN_HEALTH_RETRY_ATTEMPTS = 2 const PENDING_PRE_LISTENER_DATA_MAX_CHARS = 512 * 1024 +// Why: node-pty can spend up to 5s enumerating a Windows console tree before +// its fallback runs; leave bounded headroom for the native exit event to land. +const WINDOWS_PTY_EXIT_TIMEOUT_MS = 8_000 export type PtySubprocessOptions = { sessionId: string @@ -1236,7 +1239,7 @@ export function createPtySubprocess(opts: PtySubprocessOptions): SubprocessHandl const timeout = setTimeout(() => { exitWaiters.delete(onExit) resolve(false) - }, 3000) + }, WINDOWS_PTY_EXIT_TIMEOUT_MS) exitWaiters.add(onExit) }) }, diff --git a/src/main/providers/local-pty-provider.test.ts b/src/main/providers/local-pty-provider.test.ts index b07ccea9222..4b64c02845c 100644 --- a/src/main/providers/local-pty-provider.test.ts +++ b/src/main/providers/local-pty-provider.test.ts @@ -1006,7 +1006,7 @@ describe('LocalPtyProvider', () => { const firstStopRejected = expect(firstStop).rejects.toThrow( `Unable to verify full PTY session teardown: ${id}` ) - await vi.advanceTimersByTimeAsync(3000) + await vi.advanceTimersByTimeAsync(8000) await firstStopRejected expect((await provider.listProcesses()).some((session) => session.id === id)).toBe(true) diff --git a/src/main/providers/local-pty-provider.ts b/src/main/providers/local-pty-provider.ts index b8f3adfe377..dd7c126e4fe 100644 --- a/src/main/providers/local-pty-provider.ts +++ b/src/main/providers/local-pty-provider.ts @@ -80,7 +80,9 @@ const ptyDisposables = new Map void }[]>() const ptyCleanupCallbacks = new Map void>() const windowsImmediateKills = new WeakSet() -const WINDOWS_PTY_EXIT_TIMEOUT_MS = 3000 +// Why: node-pty can spend up to 5s enumerating a Windows console tree before +// its fallback runs; leave bounded headroom for the native exit event to land. +const WINDOWS_PTY_EXIT_TIMEOUT_MS = 8_000 let loadGeneration = 0 const ptyLoadGeneration = new Map() From 960ba8c1c7307a86d497c7885ff58fa6bad10add Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 13 Jul 2026 01:42:22 -0700 Subject: [PATCH 22/36] fix(terminals): harden verified process teardown --- ...n-foreground-confirmation-protocol.test.ts | 5 +- src/main/daemon/types.ts | 4 +- src/main/ipc/pty.ts | 3 + src/main/runtime/pty-stop-concurrency.test.ts | 48 +++++ src/main/runtime/pty-stop-concurrency.ts | 16 +- src/main/runtime/worktree-teardown.test.ts | 26 +++ src/main/runtime/worktree-teardown.ts | 14 +- src/relay/pty-session-kill.test.ts | 186 +++++++++++++++--- src/relay/pty-session-kill.ts | 102 +++++++--- 9 files changed, 350 insertions(+), 54 deletions(-) create mode 100644 src/main/runtime/pty-stop-concurrency.test.ts diff --git a/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts b/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts index 4fbb3047d6a..6bdbe5049d6 100644 --- a/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts +++ b/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts @@ -3,7 +3,10 @@ import { PREVIOUS_DAEMON_PROTOCOL_VERSIONS, PROTOCOL_VERSION } from './types' describe('foreground-confirmation daemon protocol', () => { it('rejects daemons from before the fresh-confirmation RPC', () => { - expect(PROTOCOL_VERSION).toBeGreaterThan(19) + expect(PROTOCOL_VERSION).toBe(22) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(19) + // Why: v21 lacks full-session teardown, so the client must replace it + // instead of accepting a legacy daemon that can strand descendants. + expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(21) }) }) diff --git a/src/main/daemon/types.ts b/src/main/daemon/types.ts index 550b211aa74..1643a3e2a80 100644 --- a/src/main/daemon/types.ts +++ b/src/main/daemon/types.ts @@ -16,9 +16,9 @@ import type { TuiAgent } from '../../shared/types' // when daemon-baked behavior cannot be delivered by on-disk wrapper refresh. // Why: bump when adding daemon wire behavior so same-version old daemons do // not silently accept the handshake and then reject new RPCs. -export const PROTOCOL_VERSION = 21 +export const PROTOCOL_VERSION = 22 export const PREVIOUS_DAEMON_PROTOCOL_VERSIONS = [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 ] as const // ─── Session State Machine ────────────────────────────────────────── diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 31e23d29ae9..75ace5e6e71 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -1172,6 +1172,9 @@ export function clearProviderPtyState(id: string): void { } }) rendererSerializerByPtyId.delete(id) + // Why: direct provider teardown can bypass the ordinary exit listener. The + // provider router must not retain a dead SSH/local owner after verified stop. + ptyOwnership.delete(id) // Why: the hook server's per-paneKey caches (lastPrompt / lastTool) would // otherwise accumulate entries for dead panes over the process lifetime. // Use the spawn-time paneKey mapping since the server has no other way to diff --git a/src/main/runtime/pty-stop-concurrency.test.ts b/src/main/runtime/pty-stop-concurrency.test.ts new file mode 100644 index 00000000000..4496da6348c --- /dev/null +++ b/src/main/runtime/pty-stop-concurrency.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it, vi } from 'vitest' +import { mapPtyStopsWithConcurrency, WORKTREE_PTY_STOP_CONCURRENCY } from './pty-stop-concurrency' + +describe('mapPtyStopsWithConcurrency', () => { + it('attempts and awaits every PTY before surfacing the first rejection', async () => { + const ids = Array.from( + { length: WORKTREE_PTY_STOP_CONCURRENCY + 3 }, + (_, index) => `pty-${index}` + ) + const pending = new Map void; reject: (error: Error) => void }>() + const stopPty = vi.fn( + (ptyId: string) => + new Promise((resolve, reject) => { + pending.set(ptyId, { resolve: () => resolve(ptyId), reject }) + }) + ) + + const stopping = mapPtyStopsWithConcurrency(ids, stopPty) + let settled = false + const observed = stopping.then( + () => { + settled = true + return null + }, + (error: unknown) => { + settled = true + return error + } + ) + await vi.waitFor(() => expect(stopPty).toHaveBeenCalledTimes(WORKTREE_PTY_STOP_CONCURRENCY)) + pending.get('pty-0')!.reject(new Error('first failed')) + await vi.waitFor(() => expect(stopPty).toHaveBeenCalledTimes(WORKTREE_PTY_STOP_CONCURRENCY + 1)) + await Promise.resolve() + expect(settled).toBe(false) + + for (const id of ids.slice(1)) { + pending.get(id)?.resolve() + await Promise.resolve() + } + await vi.waitFor(() => expect(stopPty).toHaveBeenCalledTimes(ids.length)) + for (const id of ids) { + pending.get(id)?.resolve() + } + + await expect(observed).resolves.toEqual(expect.objectContaining({ message: 'first failed' })) + expect(stopPty.mock.calls.map(([ptyId]) => ptyId)).toEqual(ids) + }) +}) diff --git a/src/main/runtime/pty-stop-concurrency.ts b/src/main/runtime/pty-stop-concurrency.ts index 7784766a19a..75952f30f40 100644 --- a/src/main/runtime/pty-stop-concurrency.ts +++ b/src/main/runtime/pty-stop-concurrency.ts @@ -6,11 +6,22 @@ export async function mapPtyStopsWithConcurrency( ): Promise { const results = Array(ptyIds.length) let nextIndex = 0 + let didFail = false + let firstError: unknown const stopNext = async (): Promise => { while (nextIndex < ptyIds.length) { const index = nextIndex nextIndex += 1 - results[index] = await stopPty(ptyIds[index]) + try { + results[index] = await stopPty(ptyIds[index]) + } catch (error) { + // Why: deletion must attempt and await every snapshotted PTY even when + // one provider rejects, otherwise active workers can outlive Git removal. + if (!didFail) { + firstError = error + } + didFail = true + } } } // Why: remote tools may each consume the full timeout. A small worker pool @@ -20,5 +31,8 @@ export async function mapPtyStopsWithConcurrency( () => stopNext() ) await Promise.all(workers) + if (didFail) { + throw firstError + } return results } diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index ad6f60d36b3..8748860e3f0 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -278,6 +278,32 @@ describe('killAllProcessesForWorktree', () => { expect(localProvider.shutdown).toHaveBeenCalledTimes(1) }) + it('retires runtime and provider state after authoritative fallback absence', async () => { + const onPtyExit = vi.fn() + const runtime = { + stopTerminalsForWorktree: vi.fn().mockResolvedValue({ + stopped: 0, + failedPtyIds: ['w1@@daemon'] + }), + onPtyExit + } as unknown as Parameters[1]['runtime'] + const localProvider = createProviderStub( + vi + .fn() + .mockResolvedValueOnce([{ id: 'w1@@daemon', cwd: '/tmp/w1', title: 'shell' }]) + .mockResolvedValueOnce([]) + ) + vi.mocked(localProvider.shutdown).mockRejectedValue(new Error('not found')) + listRegisteredPtysMock.mockReturnValue([]) + const onPtyStopped = vi.fn() + + await expect( + killAllProcessesForWorktree('w1', { runtime, localProvider, onPtyStopped }) + ).resolves.toEqual({ runtimeStopped: 0, providerStopped: 0, registryStopped: 0 }) + expect(onPtyStopped).toHaveBeenCalledWith('w1@@daemon') + expect(onPtyExit).toHaveBeenCalledWith('w1@@daemon', -1) + }) + it('bounds a 50-session headless fallback sweep to seven shutdown waves', async () => { const sessions = Array.from({ length: 50 }, (_, index) => ({ id: `w1@@${index}`, diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index 13ca102c378..5761fab9691 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -63,7 +63,12 @@ export async function killAllProcessesForWorktree( const failedLocalPtyIds = failedPtyIds.filter((ptyId) => parseAppSshPtyId(ptyId) === null) const fallbackResult = deps.connectionId ? { providerStopped: 0, registryStopped: 0 } - : await sweepLocalProvider(worktreeId, deps.localProvider, failedLocalPtyIds, deps.onPtyStopped) + : await sweepLocalProvider(worktreeId, deps.localProvider, failedLocalPtyIds, (ptyId) => { + clearStoppedPtyState(ptyId, deps.onPtyStopped) + // Why: provider shutdown and already-absent fallbacks do not always emit + // an exit event, so retire the runtime record before Git removes its cwd. + deps.runtime?.onPtyExit?.(ptyId, -1) + }) result.providerStopped = fallbackResult.providerStopped result.registryStopped = fallbackResult.registryStopped @@ -80,7 +85,7 @@ async function sweepLocalProvider( worktreeId: string, provider: IPtyProvider, failedLocalPtyIds: readonly string[], - onPtyStopped?: (ptyId: string) => void + retirePty?: (ptyId: string) => void ): Promise<{ providerStopped: number; registryStopped: number }> { const prefix = `${worktreeId}@@` const sessions = await provider.listProcesses() @@ -107,7 +112,7 @@ async function sweepLocalProvider( try { await provider.shutdown(ptyId, { immediate: true }) stopped.add(ptyId) - clearStoppedPtyState(ptyId, onPtyStopped) + retirePty?.(ptyId) } catch (error) { failedShutdowns.set(ptyId, error) } @@ -122,6 +127,9 @@ async function sweepLocalProvider( if (unverified.length > 0) { throw new Error(`Failed to stop local worktree terminals: ${unverified.join(', ')}`) } + for (const ptyId of failedShutdowns.keys()) { + retirePty?.(ptyId) + } } let providerStopped = 0 diff --git a/src/relay/pty-session-kill.test.ts b/src/relay/pty-session-kill.test.ts index 6b70e42e6fc..9b308f27f42 100644 --- a/src/relay/pty-session-kill.test.ts +++ b/src/relay/pty-session-kill.test.ts @@ -17,10 +17,10 @@ describe('killPosixPtySession', () => { value: undefined }) const run = vi.fn(async (file: string, args: string[]) => { - if (file === '/bin/ps' && args.includes('sid=')) { + if (file === 'ps' && args.includes('sid=')) { return { stdout: '4242 pts/7\n' } } - if (file === '/usr/bin/pgrep') { + if (file === 'pgrep') { const parents = new Set(args[1]?.split(',')) if (parents.has('4242')) { return { stdout: '4243\n4244\n' } @@ -30,7 +30,16 @@ describe('killPosixPtySession', () => { } throw emptySelection() } - if (file === '/bin/ps' && args.some((arg) => arg.includes('4245'))) { + if (file === 'ps' && args.includes('ppid=')) { + const parents = new Map([ + [4243, 4242], + [4244, 4242], + [4245, 4244] + ]) + const selected = args[1]!.split(',').map(Number) + return { stdout: selected.map((child) => `${child} ${parents.get(child)}`).join('\n') } + } + if (file === 'ps' && args.includes('stat=')) { return { stdout: '4242 Z\n4243 Z\n4244 Z\n4245 Z\n' } } throw new Error(`Unexpected command: ${file} ${args.join(' ')}`) @@ -59,26 +68,25 @@ describe('killPosixPtySession', () => { [4243, 'SIGKILL'], [4242, 'SIGKILL'] ]) - expect(run).not.toHaveBeenCalledWith( - '/bin/ps', - expect.arrayContaining(['-e']), - expect.anything() - ) - expect(run.mock.calls.filter(([file]) => file === '/usr/bin/pgrep')).toHaveLength(3) + expect(run).not.toHaveBeenCalledWith('ps', expect.arrayContaining(['-e']), expect.anything()) + expect(run.mock.calls.filter(([file]) => file === 'pgrep')).toHaveLength(3) }) it('validates Darwin root ownership before targeted descendant teardown', async () => { const run = vi.fn(async (file: string, args: string[]) => { - if (file === '/bin/ps' && args.includes('sess=')) { + if (file === 'ps' && args.includes('pgid=')) { return { stdout: '4242 ttys042\n' } } - if (file === '/usr/bin/pgrep') { + if (file === 'pgrep') { if (args[1] === '4242') { return { stdout: '4243\n' } } throw emptySelection() } - if (file === '/bin/ps' && args.includes('pid=')) { + if (file === 'ps' && args.includes('ppid=')) { + return { stdout: '4243 4242\n' } + } + if (file === 'ps' && args.includes('stat=')) { throw emptySelection() } throw new Error(`Unexpected command: ${file} ${args.join(' ')}`) @@ -88,9 +96,16 @@ describe('killPosixPtySession', () => { await expect( killPosixPtySession(4242, '/dev/ttys042', 'darwin', run, killProcess) ).resolves.toBe(true) - expect(run).toHaveBeenCalledWith('/bin/ps', ['-p', '4242', '-o', 'sess=', '-o', 'tty='], { + expect(run).toHaveBeenCalledTimes(6) + expect(run).toHaveBeenNthCalledWith(1, 'ps', ['-p', '4242', '-o', 'pgid=', '-o', 'tty='], { + timeout: PTY_SESSION_COMMAND_TIMEOUT_MS + }) + expect(run).toHaveBeenNthCalledWith(2, 'ps', ['-p', '4242', '-o', 'pgid=', '-o', 'tty='], { timeout: PTY_SESSION_COMMAND_TIMEOUT_MS }) + expect(run).toHaveBeenLastCalledWith('ps', ['-p', '4242,4243', '-o', 'pid=', '-o', 'stat='], { + timeout: expect.any(Number) + }) expect(killProcess.mock.calls).toEqual([ [4242, 'SIGSTOP'], [4243, 'SIGSTOP'], @@ -126,10 +141,38 @@ describe('killPosixPtySession', () => { expect(killProcess).not.toHaveBeenCalled() }) + it('fails closed when node-pty cannot provide the exact controlling TTY', async () => { + const run = vi.fn() + const killProcess = vi.fn() + + await expect(killPosixPtySession(4242, undefined, 'linux', run, killProcess)).resolves.toBe( + false + ) + expect(run).not.toHaveBeenCalled() + expect(killProcess).not.toHaveBeenCalled() + }) + + it('resumes the root when its ownership changes after SIGSTOP', async () => { + const run = vi + .fn() + .mockResolvedValueOnce({ stdout: '4242 pts/7\n' }) + .mockResolvedValueOnce({ stdout: '9999 pts/7\n' }) + const killProcess = vi.fn() + + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + false + ) + expect(killProcess.mock.calls).toEqual([ + [4242, 'SIGSTOP'], + [4242, 'SIGCONT'] + ]) + }) + it('resumes the frozen root when a discovered child exits before it can be frozen', async () => { const run = vi .fn() .mockResolvedValueOnce({ stdout: '4242 pts/7\n' }) + .mockResolvedValueOnce({ stdout: '4242 pts/7\n' }) .mockResolvedValueOnce({ stdout: '4243\n' }) const killProcess = vi.fn((pid: number, signal: NodeJS.Signals) => { if (pid === 4243 && signal === 'SIGSTOP') { @@ -151,6 +194,7 @@ describe('killPosixPtySession', () => { const run = vi .fn() .mockResolvedValueOnce({ stdout: '4242 pts/7\n' }) + .mockResolvedValueOnce({ stdout: '4242 pts/7\n' }) .mockRejectedValueOnce(new Error('pgrep unavailable')) const killProcess = vi.fn() @@ -164,11 +208,11 @@ describe('killPosixPtySession', () => { }) it('fails closed when a captured process remains runnable after SIGKILL', async () => { - const run = vi.fn(async (file: string) => { - if (file === '/usr/bin/pgrep') { + const run = vi.fn(async (file: string, args: string[]) => { + if (file === 'pgrep') { throw emptySelection() } - if (run.mock.calls.length === 1) { + if (file === 'ps' && args.includes('sid=')) { return { stdout: '4242 pts/7\n' } } return { stdout: '4242 D\n' } @@ -181,26 +225,120 @@ describe('killPosixPtySession', () => { }) it('uses bounded targeted command timeouts for ownership, children, and verification', async () => { - const run = vi.fn(async (file: string, _args?: string[], _options?: { timeout: number }) => { - if (file === '/usr/bin/pgrep') { + const run = vi.fn(async (file: string, args?: string[], _options?: { timeout: number }) => { + if (file === 'pgrep') { throw emptySelection() } - if (run.mock.calls.length === 1) { + if (args?.includes('sid=')) { return { stdout: '4242 pts/7\n' } } throw emptySelection() }) await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, vi.fn())).resolves.toBe(true) - expect(run).toHaveBeenNthCalledWith(1, '/bin/ps', ['-p', '4242', '-o', 'sid=', '-o', 'tty='], { + expect(run).toHaveBeenNthCalledWith(1, 'ps', ['-p', '4242', '-o', 'sid=', '-o', 'tty='], { + timeout: PTY_SESSION_COMMAND_TIMEOUT_MS + }) + expect(run).toHaveBeenNthCalledWith(2, 'ps', ['-p', '4242', '-o', 'sid=', '-o', 'tty='], { timeout: PTY_SESSION_COMMAND_TIMEOUT_MS }) - expect(run).toHaveBeenNthCalledWith(2, '/usr/bin/pgrep', ['-P', '4242'], { + expect(run).toHaveBeenNthCalledWith(3, 'pgrep', ['-P', '4242'], { timeout: expect.any(Number) }) - expect(run.mock.calls[1]?.[2]?.timeout).toBeLessThanOrEqual(PTY_SESSION_COMMAND_TIMEOUT_MS) - expect(run).toHaveBeenNthCalledWith(3, '/bin/ps', ['-p', '4242', '-o', 'pid=', '-o', 'stat='], { - timeout: PTY_SESSION_VERIFY_TIMEOUT_MS + expect(run.mock.calls[2]?.[2]?.timeout).toBeLessThanOrEqual(PTY_SESSION_COMMAND_TIMEOUT_MS) + expect(run).toHaveBeenNthCalledWith(4, 'ps', ['-p', '4242', '-o', 'pid=', '-o', 'stat='], { + timeout: expect.any(Number) }) + expect(run.mock.calls[3]?.[2]?.timeout).toBeLessThanOrEqual(PTY_SESSION_VERIFY_TIMEOUT_MS) + }) + + it('resumes every frozen process when a stopped child was reparented', async () => { + const run = vi.fn(async (file: string, args: string[]) => { + if (file === 'ps' && args.includes('sid=')) { + return { stdout: '4242 pts/7\n' } + } + if (file === 'pgrep') { + return { stdout: '4243\n' } + } + if (file === 'ps' && args.includes('ppid=')) { + return { stdout: '4243 9999\n' } + } + throw new Error(`Unexpected command: ${file} ${args.join(' ')}`) + }) + const killProcess = vi.fn() + + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + false + ) + expect(killProcess.mock.calls).toEqual([ + [4242, 'SIGSTOP'], + [4243, 'SIGSTOP'], + [4242, 'SIGCONT'], + [4243, 'SIGCONT'] + ]) + }) + + it('batches large Darwin trees under one final verification deadline', async () => { + const rootPid = 5000 + const childPids = Array.from({ length: 129 }, (_, index) => rootPid + index + 1) + let now = 10_000 + const dateNow = vi.spyOn(Date, 'now').mockImplementation(() => { + const current = now + now += 5 + return current + }) + const run = vi.fn(async (file: string, args: string[], _options?: { timeout: number }) => { + if (file === 'ps' && args.includes('pgid=')) { + return { stdout: `${rootPid} ttys042\n` } + } + if (file === 'pgrep') { + if (args[1] === String(rootPid)) { + return { stdout: childPids.join('\n') } + } + throw emptySelection() + } + if (file === 'ps' && args.includes('ppid=')) { + return { + stdout: args[1]! + .split(',') + .map((pid) => `${pid} ${rootPid}`) + .join('\n') + } + } + if (file === 'ps' && args.includes('stat=')) { + return { + stdout: args[1]! + .split(',') + .map((pid) => `${pid} Z`) + .join('\n') + } + } + throw new Error(`Unexpected command: ${file} ${args.join(' ')}`) + }) + + try { + await expect( + killPosixPtySession(rootPid, '/dev/ttys042', 'darwin', run, vi.fn()) + ).resolves.toBe(true) + } finally { + dateNow.mockRestore() + } + + const parentChecks = run.mock.calls.filter( + ([file, args]) => file === 'ps' && args.includes('ppid=') + ) + const finalChecks = run.mock.calls.filter( + ([file, args]) => file === 'ps' && args.includes('stat=') + ) + expect(parentChecks).toHaveLength(3) + expect(finalChecks).toHaveLength(3) + for (const [, args, options] of [...parentChecks, ...finalChecks]) { + expect(args[1]!.split(',').length).toBeLessThanOrEqual(64) + expect(options!.timeout).toBeGreaterThan(0) + expect(options!.timeout).toBeLessThanOrEqual(PTY_SESSION_COMMAND_TIMEOUT_MS) + } + const finalTimeouts = finalChecks.map(([, , options]) => options!.timeout) + expect(finalTimeouts.at(-1)).toBeLessThan(finalTimeouts[0]) + expect(finalTimeouts[0]).toBeLessThanOrEqual(PTY_SESSION_VERIFY_TIMEOUT_MS) }) }) diff --git a/src/relay/pty-session-kill.ts b/src/relay/pty-session-kill.ts index 056c35e1201..286c578d09c 100644 --- a/src/relay/pty-session-kill.ts +++ b/src/relay/pty-session-kill.ts @@ -30,6 +30,12 @@ export async function killPosixPtySession( if (!stopProcess(pid, stopped, killProcess)) { return false } + // Why: the PID can exit and be reused between the ownership probe and + // SIGSTOP. Re-prove the frozen root before discovering or signaling children. + if (!(await rootStillOwnsPty(pid, ptsName, platform, run))) { + resumeProcesses(stopped, killProcess) + return false + } const processTree = await freezeProcessTree(pid, stopped, run, killProcess) if (!processTree) { resumeProcesses(stopped, killProcess) @@ -44,7 +50,7 @@ export async function killPosixPtySession( } stopped.delete(candidate) } - return await verifyProcessesStopped(processTree, platform, run) + return await verifyProcessesStopped(processTree, run) } catch { resumeProcesses(stopped, killProcess) return false @@ -57,21 +63,21 @@ async function rootStillOwnsPty( platform: NodeJS.Platform, run: ExecFile ): Promise { - const sessionColumn = platform === 'darwin' ? 'sess=' : 'sid=' - const result = (await run('/bin/ps', ['-p', String(pid), '-o', sessionColumn, '-o', 'tty='], { + if (typeof ptsName !== 'string') { + // Why: without the exact controlling TTY, a recycled session-leader PID + // is not enough authority to signal an unrelated user's process tree. + return false + } + const ownerColumn = platform === 'darwin' ? 'pgid=' : 'sid=' + const result = (await run('ps', ['-p', String(pid), '-o', ownerColumn, '-o', 'tty='], { timeout: PTY_SESSION_COMMAND_TIMEOUT_MS })) as { stdout?: string | Buffer } - const [sidText, tty = ''] = String(result.stdout ?? '') + const [ownerText, tty = ''] = String(result.stdout ?? '') .trim() .split(/\s+/) - if (Number(sidText) !== pid) { + if (Number(ownerText) !== pid) { return false } - if (typeof ptsName !== 'string') { - // Why: some node-pty builds omit the private ptsName field. The - // session-leader check plus a real controlling TTY still proves ownership. - return tty.length > 0 && tty !== '??' && tty !== '?' - } const expectedTty = ptsName.startsWith('/dev/') ? ptsName.slice('/dev/'.length) : ptsName return /^[A-Za-z0-9._/-]+$/.test(expectedTty) && tty === expectedTty } @@ -94,7 +100,9 @@ async function freezeProcessTree( return null } const parents = frontier.slice(index, index + PARENT_PID_BATCH_SIZE) - for (const child of await listChildren(parents, run, remainingMs)) { + const children = await listChildren(parents, run, remainingMs) + const newlyStopped: number[] = [] + for (const child of children) { if (seen.has(child)) { continue } @@ -105,6 +113,22 @@ async function freezeProcessTree( if (!stopProcess(child, stopped, killProcess)) { return null } + newlyStopped.push(child) + } + const childVerificationRemainingMs = discoveryDeadline - Date.now() + if ( + newlyStopped.length > 0 && + (childVerificationRemainingMs <= 0 || + !(await frozenChildrenStillBelongToParents( + newlyStopped, + parents, + run, + childVerificationRemainingMs + ))) + ) { + return null + } + for (const child of newlyStopped) { processTree.push(child) nextFrontier.push(child) } @@ -120,7 +144,7 @@ async function listChildren( timeout: number ): Promise { try { - const result = (await run('/usr/bin/pgrep', ['-P', parentPids.join(',')], { + const result = (await run('pgrep', ['-P', parentPids.join(',')], { timeout })) as { stdout?: string | Buffer } return String(result.stdout ?? '') @@ -136,6 +160,39 @@ async function listChildren( } } +async function frozenChildrenStillBelongToParents( + childPids: readonly number[], + parentPids: readonly number[], + run: ExecFile, + timeout: number +): Promise { + const expectedParents = new Set(parentPids) + const verifiedChildren = new Set() + const verificationDeadline = Date.now() + timeout + for (let index = 0; index < childPids.length; index += VERIFY_PID_BATCH_SIZE) { + const remainingMs = verificationDeadline - Date.now() + if (remainingMs <= 0) { + return false + } + const batch = childPids.slice(index, index + VERIFY_PID_BATCH_SIZE) + const result = (await run('ps', ['-p', batch.join(','), '-o', 'pid=', '-o', 'ppid='], { + timeout: remainingMs + })) as { stdout?: string | Buffer } + for (const line of String(result.stdout ?? '') + .trim() + .split('\n')) { + const [pidText, parentPidText] = line.trim().split(/\s+/) + const pid = Number(pidText) + const parentPid = Number(parentPidText) + if (!batch.includes(pid) || !expectedParents.has(parentPid) || verifiedChildren.has(pid)) { + return false + } + verifiedChildren.add(pid) + } + } + return verifiedChildren.size === childPids.length +} + function stopProcess(pid: number, stopped: Set, killProcess: KillProcess): boolean { try { killProcess(pid, 'SIGSTOP') @@ -162,18 +219,17 @@ function resumeProcesses(stopped: Set, killProcess: KillProcess): void { stopped.clear() } -async function verifyProcessesStopped( - pids: number[], - platform: NodeJS.Platform, - run: ExecFile -): Promise { - const batchSize = platform === 'darwin' ? 1 : VERIFY_PID_BATCH_SIZE - for (let index = 0; index < pids.length; index += batchSize) { - const batch = pids.slice(index, index + batchSize) +async function verifyProcessesStopped(pids: number[], run: ExecFile): Promise { + const verificationDeadline = Date.now() + PTY_SESSION_VERIFY_TIMEOUT_MS + for (let index = 0; index < pids.length; index += VERIFY_PID_BATCH_SIZE) { + const remainingMs = verificationDeadline - Date.now() + if (remainingMs <= 0) { + return false + } + const batch = pids.slice(index, index + VERIFY_PID_BATCH_SIZE) try { - const pidSelector = platform === 'darwin' ? String(batch[0]) : batch.join(',') - const result = (await run('/bin/ps', ['-p', pidSelector, '-o', 'pid=', '-o', 'stat='], { - timeout: PTY_SESSION_VERIFY_TIMEOUT_MS + const result = (await run('ps', ['-p', batch.join(','), '-o', 'pid=', '-o', 'stat='], { + timeout: remainingMs })) as { stdout?: string | Buffer } if (!hasOnlyExitedProcesses(result.stdout)) { return false From 244c4a7bcbdfa08dfe59c247d2219ad5013bc06e Mon Sep 17 00:00:00 2001 From: bbingz Date: Mon, 13 Jul 2026 16:58:34 +0800 Subject: [PATCH 23/36] test(relay): remove stale teardown compatibility patch --- src/relay/pty-session-kill.test.ts | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/relay/pty-session-kill.test.ts b/src/relay/pty-session-kill.test.ts index 9b308f27f42..b55c668c96b 100644 --- a/src/relay/pty-session-kill.test.ts +++ b/src/relay/pty-session-kill.test.ts @@ -11,11 +11,6 @@ function emptySelection(): Error & { code: number; stdout: string } { describe('killPosixPtySession', () => { it('freezes and kills a Linux descendant that escaped into a new session', async () => { - const toReversedDescriptor = Object.getOwnPropertyDescriptor(Array.prototype, 'toReversed') - Object.defineProperty(Array.prototype, 'toReversed', { - configurable: true, - value: undefined - }) const run = vi.fn(async (file: string, args: string[]) => { if (file === 'ps' && args.includes('sid=')) { return { stdout: '4242 pts/7\n' } @@ -46,17 +41,9 @@ describe('killPosixPtySession', () => { }) const killProcess = vi.fn() - try { - await expect( - killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess) - ).resolves.toBe(true) - } finally { - if (toReversedDescriptor) { - Object.defineProperty(Array.prototype, 'toReversed', toReversedDescriptor) - } else { - Reflect.deleteProperty(Array.prototype, 'toReversed') - } - } + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + true + ) expect(killProcess.mock.calls).toEqual([ [4242, 'SIGSTOP'], From 73dd9161048c17049c9a441764eb3b4cae915929 Mon Sep 17 00:00:00 2001 From: bbingz Date: Mon, 13 Jul 2026 17:59:09 +0800 Subject: [PATCH 24/36] fix(relay): poll for process teardown completion --- src/relay/pty-session-kill.test.ts | 42 +++++++++++++++++ src/relay/pty-session-kill.ts | 76 +++++++++++++++++++++++------- 2 files changed, 101 insertions(+), 17 deletions(-) diff --git a/src/relay/pty-session-kill.test.ts b/src/relay/pty-session-kill.test.ts index b55c668c96b..57dc327998f 100644 --- a/src/relay/pty-session-kill.test.ts +++ b/src/relay/pty-session-kill.test.ts @@ -211,6 +211,48 @@ describe('killPosixPtySession', () => { ) }) + it('waits for a captured process to finish exiting after SIGKILL', async () => { + let verificationChecks = 0 + const run = vi.fn(async (file: string, args: string[]) => { + if (file === 'pgrep') { + throw emptySelection() + } + if (file === 'ps' && args.includes('sid=')) { + return { stdout: '4242 pts/7\n' } + } + if (file === 'ps' && args.includes('stat=')) { + verificationChecks += 1 + if (verificationChecks === 1) { + return { stdout: '4242 R\n' } + } + throw emptySelection() + } + throw new Error(`Unexpected command: ${file} ${args.join(' ')}`) + }) + + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, vi.fn())).resolves.toBe(true) + expect(verificationChecks).toBe(2) + }) + + it('fails closed when verification returns a non-canonical PID token', async () => { + const run = vi.fn(async (file: string, args: string[]) => { + if (file === 'pgrep') { + throw emptySelection() + } + if (file === 'ps' && args.includes('sid=')) { + return { stdout: '4242 pts/7\n' } + } + if (file === 'ps' && args.includes('stat=')) { + return { stdout: '+4242 Z\n' } + } + throw new Error(`Unexpected command: ${file} ${args.join(' ')}`) + }) + + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, vi.fn())).resolves.toBe( + false + ) + }) + it('uses bounded targeted command timeouts for ownership, children, and verification', async () => { const run = vi.fn(async (file: string, args?: string[], _options?: { timeout: number }) => { if (file === 'pgrep') { diff --git a/src/relay/pty-session-kill.ts b/src/relay/pty-session-kill.ts index 286c578d09c..ce8c1127ec3 100644 --- a/src/relay/pty-session-kill.ts +++ b/src/relay/pty-session-kill.ts @@ -1,4 +1,5 @@ import { execFile as execFileCallback } from 'node:child_process' +import { setTimeout as wait } from 'node:timers/promises' import { promisify } from 'node:util' type ExecFile = (file: string, args: string[], options: { timeout: number }) => Promise @@ -9,6 +10,7 @@ export const PTY_SESSION_COMMAND_TIMEOUT_MS = 2500 export const PTY_SESSION_VERIFY_TIMEOUT_MS = 500 export const MAX_PTY_PROCESS_TREE_SIZE = 1024 const VERIFY_PID_BATCH_SIZE = 64 +const VERIFY_POLL_INTERVAL_MS = 10 const PARENT_PID_BATCH_SIZE = 64 export async function killPosixPtySession( @@ -221,35 +223,75 @@ function resumeProcesses(stopped: Set, killProcess: KillProcess): void { async function verifyProcessesStopped(pids: number[], run: ExecFile): Promise { const verificationDeadline = Date.now() + PTY_SESSION_VERIFY_TIMEOUT_MS - for (let index = 0; index < pids.length; index += VERIFY_PID_BATCH_SIZE) { - const remainingMs = verificationDeadline - Date.now() - if (remainingMs <= 0) { - return false - } - const batch = pids.slice(index, index + VERIFY_PID_BATCH_SIZE) - try { - const result = (await run('ps', ['-p', batch.join(','), '-o', 'pid=', '-o', 'stat='], { - timeout: remainingMs - })) as { stdout?: string | Buffer } - if (!hasOnlyExitedProcesses(result.stdout)) { + let pending = pids + while (pending.length > 0) { + const nextPending: number[] = [] + for (let index = 0; index < pending.length; index += VERIFY_PID_BATCH_SIZE) { + const remainingMs = verificationDeadline - Date.now() + if (remainingMs <= 0) { return false } - } catch (error) { - if (!isEmptyProcessSelection(error)) { - return false + const batch = pending.slice(index, index + VERIFY_PID_BATCH_SIZE) + try { + const result = (await run('ps', ['-p', batch.join(','), '-o', 'pid=', '-o', 'stat='], { + timeout: remainingMs + })) as { stdout?: string | Buffer } + const livePids = readLiveProcessIds(result.stdout, batch) + if (!livePids) { + return false + } + nextPending.push(...livePids) + } catch (error) { + if (!isEmptyProcessSelection(error)) { + return false + } } } + pending = nextPending + if (pending.length === 0) { + return true + } + const remainingMs = verificationDeadline - Date.now() + if (remainingMs <= 0) { + return false + } + // Why: signal delivery and process reaping are asynchronous; retry the + // still-live subset without busy-spawning ps until the bounded deadline. + await wait(Math.min(VERIFY_POLL_INTERVAL_MS, remainingMs)) } return true } -function hasOnlyExitedProcesses(stdout: string | Buffer | undefined): boolean { - return String(stdout ?? '') +function readLiveProcessIds( + stdout: string | Buffer | undefined, + expectedPids: readonly number[] +): number[] | null { + const expected = new Set(expectedPids) + const seen = new Set() + const livePids: number[] = [] + const lines = String(stdout ?? '') .trim() .split('\n') .map((line) => line.trim()) .filter(Boolean) - .every((line) => line.split(/\s+/).at(-1)?.startsWith('Z') === true) + for (const line of lines) { + const columns = line.split(/\s+/) + const [pidText, status] = columns + // Why: numeric coercion accepts signs, decimals, and exponents; process + // verification must accept only the canonical PID text emitted by ps. + if (columns.length !== 2 || !pidText || !/^[1-9]\d*$/.test(pidText)) { + return null + } + const pid = Number(pidText) + if (!Number.isSafeInteger(pid) || !expected.has(pid) || seen.has(pid) || !status) { + return null + } + seen.add(pid) + if (!status.startsWith('Z')) { + livePids.push(pid) + } + } + return livePids } function isEmptyProcessSelection(error: unknown): boolean { From a3bba3198e100cc77cbf016ea1fdcaacbab6d1f4 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:07:46 -0700 Subject: [PATCH 25/36] test(terminals): make teardown suite platform-explicit --- src/relay/pty-handler.test.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index c0f9b722797..c0b2935dcae 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -42,6 +42,12 @@ vi.mock('./pty-session-kill', () => ({ import { PtyHandler, attachIdentityMismatches } from './pty-handler' import type { RelayDispatcher } from './dispatcher' +const hostPlatform = process.platform + +function setProcessPlatform(platform: NodeJS.Platform): void { + Object.defineProperty(process, 'platform', { configurable: true, value: platform }) +} + function createMockDispatcher() { const requestHandlers = new Map< string, @@ -100,6 +106,9 @@ describe('PtyHandler', () => { let handler: PtyHandler beforeEach(() => { + // Why: most relay shutdown cases assert POSIX signal semantics; make that + // baseline deterministic and let the explicit ConPTY cases opt into win32. + setProcessPlatform('linux') vi.useFakeTimers() mockPtySpawn.mockReset() mockPtyInstance.onData.mockReset() @@ -118,8 +127,13 @@ describe('PtyHandler', () => { }) afterEach(() => { - handler.dispose() - vi.useRealTimers() + try { + setProcessPlatform('linux') + handler.dispose() + } finally { + vi.useRealTimers() + setProcessPlatform(hostPlatform) + } }) it('registers all expected handlers', () => { From 2d1b78188f6d92f63d1bc67182606baacffdc07f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:24:36 -0700 Subject: [PATCH 26/36] fix(terminals): align remote teardown budgets --- src/main/daemon/pty-subprocess.ts | 5 +-- src/main/providers/ssh-pty-provider.test.ts | 10 ++++++ src/main/providers/ssh-pty-provider.ts | 6 ++-- src/relay/pty-handler.test.ts | 34 +++++++++++++++++++-- src/relay/pty-handler.ts | 4 +-- src/relay/pty-session-kill.test.ts | 5 +++ src/relay/pty-session-kill.ts | 26 ++++++++++++---- src/shared/terminal-teardown-timeouts.ts | 20 ++++++++++++ 8 files changed, 93 insertions(+), 17 deletions(-) create mode 100644 src/shared/terminal-teardown-timeouts.ts diff --git a/src/main/daemon/pty-subprocess.ts b/src/main/daemon/pty-subprocess.ts index 0294b0a43e5..c7f993b24c7 100644 --- a/src/main/daemon/pty-subprocess.ts +++ b/src/main/daemon/pty-subprocess.ts @@ -42,6 +42,7 @@ import { } from '../pty/powerlevel10k-wizard-env' import { isWindowsGitBashShellPath, resolveWindowsGitBashShellPath } from '../git-bash' import { WINDOWS_GIT_BASH_SHELL } from '../../shared/windows-terminal-shell' +import { WINDOWS_PTY_EXIT_TIMEOUT_MS } from '../../shared/terminal-teardown-timeouts' import { resolveAgentForegroundProcessWithAvailability } from '../providers/agent-foreground-process' import { readWindowsConptyProcessIds } from '../providers/windows-conpty-process-membership' import { @@ -82,10 +83,6 @@ const PTY_SPAWN_HEALTH_TIMEOUT_MS = 4_000 // fallback (losing daemon persistence) until a manual restart. const PTY_SPAWN_HEALTH_RETRY_ATTEMPTS = 2 const PENDING_PRE_LISTENER_DATA_MAX_CHARS = 512 * 1024 -// Why: node-pty can spend up to 5s enumerating a Windows console tree before -// its fallback runs; leave bounded headroom for the native exit event to land. -const WINDOWS_PTY_EXIT_TIMEOUT_MS = 8_000 - export type PtySubprocessOptions = { sessionId: string cols: number diff --git a/src/main/providers/ssh-pty-provider.test.ts b/src/main/providers/ssh-pty-provider.test.ts index 032f0bf9de5..336dc40670e 100644 --- a/src/main/providers/ssh-pty-provider.test.ts +++ b/src/main/providers/ssh-pty-provider.test.ts @@ -7,6 +7,10 @@ import { } from './ssh-pty-provider' import { POWERLEVEL10K_WIZARD_DISABLE_ENV } from '../pty/powerlevel10k-wizard-env' import { JsonRpcErrorCode } from '../ssh/relay-protocol' +import { + POSIX_PTY_SESSION_TEARDOWN_TIMEOUT_MS, + RELAY_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS +} from '../../shared/terminal-teardown-timeouts' type MockMultiplexer = { request: ReturnType @@ -433,6 +437,12 @@ describe('SshPtyProvider', () => { }, { timeoutMs: SSH_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS } ) + expect(SSH_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS).toBeGreaterThan( + RELAY_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS + ) + expect(RELAY_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS).toBeGreaterThanOrEqual( + POSIX_PTY_SESSION_TEARDOWN_TIMEOUT_MS + ) }) it('keeps graceful legacy liveness authoritative until the relay exit event', async () => { diff --git a/src/main/providers/ssh-pty-provider.ts b/src/main/providers/ssh-pty-provider.ts index 85b25adac70..24d61237d26 100644 --- a/src/main/providers/ssh-pty-provider.ts +++ b/src/main/providers/ssh-pty-provider.ts @@ -3,6 +3,9 @@ import type { IPtyProvider, PtyProcessInfo, PtySpawnOptions, PtySpawnResult } fr import { toAppSshPtyId, toRelaySshPtyId } from './ssh-pty-id' import { SshPtyLiveness } from './ssh-pty-liveness' import { seedPowerlevel10kWizardEnv } from '../pty/powerlevel10k-wizard-env' +import { SSH_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS } from '../../shared/terminal-teardown-timeouts' + +export { SSH_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS } from '../../shared/terminal-teardown-timeouts' type DataCallback = (payload: { id: string; data: string }) => void type ReplayCallback = (payload: { id: string; data: string }) => void @@ -17,9 +20,6 @@ type RemoteCliBridgeEnv = { export const SSH_SESSION_EXPIRED_ERROR = 'SSH_SESSION_EXPIRED' export const SSH_PTY_IDENTITY_MISMATCH_ERROR = 'SSH_PTY_IDENTITY_MISMATCH' -// Why: eight bounded workers keep a 50-PTY failure near one 30s batch, -// while leaving enough time for the relay's targeted 3s kill-and-verify path. -export const SSH_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS = 3250 export const SSH_PTY_TEARDOWN_LIVENESS_TIMEOUT_MS = 1000 export const SSH_PTY_LEGACY_INVENTORY_TIMEOUT_MS = 2000 diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index c0b2935dcae..38261cc04c5 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -10,6 +10,7 @@ import { resolveSetupAgentSequenceLaunchCommand, SETUP_AGENT_SEQUENCE_STARTUP_COMMAND_ENV } from '../shared/setup-agent-sequencing' +import { RELAY_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS } from '../shared/terminal-teardown-timeouts' const { mockPtySpawn, mockPtyInstance } = vi.hoisted(() => ({ mockPtySpawn: vi.fn(), @@ -1240,6 +1241,35 @@ describe('PtyHandler', () => { } }) + it('allows a slow Windows ConPTY exit within the shared relay budget', async () => { + const originalPlatform = process.platform + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + const mockKill = vi.fn() + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + }) + try { + await dispatcher.callRequest('pty.spawn', {}) + const shutdown = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + + await vi.advanceTimersByTimeAsync(5_100) + expect(handler.activePtyCount).toBe(1) + expect(mockKill).toHaveBeenCalledTimes(1) + + onExitCb!({ exitCode: 0 }) + await expect(shutdown).resolves.toBeUndefined() + expect(handler.activePtyCount).toBe(0) + } finally { + Object.defineProperty(process, 'platform', { configurable: true, value: originalPlatform }) + } + }) + it('retains a Windows ConPTY owner when immediate exit times out', async () => { const originalPlatform = process.platform Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) @@ -1256,7 +1286,7 @@ describe('PtyHandler', () => { const rejection = expect(shutdown).rejects.toThrow( 'Timed out waiting for Windows PTY teardown: pty-1' ) - await vi.advanceTimersByTimeAsync(3_000) + await vi.advanceTimersByTimeAsync(RELAY_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS) await rejection expect(mockKill).toHaveBeenCalledTimes(1) @@ -1285,7 +1315,7 @@ describe('PtyHandler', () => { const firstRejection = expect(first).rejects.toThrow( 'Timed out waiting for Windows PTY teardown: pty-1' ) - await vi.advanceTimersByTimeAsync(3_000) + await vi.advanceTimersByTimeAsync(RELAY_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS) await firstRejection const retry = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index 659db4139d5..c8baf5259e9 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -18,6 +18,7 @@ import { DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS } from '../shared/ssh-types' import { shouldUseShellReadyStartupDelivery } from '../shared/codex-startup-delivery' import { buildStartupCommandSubmission } from '../shared/startup-command-submission' import { resolveSetupAgentSequenceLaunchCommand } from '../shared/setup-agent-sequencing' +import { RELAY_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS } from '../shared/terminal-teardown-timeouts' import { createShellReadyScanState, drainShellReadyHeldBytes, @@ -132,7 +133,6 @@ function disposeManagedPty(managed: ManagedPty): void { } } const DEFAULT_GRACE_TIME_MS = DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS * 1000 -const WINDOWS_IMMEDIATE_EXIT_TIMEOUT_MS = 3_000 export const REPLAY_BUFFER_MAX = 100 * 1024 const PTY_OUTPUT_BATCH_INTERVAL_MS = 8 const PTY_OUTPUT_DRAIN_CONTINUE_MS = 1 @@ -490,7 +490,7 @@ export class PtyHandler { this.exitWaitersByPty.delete(id) } resolvePromise(false) - }, WINDOWS_IMMEDIATE_EXIT_TIMEOUT_MS) + }, RELAY_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS) const promise = new Promise((resolve) => { resolvePromise = resolve }) diff --git a/src/relay/pty-session-kill.test.ts b/src/relay/pty-session-kill.test.ts index 57dc327998f..6941af92c8c 100644 --- a/src/relay/pty-session-kill.test.ts +++ b/src/relay/pty-session-kill.test.ts @@ -195,6 +195,7 @@ describe('killPosixPtySession', () => { }) it('fails closed when a captured process remains runnable after SIGKILL', async () => { + let verificationChecks = 0 const run = vi.fn(async (file: string, args: string[]) => { if (file === 'pgrep') { throw emptySelection() @@ -202,6 +203,7 @@ describe('killPosixPtySession', () => { if (file === 'ps' && args.includes('sid=')) { return { stdout: '4242 pts/7\n' } } + verificationChecks += 1 return { stdout: '4242 D\n' } }) const killProcess = vi.fn() @@ -209,6 +211,9 @@ describe('killPosixPtySession', () => { await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( false ) + // Initial 10ms backoff doubling to 100ms permits at most nine ps rounds + // inside the shared 500ms verification deadline. + expect(verificationChecks).toBeLessThanOrEqual(9) }) it('waits for a captured process to finish exiting after SIGKILL', async () => { diff --git a/src/relay/pty-session-kill.ts b/src/relay/pty-session-kill.ts index ce8c1127ec3..bc56afd14bc 100644 --- a/src/relay/pty-session-kill.ts +++ b/src/relay/pty-session-kill.ts @@ -1,16 +1,24 @@ import { execFile as execFileCallback } from 'node:child_process' import { setTimeout as wait } from 'node:timers/promises' import { promisify } from 'node:util' +import { + PTY_SESSION_COMMAND_TIMEOUT_MS, + PTY_SESSION_VERIFY_TIMEOUT_MS +} from '../shared/terminal-teardown-timeouts' + +export { + PTY_SESSION_COMMAND_TIMEOUT_MS, + PTY_SESSION_VERIFY_TIMEOUT_MS +} from '../shared/terminal-teardown-timeouts' type ExecFile = (file: string, args: string[], options: { timeout: number }) => Promise type KillProcess = (pid: number, signal: NodeJS.Signals) => void const execFile = promisify(execFileCallback) as ExecFile -export const PTY_SESSION_COMMAND_TIMEOUT_MS = 2500 -export const PTY_SESSION_VERIFY_TIMEOUT_MS = 500 export const MAX_PTY_PROCESS_TREE_SIZE = 1024 const VERIFY_PID_BATCH_SIZE = 64 -const VERIFY_POLL_INTERVAL_MS = 10 +const VERIFY_POLL_INITIAL_MS = 10 +const VERIFY_POLL_MAX_MS = 100 const PARENT_PID_BATCH_SIZE = 64 export async function killPosixPtySession( @@ -224,6 +232,7 @@ function resumeProcesses(stopped: Set, killProcess: KillProcess): void { async function verifyProcessesStopped(pids: number[], run: ExecFile): Promise { const verificationDeadline = Date.now() + PTY_SESSION_VERIFY_TIMEOUT_MS let pending = pids + let pollDelayMs = VERIFY_POLL_INITIAL_MS while (pending.length > 0) { const nextPending: number[] = [] for (let index = 0; index < pending.length; index += VERIFY_PID_BATCH_SIZE) { @@ -255,9 +264,14 @@ async function verifyProcessesStopped(pids: number[], run: ExecFile): Promise Date: Mon, 13 Jul 2026 12:48:23 -0700 Subject: [PATCH 27/36] fix(daemon): reject unverified legacy teardown --- src/main/daemon/daemon-pty-adapter.test.ts | 12 +++++ src/main/daemon/daemon-pty-adapter.ts | 5 +++ src/main/daemon/daemon-pty-router.test.ts | 15 ++++++- src/main/daemon/daemon-pty-router.ts | 4 ++ .../degraded-daemon-pty-provider.test.ts | 18 +++++++- .../daemon/degraded-daemon-pty-provider.ts | 3 ++ src/main/daemon/types.ts | 3 ++ src/main/ipc/pty.test.ts | 44 +++++++++++++++++++ src/main/ipc/pty.ts | 7 +++ src/main/providers/types.ts | 4 ++ 10 files changed, 112 insertions(+), 3 deletions(-) diff --git a/src/main/daemon/daemon-pty-adapter.test.ts b/src/main/daemon/daemon-pty-adapter.test.ts index 37896ab8336..29374b5793a 100644 --- a/src/main/daemon/daemon-pty-adapter.test.ts +++ b/src/main/daemon/daemon-pty-adapter.test.ts @@ -191,6 +191,18 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { }) }) + describe('verified full-session teardown capability', () => { + it('accepts v22 and rejects a retained v21 daemon', () => { + const legacy = new DaemonPtyAdapter({ socketPath, tokenPath, protocolVersion: 21 }) + try { + expect(adapter.canVerifyFullSessionTeardown('current-session')).toBe(true) + expect(legacy.canVerifyFullSessionTeardown('legacy-session')).toBe(false) + } finally { + legacy.dispose() + } + }) + }) + describe('producer flow control', () => { it('routes pausePty/resumePty notifications to the daemon session subprocess', async () => { const { id } = await adapter.spawn({ cols: 80, rows: 24 }) diff --git a/src/main/daemon/daemon-pty-adapter.ts b/src/main/daemon/daemon-pty-adapter.ts index 25a9be9754a..128ecdd046f 100644 --- a/src/main/daemon/daemon-pty-adapter.ts +++ b/src/main/daemon/daemon-pty-adapter.ts @@ -12,6 +12,7 @@ import { supportsPtyStartupBarrier } from './shell-ready' import { CODEX_SHELL_READY_TIMEOUT_MS } from './session' import { PROTOCOL_VERSION, + VERIFIED_FULL_SESSION_TEARDOWN_PROTOCOL_VERSION, type CreateOrAttachResult, type DaemonEvent, type GetSnapshotResult, @@ -441,6 +442,10 @@ export class DaemonPtyAdapter implements IPtyProvider { return this.activeSessionIds.has(id) } + canVerifyFullSessionTeardown(_id: string): boolean { + return this.protocolVersion >= VERIFIED_FULL_SESSION_TEARDOWN_PROTOCOL_VERSION + } + write(id: string, data: string): void { this.markSessionDirty(id) this.client.notify('write', { sessionId: id, data }) diff --git a/src/main/daemon/daemon-pty-router.test.ts b/src/main/daemon/daemon-pty-router.test.ts index a5de628ef65..01a8c2dd871 100644 --- a/src/main/daemon/daemon-pty-router.test.ts +++ b/src/main/daemon/daemon-pty-router.test.ts @@ -22,7 +22,8 @@ function buildSessionIds(prefix: string, count: number): string[] { function createAdapter( label: string, sessions: string[] = [], - reconcileResult?: { alive: string[]; killed: string[] } + reconcileResult?: { alive: string[]; killed: string[] }, + protocolVersion = 22 ): AdapterMock { const writes: { id: string; data: string }[] = [] const dataListeners: ((payload: { id: string; data: string; sequenceChars?: number }) => void)[] = @@ -30,6 +31,7 @@ function createAdapter( const backgroundListeners: ((payload: PtyBackgroundStreamEvent) => void)[] = [] const exitListeners: ((payload: { id: string; code: number }) => void)[] = [] return { + protocolVersion, spawn: vi.fn(async (opts: PtySpawnOptions): Promise => { const id = opts.sessionId ?? `${label}-new` sessions.push(id) @@ -43,6 +45,7 @@ function createAdapter( })) ), hasPty: vi.fn((id: string) => sessions.includes(id)), + canVerifyFullSessionTeardown: vi.fn(() => protocolVersion >= 22), write: vi.fn((id: string, data: string) => { writes.push({ id, data }) }), @@ -156,6 +159,16 @@ describe('DaemonPtyRouter', () => { expect(current.write).toHaveBeenCalledWith(fresh.id, 'new\n') }) + it('reports that a v21 legacy owner cannot verify full-session teardown', async () => { + const current = createAdapter('current') + const legacy = createAdapter('legacy', ['legacy-session'], undefined, 21) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + await router.discoverLegacySessions() + + expect(router.canVerifyFullSessionTeardown('legacy-session')).toBe(false) + expect(router.canVerifyFullSessionTeardown('current-session')).toBe(true) + }) + it('routes background hints and authoritative snapshots to the session owner', async () => { const current = createAdapter('current') const legacy = createAdapter('legacy', ['legacy-session']) diff --git a/src/main/daemon/daemon-pty-router.ts b/src/main/daemon/daemon-pty-router.ts index dc17340f72a..c3d2a3fc052 100644 --- a/src/main/daemon/daemon-pty-router.ts +++ b/src/main/daemon/daemon-pty-router.ts @@ -83,6 +83,10 @@ export class DaemonPtyRouter implements IPtyProvider { return this.current.hasPty(id) || this.legacy.some((adapter) => adapter.hasPty(id)) } + canVerifyFullSessionTeardown(id: string): boolean { + return this.adapterFor(id).canVerifyFullSessionTeardown(id) + } + write(id: string, data: string): void { this.adapterFor(id).write(id, data) } diff --git a/src/main/daemon/degraded-daemon-pty-provider.test.ts b/src/main/daemon/degraded-daemon-pty-provider.test.ts index 3a34db0c700..eb31e610e22 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.test.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.test.ts @@ -92,11 +92,13 @@ function createProvider(label: string, sessions: string[] = []): ProviderMock { function createDaemonAdapter( label: string, - sessions: string[] = [] + sessions: string[] = [], + protocolVersion = 22 ): DaemonPtyAdapter & ProviderMock { return { ...createProvider(label, sessions), - protocolVersion: 13, + protocolVersion, + canVerifyFullSessionTeardown: vi.fn(() => protocolVersion >= 22), listSessions: vi.fn(async () => []), ackColdRestore: vi.fn(), clearTombstone: vi.fn(), @@ -140,6 +142,18 @@ describe('DegradedDaemonPtyProvider', () => { expect(fallback.write).toHaveBeenCalledWith(fresh.id, 'new\n') }) + it('rejects verified teardown for v21 daemon sessions but permits the local fallback', async () => { + const current = createDaemonAdapter('daemon') + const legacy = createDaemonAdapter('legacy', ['legacy-session'], 21) + const fallback = createProvider('fallback') + const provider = new DegradedDaemonPtyProvider({ current, legacy: [legacy], fallback }) + await provider.discoverDaemonSessions() + const fresh = await provider.spawn({ cols: 80, rows: 24 }) + + expect(provider.canVerifyFullSessionTeardown('legacy-session')).toBe(false) + expect(provider.canVerifyFullSessionTeardown(fresh.id)).toBe(true) + }) + it('routes a previously daemon-backed id to fallback after daemon exit removes the mapping', async () => { const current = createDaemonAdapter('daemon', ['daemon-session']) const fallback = createProvider('fallback') diff --git a/src/main/daemon/degraded-daemon-pty-provider.ts b/src/main/daemon/degraded-daemon-pty-provider.ts index 06696291114..7d247b08e4d 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.ts @@ -103,6 +103,9 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { return this.findProviderForExistingSession(id) !== null } + canVerifyFullSessionTeardown(id: string): boolean { + return this.providerFor(id).canVerifyFullSessionTeardown?.(id) ?? true + } write(id: string, data: string): void { this.providerFor(id).write(id, data) } diff --git a/src/main/daemon/types.ts b/src/main/daemon/types.ts index 1643a3e2a80..875eb9da883 100644 --- a/src/main/daemon/types.ts +++ b/src/main/daemon/types.ts @@ -17,6 +17,9 @@ import type { TuiAgent } from '../../shared/types' // Why: bump when adding daemon wire behavior so same-version old daemons do // not silently accept the handshake and then reject new RPCs. export const PROTOCOL_VERSION = 22 +// Why: v22 is the first daemon whose immediate kill awaits and verifies the +// complete POSIX process tree or Windows ConPTY exit instead of only its leader. +export const VERIFIED_FULL_SESSION_TEARDOWN_PROTOCOL_VERSION = 22 export const PREVIOUS_DAEMON_PROTOCOL_VERSIONS = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 ] as const diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index a130fb3b144..3c86612ffe2 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -2967,6 +2967,50 @@ describe('registerPtyHandlers', () => { expect(listProcesses).not.toHaveBeenCalled() }) + it('fails closed before shutdown when the provider cannot verify the full session', async () => { + const shutdown = vi.fn(async () => undefined) + const hasPty = vi.fn(() => false) + const runtime = { + setPtyController: vi.fn(), + onPtyExit: vi.fn() + } + setLocalPtyProvider({ + spawn: vi.fn(), + hasPty, + canVerifyFullSessionTeardown: vi.fn(() => false), + write: vi.fn(), + resize: vi.fn(), + shutdown, + sendSignal: vi.fn(), + getCwd: vi.fn(), + getInitialCwd: vi.fn(), + clearBuffer: vi.fn(), + acknowledgeDataEvent: vi.fn(), + hasChildProcesses: vi.fn(), + getForegroundProcess: vi.fn(), + serialize: vi.fn(), + revive: vi.fn(), + onData: vi.fn(() => () => {}), + onReplay: vi.fn(() => () => {}), + onExit: vi.fn(() => () => {}), + listProcesses: vi.fn(async () => []), + attach: vi.fn(), + getDefaultShell: vi.fn(), + getProfiles: vi.fn() + } as never) + handlers.clear() + registerPtyHandlers(mainWindow as never, runtime as never) + const controller = runtime.setPtyController.mock.calls[0]?.[0] as { + stopAndWait: (ptyId: string) => Promise + } + + await expect(controller.stopAndWait('legacy-daemon-pty')).resolves.toBe(false) + + expect(shutdown).not.toHaveBeenCalled() + expect(hasPty).not.toHaveBeenCalled() + expect(runtime.onPtyExit).not.toHaveBeenCalled() + }) + it('finishes headless current-relay SSH shutdown before remote removal without inventory', async () => { const order: string[] = [] const request = vi.fn(async (method: string) => { diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index de537ec9752..ec213d18e76 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -3421,6 +3421,13 @@ export function registerPtyHandlers( } return false } + // Why: protocol-v21 and older daemons acknowledge leader-only kills. + // Worktree deletion must preserve the session rather than accept that as + // proof that resistant descendants or a Windows ConPTY have exited. + if (provider.canVerifyFullSessionTeardown?.(ptyId) === false) { + console.warn(`[pty] Provider cannot verify full PTY session teardown: ${ptyId}`) + return false + } let providerExitObserved = false try { providerExitObserved = await shutdownProviderAndDetectExit(provider, ptyId, { diff --git a/src/main/providers/types.ts b/src/main/providers/types.ts index 0c5a6c3bde3..2a2f71fbf67 100644 --- a/src/main/providers/types.ts +++ b/src/main/providers/types.ts @@ -203,6 +203,10 @@ export type IPtyProvider = { */ getAppliedSize?: (id: string) => Promise<{ cols: number; rows: number } | null> + /** False when this provider can only stop the PTY leader and cannot prove + * the complete process/session tree exited. Destructive teardown must abort. */ + canVerifyFullSessionTeardown?: (id: string) => boolean + shutdown(id: string, opts: { immediate?: boolean; keepHistory?: boolean }): Promise sendSignal(id: string, signal: string): Promise getCwd(id: string): Promise From c4ff460d5276ff952cae0144095836b6d355cf3b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:58:52 -0700 Subject: [PATCH 28/36] fix(daemon): retry failed ConPTY teardown --- src/main/daemon/pty-subprocess.test.ts | 22 ++++++++++++++++++++++ src/main/daemon/pty-subprocess.ts | 3 +++ 2 files changed, 25 insertions(+) diff --git a/src/main/daemon/pty-subprocess.test.ts b/src/main/daemon/pty-subprocess.test.ts index 659a84c52aa..c321361f7a2 100644 --- a/src/main/daemon/pty-subprocess.test.ts +++ b/src/main/daemon/pty-subprocess.test.ts @@ -2680,6 +2680,28 @@ describe('createPtySubprocess', () => { } }) + it('retries verified Windows teardown after node-pty kill throws transiently', async () => { + const proc = mockPtyProcess(123456) + proc.kill + .mockImplementationOnce(() => { + throw new Error('transient ConPTY close failure') + }) + .mockImplementationOnce(() => proc._simulateExit(0)) + spawnMock.mockReturnValue(proc) + const origPlatform = Object.getOwnPropertyDescriptor(process, 'platform') + Object.defineProperty(process, 'platform', { value: 'win32' }) + try { + const handle = createPtySubprocess({ sessionId: 'test', cols: 80, rows: 24 }) + + await expect(handle.forceKillAndWait!()).resolves.toBe(false) + await expect(handle.forceKillAndWait!()).resolves.toBe(true) + + expect(proc.kill).toHaveBeenCalledTimes(2) + } finally { + restorePlatform(origPlatform) + } + }) + it('dispose() on Windows skips destroy after forceKill falls back to node-pty kill()', () => { const proc = mockPtyProcess(123456) as ReturnType & { destroy: ReturnType diff --git a/src/main/daemon/pty-subprocess.ts b/src/main/daemon/pty-subprocess.ts index c7f993b24c7..9dd67a4266f 100644 --- a/src/main/daemon/pty-subprocess.ts +++ b/src/main/daemon/pty-subprocess.ts @@ -1222,6 +1222,9 @@ export function createPtySubprocess(opts: PtySubprocessOptions): SubprocessHandl nodePtyKillIssued = true proc.kill() } catch { + // Why: a transient ConPTY close failure must not permanently block + // a later verified worktree-deletion retry. + nodePtyKillIssued = false return false } } From 20f470deff9ff5faaa62cdae339ed4b4a235b2ec Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:16:48 -0700 Subject: [PATCH 29/36] fix: avoid double-closing relay ConPTY --- src/relay/pty-handler.test.ts | 28 ++++++++++++++++++++++++++-- src/relay/pty-handler.ts | 14 +++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index 38261cc04c5..1aaa4ab9227 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -1114,6 +1114,30 @@ describe('PtyHandler', () => { }) }) + it('waits without double-closing when immediate teardown follows graceful shutdown', async () => { + await withWindowsPlatform(async () => { + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + const mockKill = vi.fn() + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + }) + await dispatcher.callRequest('pty.spawn', {}) + + await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: false }) + const immediate = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + + expectBareKills(mockKill, 1) + onExitCb!({ exitCode: 0 }) + await immediate + expectBareKills(mockKill, 1) + }) + }) + it('on stale-spawn cleanup including the SIGKILL fallback', async () => { await withWindowsPlatform(async () => { const mockKill = mockKillablePty() @@ -1124,14 +1148,14 @@ describe('PtyHandler', () => { }) }) - it('on graceful-shutdown SIGKILL fallback', async () => { + it('does not double-close ConPTY on the graceful-shutdown fallback', async () => { await withWindowsPlatform(async () => { const mockKill = mockKillablePty() await dispatcher.callRequest('pty.spawn', {}) await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: false }) expectBareKills(mockKill, 1) vi.advanceTimersByTime(5000) - expectBareKills(mockKill, 2) + expectBareKills(mockKill, 1) }) }) diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index c8baf5259e9..c79f80bee17 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -969,7 +969,19 @@ export class PtyHandler { this.clearPtyFlowState(id) } else { this.releaseStartupCommand(managed) - killPtyProcess(managed.pty, 'SIGTERM') + if (process.platform === 'win32') { + // Why: ConPTY close is already the force-capable operation. Remember + // it so an overlapping deletion waits instead of double-closing it. + managed.windowsImmediateKillIssued = true + try { + killPtyProcess(managed.pty, 'SIGTERM') + } catch (error) { + managed.windowsImmediateKillIssued = false + throw error + } + } else { + killPtyProcess(managed.pty, 'SIGTERM') + } // Why: Some processes ignore SIGTERM (e.g. a hung child, a custom signal // handler). Without a SIGKILL fallback the PTY process would leak and the From 04b3782dcea10a8954a69c96708463ce9fc199b8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 14 Jul 2026 06:56:51 -0700 Subject: [PATCH 30/36] fix disconnected SSH forget-local cleanup --- src/main/ipc/worktrees.test.ts | 19 +++++++++++++------ src/main/ipc/worktrees.ts | 6 ++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/main/ipc/worktrees.test.ts b/src/main/ipc/worktrees.test.ts index 1c1e3e07cb2..f2e280bcdee 100644 --- a/src/main/ipc/worktrees.test.ts +++ b/src/main/ipc/worktrees.test.ts @@ -8716,7 +8716,7 @@ describe('registerWorktreeHandlers', () => { expect(getSshGitProviderMock).not.toHaveBeenCalled() }) - it('keeps metadata when local terminal teardown cannot be verified', async () => { + it('forgets metadata when disconnected SSH teardown cannot be verified', async () => { const worktreeId = 'repo-1::/workspace/feature-wt' store.getRepo.mockReturnValue({ id: 'repo-1', @@ -8729,13 +8729,20 @@ describe('registerWorktreeHandlers', () => { killAllProcessesForWorktreeMock.mockRejectedValue( new Error('Failed to stop remote worktree terminals') ) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) - await expect(handlers['worktrees:forgetLocal'](null, { worktreeId })).rejects.toThrow( - 'Failed to stop remote worktree terminals' - ) + try { + await expect(handlers['worktrees:forgetLocal'](null, { worktreeId })).resolves.toEqual({}) - expect(store.removeWorktreeMeta).not.toHaveBeenCalled() - expect(deleteWorktreeHistoryDirMock).not.toHaveBeenCalled() + expect(warn).toHaveBeenCalledWith( + `[worktree-teardown] forget-local failed for ${worktreeId}:`, + expect.objectContaining({ message: 'Failed to stop remote worktree terminals' }) + ) + expect(store.removeWorktreeMeta).toHaveBeenCalledWith(worktreeId) + expect(deleteWorktreeHistoryDirMock).toHaveBeenCalledWith(worktreeId) + } finally { + warn.mockRestore() + } }) it('rejects forgetting a folder project root', async () => { diff --git a/src/main/ipc/worktrees.ts b/src/main/ipc/worktrees.ts index f0d4d745fc9..f2365922c1b 100644 --- a/src/main/ipc/worktrees.ts +++ b/src/main/ipc/worktrees.ts @@ -2064,13 +2064,15 @@ export function registerWorktreeHandlers( ) } - // Why: forgetting metadata releases Orca's last addressable owner. An - // unverified terminal must keep that owner intact so cleanup can retry. + // Why: Forget Locally is the escape hatch for a disconnected target. + // Provider absence cannot block removing only Orca's local metadata. await killAllProcessesForWorktree(args.worktreeId, { runtime, localProvider: getLocalPtyProvider(), connectionId: repo.connectionId ?? null, onPtyStopped: clearProviderPtyState + }).catch((error) => { + console.warn(`[worktree-teardown] forget-local failed for ${args.worktreeId}:`, error) }) runtime.clearOptimisticReconcileToken(args.worktreeId) From c3073823f8719fd6f2031fb35649958af52b715f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:13:00 -0700 Subject: [PATCH 31/36] fix(relay): skip redundant ConPTY close in the graceful SIGKILL fallback --- src/relay/pty-handler.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index c79f80bee17..1e31c39f580 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -995,7 +995,12 @@ export class PtyHandler { managed.killTimer = setTimeout(() => { const still = this.ptys.get(id) if (still && !still.disposed) { - killPtyProcess(still.pty, 'SIGKILL') + // Why: on Windows the graceful path already closed ConPTY, which is + // force-capable; a second close can tear down an unrelated conout + // pipe pairing. Keep the cleanup guarantee, skip the redundant kill. + if (!still.windowsImmediateKillIssued) { + killPtyProcess(still.pty, 'SIGKILL') + } this.flushPtyOutput(id) // Why: emit pty.exit BEFORE disposeManagedPty sets disposed=true. // The natural onExit short-circuits on `managed.disposed`, so From 0bf7c87f726e10e262ed9edabbc7009c33bb0750 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:13:01 -0700 Subject: [PATCH 32/36] test: expect isWsl registerPty argument from latest main --- src/main/ipc/pty.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index 3c86612ffe2..96a8b02f37b 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -862,7 +862,8 @@ describe('registerPtyHandlers', () => { expect.any(String), 'wt-renderer', null, - undefined + undefined, + false ) expect(releaseWorktreeSpawn).toHaveBeenCalledTimes(1) }) From 225cd112efe06c0ce67ae24c42fc2de19956d3d3 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:35:37 -0700 Subject: [PATCH 33/36] fix(worktrees): make failed-teardown delete errors user-readable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fail-closed teardown path threw internal pty ids ("Failed to stop local worktree terminals: @@daemon") straight into the delete toast and CLI. Replace both throws with an actionable sentence (retry, or close the terminal) and log the pty ids for triage instead. Also strip Electron's "Error invoking remote method '…': Error:" IPC wrapper in the renderer delete path so the toast shows the message, not transport plumbing. --- src/main/ipc/worktrees.test.ts | 15 +++++++++---- src/main/runtime/orca-runtime.test.ts | 6 +++--- src/main/runtime/worktree-teardown.test.ts | 8 ++++--- src/main/runtime/worktree-teardown.ts | 18 +++++++++++++--- .../src/store/slices/store-cascades.test.ts | 21 ++++++++++++------- src/renderer/src/store/slices/worktrees.ts | 5 ++++- 6 files changed, 51 insertions(+), 22 deletions(-) diff --git a/src/main/ipc/worktrees.test.ts b/src/main/ipc/worktrees.test.ts index 21977909b33..d2e801ce56b 100644 --- a/src/main/ipc/worktrees.test.ts +++ b/src/main/ipc/worktrees.test.ts @@ -6768,11 +6768,13 @@ describe('registerWorktreeHandlers', () => { mockKnownFeatureWorktree() store.getWorktreeMeta.mockReturnValue(makeWorktreeMeta()) killAllProcessesForWorktreeMock.mockRejectedValue( - new Error(`Failed to stop local worktree terminals: ${worktreeId}@@daemon`) + new Error( + "This workspace still has a terminal running that Orca couldn't stop. Try again, or close the terminal and retry." + ) ) await expect(handlers['worktrees:remove'](null, { worktreeId })).rejects.toThrow( - 'Failed to stop local worktree terminals' + "This workspace still has a terminal running that Orca couldn't stop. Try again, or close the terminal and retry." ) expect(removeWorktreeMock).not.toHaveBeenCalled() @@ -8766,7 +8768,9 @@ describe('registerWorktreeHandlers', () => { connectionId: 'removed-ssh-target' }) killAllProcessesForWorktreeMock.mockRejectedValue( - new Error('Failed to stop remote worktree terminals') + new Error( + "Orca couldn't stop this workspace's terminals on the remote host. Reconnect the host and try again." + ) ) const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) @@ -8775,7 +8779,10 @@ describe('registerWorktreeHandlers', () => { expect(warn).toHaveBeenCalledWith( `[worktree-teardown] forget-local failed for ${worktreeId}:`, - expect.objectContaining({ message: 'Failed to stop remote worktree terminals' }) + expect.objectContaining({ + message: + "Orca couldn't stop this workspace's terminals on the remote host. Reconnect the host and try again." + }) ) expect(store.removeWorktreeMeta).toHaveBeenCalledWith(worktreeId) expect(deleteWorktreeHistoryDirMock).toHaveBeenCalledWith(worktreeId) diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 8717c869c2b..0592e0beade 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -30626,7 +30626,7 @@ describe('OrcaRuntimeService', () => { }) await expect(runtime.removeManagedWorktree(TEST_WORKTREE_ID)).rejects.toThrow( - `Failed to stop local worktree terminals: ${ptyId}` + "This workspace still has a terminal running that Orca couldn't stop. Try again, or close the terminal and retry." ) expect(removeWorktree).not.toHaveBeenCalled() @@ -30867,7 +30867,7 @@ describe('OrcaRuntimeService', () => { try { await expect(runtime.removeManagedWorktree(remoteWorktreeId)).rejects.toThrow( - 'Failed to stop remote worktree terminals: ssh:ssh-1@@survivor' + "Orca couldn't stop this workspace's terminals on the remote host. Reconnect the host and try again." ) expect(gitProvider.removeWorktree).not.toHaveBeenCalled() } finally { @@ -30936,7 +30936,7 @@ describe('OrcaRuntimeService', () => { try { await expect(runtime.removeManagedWorktree(remoteWorktreeId)).rejects.toThrow( - 'Failed to stop remote worktree terminals: ssh:ssh-1@@detached-survivor' + "Orca couldn't stop this workspace's terminals on the remote host. Reconnect the host and try again." ) expect(stopAndWait).toHaveBeenCalledWith('ssh:ssh-1@@detached-survivor') expect(gitProvider.removeWorktree).not.toHaveBeenCalled() diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index 8748860e3f0..dd89f6c55fd 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -188,7 +188,7 @@ describe('killAllProcessesForWorktree', () => { listRegisteredPtysMock.mockReturnValue([]) await expect(killAllProcessesForWorktree('w1', { runtime, localProvider })).rejects.toThrow( - 'Failed to stop remote worktree terminals: ssh:ssh-1@@relay-pty' + "Orca couldn't stop this workspace's terminals on the remote host. Reconnect the host and try again." ) expect(localProvider.listProcesses).toHaveBeenCalledTimes(1) }) @@ -231,7 +231,7 @@ describe('killAllProcessesForWorktree', () => { listRegisteredPtysMock.mockReturnValue([]) await expect(killAllProcessesForWorktree('w1', { runtime, localProvider })).rejects.toThrow( - 'Failed to stop local worktree terminals: w1@@daemon' + "This workspace still has a terminal running that Orca couldn't stop. Try again, or close the terminal and retry." ) }) @@ -377,7 +377,9 @@ describe('killAllProcessesForWorktree', () => { sessions.length ) - await expect(teardown).rejects.toThrow('Failed to stop local worktree terminals') + await expect(teardown).rejects.toThrow( + "This workspace still has a terminal running that Orca couldn't stop. Try again, or close the terminal and retry." + ) expect(localProvider.listProcesses).toHaveBeenCalledTimes(2) expect(peak).toBe(8) expect(waves).toBe(7) diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index 5761fab9691..f1ad74a429c 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -74,8 +74,14 @@ export async function killAllProcessesForWorktree( if (failedRemotePtyIds.length > 0) { // Why: the local provider cannot prove an SSH-owned PTY dead; remote Git - // removal must not proceed after an unverified exact stop. - throw new Error(`Failed to stop remote worktree terminals: ${failedRemotePtyIds.join(', ')}`) + // removal must not proceed after an unverified exact stop. The pty ids are + // logged for triage; the thrown text is what the user reads in a toast/CLI. + console.warn( + `[worktree-teardown] unverified remote terminals for ${worktreeId}: ${failedRemotePtyIds.join(', ')}` + ) + throw new Error( + "Orca couldn't stop this workspace's terminals on the remote host. Reconnect the host and try again." + ) } return result @@ -125,7 +131,13 @@ async function sweepLocalProvider( const remainingIds = new Set((await provider.listProcesses()).map((session) => session.id)) const unverified = [...failedShutdowns.keys()].filter((ptyId) => remainingIds.has(ptyId)) if (unverified.length > 0) { - throw new Error(`Failed to stop local worktree terminals: ${unverified.join(', ')}`) + // Why: pty ids logged for triage; the thrown text is user-facing. + console.warn( + `[worktree-teardown] unverified local terminals for ${worktreeId}: ${unverified.join(', ')}` + ) + throw new Error( + "This workspace still has a terminal running that Orca couldn't stop. Try again, or close the terminal and retry." + ) } for (const ptyId of failedShutdowns.keys()) { retirePty?.(ptyId) diff --git a/src/renderer/src/store/slices/store-cascades.test.ts b/src/renderer/src/store/slices/store-cascades.test.ts index 30a19084ec4..3a680060ae7 100644 --- a/src/renderer/src/store/slices/store-cascades.test.ts +++ b/src/renderer/src/store/slices/store-cascades.test.ts @@ -364,10 +364,12 @@ describe('removeWorktree cascade', () => { it('offers force delete for Electron-wrapped local dirty preflight errors', async () => { const store = createTestStore() const worktreeId = 'repo1::/workspace/feature-wt' - const error = + const wrapped = "Error invoking remote method 'worktrees:remove': Error: Failed to delete worktree at /workspace/feature-wt. ?? scratch.txt" + // Why: the store now strips Electron's IPC wrapper; assert on the clean body. + const error = 'Failed to delete worktree at /workspace/feature-wt. ?? scratch.txt' - mockApi.worktrees.remove.mockRejectedValueOnce(new Error(error)) + mockApi.worktrees.remove.mockRejectedValueOnce(new Error(wrapped)) seedStore(store, { worktreesByRepo: { @@ -449,10 +451,11 @@ describe('removeWorktree cascade', () => { it('offers force delete when Git already removed an unregistered worktree', async () => { const store = createTestStore() const worktreeId = 'repo1::/workspace/deleted-wt' - const error = + const wrapped = "Error invoking remote method 'worktrees:remove': Error: Worktree is no longer registered with Git and its directory is already gone." + const error = 'Worktree is no longer registered with Git and its directory is already gone.' - mockApi.worktrees.remove.mockRejectedValueOnce(new Error(error)) + mockApi.worktrees.remove.mockRejectedValueOnce(new Error(wrapped)) seedStore(store, { worktreesByRepo: { @@ -564,10 +567,11 @@ describe('removeWorktree cascade', () => { it('does not offer force delete when Electron wraps SSH filesystem provider failures', async () => { const store = createTestStore() const worktreeId = 'repo1::/path/wt1' - const error = + const wrapped = "Error invoking remote method 'worktrees:remove': Error: SSH filesystem provider unavailable" + const error = 'SSH filesystem provider unavailable' - mockApi.worktrees.remove.mockRejectedValueOnce(new Error(error)) + mockApi.worktrees.remove.mockRejectedValueOnce(new Error(wrapped)) seedStore(store, { worktreesByRepo: { @@ -598,7 +602,8 @@ describe('removeWorktree cascade', () => { async (runtimeFailure) => { const store = createTestStore() const worktreeId = 'repo1::/path/wt1' - const error = `Error invoking remote method 'runtime-environments:call': Error: ${runtimeFailure}` + const wrapped = `Error invoking remote method 'runtime-environments:call': Error: ${runtimeFailure}` + const error = runtimeFailure mockApi.runtimeEnvironments.call.mockImplementation((args: { method: string }) => { const compatibility = createCompatibleRuntimeStatusResponseIfNeeded(args) @@ -613,7 +618,7 @@ describe('removeWorktree cascade', () => { _meta: { runtimeId: 'remote-runtime' } }) } - return Promise.reject(new Error(error)) + return Promise.reject(new Error(wrapped)) }) seedStore(store, { diff --git a/src/renderer/src/store/slices/worktrees.ts b/src/renderer/src/store/slices/worktrees.ts index 1f15f2217d5..8fb3982655f 100644 --- a/src/renderer/src/store/slices/worktrees.ts +++ b/src/renderer/src/store/slices/worktrees.ts @@ -27,6 +27,7 @@ import { } from './worktree-helpers' import { findRepoForHost } from './repo-host-identity' import { ensureHooksConfirmed } from '@/lib/ensure-hooks-confirmed' +import { extractIpcErrorMessage } from '@/lib/ipc-error' import { cleanupEphemeralVmRuntimesForDeleted } from '@/lib/ephemeral-vm-runtime-cleanup' import { tabHasLivePty } from '@/lib/tab-has-live-pty' import { disposeRemovedWorktreeParkedTerminalWatchers } from '../../components/terminal-pane/terminal-parked-watcher-registry' @@ -3635,7 +3636,9 @@ export const createWorktreeSlice: StateCreator // Why: git refusing a non-force delete for dirty/untracked files is a // handled user decision point surfaced by the delete toast, not an app error. console.warn('Failed to remove worktree:', err) - const error = err instanceof Error ? err.message : String(err) + // Why: strip Electron's "Error invoking remote method '…': Error:" wrapper + // so the delete toast shows the main-process message, not IPC plumbing. + const error = extractIpcErrorMessage(err, String(err)) const forceDeleteReason = classifyWorktreeForceDeleteReason(error, force) const locked = isLockedWorktreeRemovalError(error) set((s) => ({ From 8a745733d9ad15b723d9ed5be6500692dbd14948 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:16:06 -0700 Subject: [PATCH 34/36] fix(terminals): keep worktree teardown fail-closed --- config/reliability-gates.jsonc | 124 +++++++++ src/main/ipc/worktrees.test.ts | 16 +- src/main/ipc/worktrees.ts | 5 +- src/main/runtime/orca-runtime.test.ts | 14 ++ src/main/runtime/orca-runtime.ts | 8 +- src/main/runtime/worktree-teardown.test.ts | 17 ++ src/main/runtime/worktree-teardown.ts | 6 + src/relay/pty-handler.test.ts | 40 ++- src/relay/pty-handler.ts | 39 +-- src/relay/pty-session-exit-verification.ts | 126 ++++++++++ src/relay/pty-session-kill.test.ts | 198 ++++++++++++++- src/relay/pty-session-kill.ts | 277 ++++++++------------- src/relay/pty-session-membership.ts | 151 +++++++++++ 13 files changed, 819 insertions(+), 202 deletions(-) create mode 100644 src/relay/pty-session-exit-verification.ts create mode 100644 src/relay/pty-session-membership.ts diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 9965e738a8a..edec03e3ee4 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -550,6 +550,130 @@ ], "demotionRule": "Demote or quarantine if the gate flakes once without a product bug or harness bug filed to the owner." }, + { + "id": "terminal-session.worktree-removal-isolation", + "title": "Worktree removal verifies target PTYs without disturbing unrelated sessions", + "maturity": "experimental", + "protection": "partial", + "owner": "terminal-runtime", + "layer": "main-daemon-relay-contract", + "surfaces": [ + "destructive worktree removal", + "terminal spawn admission", + "daemon and relay PTY teardown", + "local filesystem watcher ownership" + ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "ssh"], + "coverageNotes": "Deterministic tests force Linux, Darwin, and Windows teardown semantics and cover local, current/legacy daemon, degraded-daemon, current/legacy SSH relay, disconnected SSH, runtime/headless, desktop IPC, duplicate removal, and watcher ordering. A local macOS live PTY harness verifies victim-tree death and unrelated-witness survival. Exact-head physical Windows, WSL, and live SSH runs remain explicit gaps.", + "motivatingLinks": [ + "https://github.com/stablyai/orca/issues/8275", + "https://github.com/stablyai/orca/pull/8284" + ], + "invariant": "Once worktree removal owns terminal-session.worktree-removal-isolation, no PTY for that host/worktree may enter or finish registration; every already-admitted target PTY must be stopped with full-session evidence before Git, recursive filesystem, registry, or metadata removal; failed or ambiguous teardown preserves the worktree, its watcher, and every unrelated PTY/daemon/relay session.", + "oracle": "Race renderer/runtime spawns, duplicate remove/forget calls, and bounded target shutdowns across local, daemon, and SSH ownership; require Git and metadata mutation to begin only after admission drains and every exact stop settles; inject legacy protocols, disconnects, malformed process inventories, PID/TTY ownership changes, ConPTY exit timeouts, graceful-to-immediate overlap, and retry; then assert deletion fails closed, the target owner remains retryable, no second ConPTY close occurs, and unrelated sessions are never signaled or retired.", + "commands": [ + "pnpm exec vitest run --config config/vitest.config.ts src/relay/pty-session-kill.test.ts src/relay/pty-handler.test.ts src/main/runtime/worktree-pty-admission.test.ts src/main/runtime/pty-stop-concurrency.test.ts src/main/runtime/worktree-teardown.test.ts src/main/providers/local-pty-provider.test.ts src/main/providers/ssh-pty-liveness.test.ts src/main/providers/ssh-pty-provider.test.ts src/main/daemon/pty-subprocess.test.ts src/main/daemon/session.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts src/main/ipc/pty.test.ts src/main/ipc/worktrees.test.ts" + ], + "testFiles": [ + "src/relay/pty-session-kill.test.ts", + "src/relay/pty-handler.test.ts", + "src/main/runtime/worktree-pty-admission.test.ts", + "src/main/runtime/pty-stop-concurrency.test.ts", + "src/main/runtime/worktree-teardown.test.ts", + "src/main/providers/local-pty-provider.test.ts", + "src/main/providers/ssh-pty-liveness.test.ts", + "src/main/providers/ssh-pty-provider.test.ts", + "src/main/daemon/pty-subprocess.test.ts", + "src/main/daemon/session.test.ts", + "src/main/daemon/terminal-host.test.ts", + "src/main/daemon/daemon-pty-router.test.ts", + "src/main/daemon/degraded-daemon-pty-provider.test.ts", + "src/main/ipc/pty.test.ts", + "src/main/ipc/worktrees.test.ts" + ], + "assertionRefs": [ + { + "file": "src/main/runtime/worktree-pty-admission.test.ts", + "assertions": [ + "already-admitted spawns drain before teardown while later spawns reject until mutation settles", + "different worktrees remain independent, duplicate owners reject, and a timed-out drain reopens admission without leaking its waiter" + ] + }, + { + "file": "src/main/runtime/worktree-teardown.test.ts", + "assertions": [ + "runtime, provider, and registry targets are deduplicated while unrelated and cross-host sessions survive", + "unverified local, legacy-daemon, disconnected-SSH, and provider-only sessions block destructive removal", + "50-session success and failure sweeps peak at eight workers and settle every snapshotted stop before returning" + ] + }, + { + "file": "src/relay/pty-session-kill.test.ts", + "assertions": [ + "root PID, TTY, session, child-parent, and final-exit evidence use canonical bounded inventories and fail closed on malformed or changed ownership", + "Linux and Darwin descendant-first teardown captures reparented/session-escaped members without broad process scans", + "a stuck process tree performs at most two signal-0 probes per PID before bounded batched ps polling" + ] + }, + { + "file": "src/relay/pty-handler.test.ts", + "assertions": [ + "Windows immediate teardown waits for native exit and never double-closes ConPTY after graceful or previous immediate shutdown", + "a graceful-fallback ConPTY without an exit event remains an unverified owner across repeated teardown timeouts until native exit arrives" + ] + }, + { + "file": "src/main/ipc/worktrees.test.ts", + "assertions": [ + "verified PTY shutdown precedes local watcher close, Git removal, and metadata cleanup", + "failed verification leaves Git, metadata, and the local watcher untouched", + "identical cross-surface removals join while conflicting remove/forget ownership cannot mutate twice" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-07-14", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/relay/pty-session-kill.test.ts src/relay/pty-handler.test.ts src/main/runtime/worktree-pty-admission.test.ts src/main/runtime/pty-stop-concurrency.test.ts src/main/runtime/worktree-teardown.test.ts src/main/providers/local-pty-provider.test.ts src/main/providers/ssh-pty-liveness.test.ts src/main/providers/ssh-pty-provider.test.ts src/main/daemon/pty-subprocess.test.ts src/main/daemon/session.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts src/main/ipc/pty.test.ts src/main/ipc/worktrees.test.ts", + "result": "passed", + "durationSeconds": 11.56, + "summary": "15 files and 966 tests passed locally, including canonical POSIX ownership parsing, bounded exit-probe counts, ConPTY graceful-fallback retry retention, watcher-after-verification ordering, eight-worker stop bounds, and local/daemon/SSH fail-closed isolation." + } + ], + "runtimeBudget": { + "p95Seconds": 30, + "scope": "focused main/daemon/provider/relay worktree-removal contract tests" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "The focused slice passes locally but has not accumulated exact-head CI or soak history." + }, + "redGreenEvidence": { + "status": "partial", + "evidence": "The new malformed-child-inventory assertions reject the prior silent token filtering, and the graceful-fallback retry assertion rejects the prior deletion of an unverified ConPTY owner. Saved intentional-break or CI red artifacts remain pending." + }, + "performanceBudget": { + "required": true, + "evidence": "Destructive teardown adds no polling or scans to typing, output, focus, startup, or ordinary render paths. Fifty snapshotted PTYs use exactly fifty stops with an eight-worker ceiling; current SSH uses one targeted liveness request per PTY while a legacy relay shares one bounded inventory; POSIX discovery caps the tree at 1,024 PIDs under one 2.5-second deadline, final verification at 500 ms, PID batches at 64, and signal-0 fast probes at two per PID before batched ps polling. A final-source local victim-root-plus-child teardown completed in 99 ms against the 8,000 ms total budget while an unrelated witness PTY stayed alive." + }, + "promotionCriteria": [ + "Run the focused gate for at least 100 consecutive passes or 14 days across required CI platforms.", + "Attach exact-head physical Windows evidence for repeated target removals, ConPTY timeout/retry, stable daemon PID, and unrelated split-pane survival.", + "Attach live Linux SSH evidence for foreground/background target groups, disconnected fail-closed behavior, and unrelated relay witness survival.", + "Exercise native WSL worktree removal and retain explicit provider identity/path coverage." + ], + "knownGaps": [ + "Exact-head physical Windows and native WSL evidence is not available from this macOS review run.", + "SSH and remote-runtime behavior is deterministic contract coverage here; the retained live SSH evidence predates the final review fixes.", + "Daemon-crash restoration/reporting remains defense in depth outside this removal-path root-cause fix." + ], + "demotionRule": "Keep experimental or demote if the gate flakes without a product/harness issue, if any destructive mutation precedes verified stop settlement, if an unverified owner disappears across retry, if stop concurrency exceeds eight, or if an unrelated PTY receives a shutdown or exit transition." + }, { "id": "terminal-session.kill-all-surface-cleanup", "title": "Kill all sessions removes only the confirmed terminal surfaces and current bindings", diff --git a/src/main/ipc/worktrees.test.ts b/src/main/ipc/worktrees.test.ts index d2e801ce56b..7fc1661ad32 100644 --- a/src/main/ipc/worktrees.test.ts +++ b/src/main/ipc/worktrees.test.ts @@ -9,6 +9,7 @@ import * as localWorktreeFilesystem from '../local-worktree-filesystem' const ORIGINAL_PLATFORM = process.platform const removeWorktreeLinkedPathsMock = vi.hoisted(() => vi.fn()) +const closeLocalWatcherForWorktreePathMock = vi.hoisted(() => vi.fn()) function setPlatform(platform: NodeJS.Platform): void { Object.defineProperty(process, 'platform', { @@ -178,6 +179,10 @@ vi.mock('./worktree-symlinks', () => ({ removeWorktreeLinkedPaths: removeWorktreeLinkedPathsMock })) +vi.mock('./filesystem-watcher', () => ({ + closeLocalWatcherForWorktreePath: closeLocalWatcherForWorktreePathMock +})) + vi.mock('./ssh', () => ({ getActiveMultiplexer: getActiveMultiplexerMock })) @@ -356,7 +361,8 @@ describe('registerWorktreeHandlers', () => { getLocalPtyProviderMock, deleteWorktreeHistoryDirMock, advertisedUrlWatcherForgetWorktreeMock, - removeWorktreeLinkedPathsMock + removeWorktreeLinkedPathsMock, + closeLocalWatcherForWorktreePathMock ]) { m.mockReset() } @@ -367,6 +373,7 @@ describe('registerWorktreeHandlers', () => { }) assertWorktreeCleanForRemovalMock.mockResolvedValue(undefined) getLocalPtyProviderMock.mockReturnValue({} as never) + closeLocalWatcherForWorktreePathMock.mockResolvedValue(undefined) for (const key of Object.keys(handlers)) { delete handlers[key] @@ -6779,6 +6786,7 @@ describe('registerWorktreeHandlers', () => { expect(removeWorktreeMock).not.toHaveBeenCalled() expect(store.removeWorktreeMeta).not.toHaveBeenCalled() + expect(closeLocalWatcherForWorktreePathMock).not.toHaveBeenCalled() }) it('holds PTY admission until Git and metadata removal both complete', async () => { @@ -6791,6 +6799,12 @@ describe('registerWorktreeHandlers', () => { await handlers['worktrees:remove'](null, { worktreeId }) + expect(killAllProcessesForWorktreeMock.mock.invocationCallOrder[0]).toBeLessThan( + closeLocalWatcherForWorktreePathMock.mock.invocationCallOrder[0] + ) + expect(closeLocalWatcherForWorktreePathMock.mock.invocationCallOrder[0]).toBeLessThan( + removeWorktreeMock.mock.invocationCallOrder[0] + ) expect(removeWorktreeMock.mock.invocationCallOrder[0]).toBeLessThan( store.removeWorktreeMeta.mock.invocationCallOrder[0] ) diff --git a/src/main/ipc/worktrees.ts b/src/main/ipc/worktrees.ts index 7b35a07765a..2592c115387 100644 --- a/src/main/ipc/worktrees.ts +++ b/src/main/ipc/worktrees.ts @@ -1866,8 +1866,6 @@ export function registerWorktreeHandlers( ) let removalResult: RemoveWorktreeResult | undefined try { - await closeLocalWatcherForRemoval(canonicalWorktreePath) - await killAllProcessesForWorktree(args.worktreeId, { runtime, localProvider: getLocalPtyProvider(), @@ -1881,6 +1879,9 @@ export function registerWorktreeHandlers( ) } }) + // Why: a failed verified teardown leaves this workspace usable. Keep + // its watcher alive until terminals are proven stopped for deletion. + await closeLocalWatcherForRemoval(canonicalWorktreePath) try { const removeOptions = { diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 0592e0beade..6845dd45a22 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -97,12 +97,17 @@ const ORIGINAL_PLATFORM = process.platform const ORIGINAL_PLATFORM_DESCRIPTOR = Object.getOwnPropertyDescriptor(process, 'platform') const removeWorktreeLinkedPathsMock = vi.hoisted(() => vi.fn()) const resolveLocalGitUsernameMock = vi.hoisted(() => vi.fn(async () => '')) +const closeLocalWatcherForWorktreePathMock = vi.hoisted(() => vi.fn()) vi.mock('../ipc/worktree-symlinks', () => ({ createWorktreeLinkedPaths: vi.fn(), removeWorktreeLinkedPaths: removeWorktreeLinkedPathsMock })) +vi.mock('../ipc/filesystem-watcher', () => ({ + closeLocalWatcherForWorktreePath: closeLocalWatcherForWorktreePathMock +})) + async function waitForMobileSessionTabsEvents( events: RuntimeMobileSessionTabsResult[], count: number @@ -565,6 +570,8 @@ function resetRuntimeTestMocks(): void { vi.mocked(assertWorktreeCleanForRemoval).mockResolvedValue(undefined) vi.mocked(removeWorktree).mockReset() removeWorktreeLinkedPathsMock.mockReset() + closeLocalWatcherForWorktreePathMock.mockReset() + closeLocalWatcherForWorktreePathMock.mockResolvedValue(undefined) resolveLocalGitUsernameMock.mockReset().mockResolvedValue('') vi.mocked(forceDeleteLocalBranchMock).mockReset() vi.mocked(forceDeleteLocalBranchMock).mockResolvedValue(undefined) @@ -30613,6 +30620,12 @@ describe('OrcaRuntimeService', () => { expect(killIdx).toBeGreaterThan(preflightIdx) expect(killIdx).toBeGreaterThanOrEqual(0) expect(gitIdx).toBeGreaterThan(killIdx) + expect(killSpy.mock.invocationCallOrder[0]).toBeLessThan( + closeLocalWatcherForWorktreePathMock.mock.invocationCallOrder[0] + ) + expect(closeLocalWatcherForWorktreePathMock.mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(removeWorktree).mock.invocationCallOrder[0] + ) }) it('does not invoke Git removal when local provider teardown remains unverified', async () => { @@ -30630,6 +30643,7 @@ describe('OrcaRuntimeService', () => { ) expect(removeWorktree).not.toHaveBeenCalled() + expect(closeLocalWatcherForWorktreePathMock).not.toHaveBeenCalled() }) it('keeps spawn admission closed through post-Git metadata cleanup', async () => { diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index bf58fb6b772..1db1adf206b 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -17558,9 +17558,6 @@ export class OrcaRuntimeService { let removalResult: RemoveWorktreeResult | undefined try { const localProvider = this.getLocalProvider() - await closeLocalWatcherForWorktreePath(canonicalWorktreePath).catch((err) => { - console.warn(`[filesystem-watcher] failed to close ${canonicalWorktreePath}:`, err) - }) if (localProvider) { // Why: kill PTYs before git-level removal so Windows handles cannot // keep the directory busy and deleted-cwd shells cannot survive recovery. @@ -17583,6 +17580,11 @@ export class OrcaRuntimeService { } }) } + // Why: a failed verified teardown leaves this workspace usable. Keep + // its watcher alive until terminals are proven stopped for deletion. + await closeLocalWatcherForWorktreePath(canonicalWorktreePath).catch((err) => { + console.warn(`[filesystem-watcher] failed to close ${canonicalWorktreePath}:`, err) + }) try { const removeOptions = { diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index dd89f6c55fd..007803cc5de 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -122,6 +122,23 @@ describe('killAllProcessesForWorktree', () => { await expect(killAllProcessesForWorktree('w1', { localProvider })).rejects.toThrow('boom') }) + it('preserves provider-only sessions when their owner cannot verify full teardown', async () => { + const ptyId = 'w1@@legacy-daemon' + const localProvider = createProviderStub(async () => [ + { id: ptyId, cwd: '/tmp/w1', title: 'shell' } + ]) + localProvider.canVerifyFullSessionTeardown = vi.fn(() => false) + listRegisteredPtysMock.mockReturnValue([]) + + await expect(killAllProcessesForWorktree('w1', { localProvider })).rejects.toThrow( + "This workspace still has a terminal running that Orca couldn't stop. Try again, or close the terminal and retry." + ) + + expect(localProvider.canVerifyFullSessionTeardown).toHaveBeenCalledWith(ptyId) + expect(localProvider.shutdown).not.toHaveBeenCalled() + expect(localProvider.listProcesses).toHaveBeenCalledTimes(2) + }) + it('does not let cleanup hook failures abort teardown', async () => { const localProvider = createProviderStub(async () => [ { id: 'w1@@aaaa', cwd: '/tmp/w1', title: 'shell' } diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index f1ad74a429c..c9b725d1d67 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -116,6 +116,12 @@ async function sweepLocalProvider( const stopped = new Set() await mapPtyStopsWithConcurrency([...targets.keys()], async (ptyId) => { try { + if (provider.canVerifyFullSessionTeardown?.(ptyId) === false) { + // Why: provider-only sessions bypass runtime.stopAndWait. Preserve a + // legacy daemon PTY whose leader-only acknowledgement cannot prove safety. + failedShutdowns.set(ptyId, new Error('Provider cannot verify full PTY teardown')) + return + } await provider.shutdown(ptyId, { immediate: true }) stopped.add(ptyId) retirePty?.(ptyId) diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index 45e1ecc455b..64adcca750c 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -1239,14 +1239,48 @@ describe('PtyHandler', () => { }) }) - it('does not double-close ConPTY on the graceful-shutdown fallback', async () => { + it('retains an unverified ConPTY across graceful-fallback teardown retries', async () => { await withWindowsPlatform(async () => { - const mockKill = mockKillablePty() + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + const mockKill = vi.fn() + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + }) await dispatcher.callRequest('pty.spawn', {}) await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: false }) expectBareKills(mockKill, 1) - vi.advanceTimersByTime(5000) + await vi.advanceTimersByTimeAsync(5000) + + expect(handler.activePtyCount).toBe(1) + const firstRetry = dispatcher.callRequest('pty.shutdown', { + id: 'pty-1', + immediate: true + }) + const firstRejection = expect(firstRetry).rejects.toThrow( + 'Timed out waiting for Windows PTY teardown: pty-1' + ) + await vi.advanceTimersByTimeAsync(RELAY_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS) + await firstRejection + + const secondRetry = dispatcher.callRequest('pty.shutdown', { + id: 'pty-1', + immediate: true + }) + const secondRejection = expect(secondRetry).rejects.toThrow( + 'Timed out waiting for Windows PTY teardown: pty-1' + ) + await vi.advanceTimersByTimeAsync(RELAY_PTY_IMMEDIATE_SHUTDOWN_TIMEOUT_MS) + await secondRejection + expectBareKills(mockKill, 1) + expect(handler.activePtyCount).toBe(1) + onExitCb!({ exitCode: 0 }) + expect(handler.activePtyCount).toBe(0) }) }) diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index 6ab37e1189e..8a27b211c38 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -445,6 +445,10 @@ export class PtyHandler { managed.pty.onExit(({ exitCode }: { exitCode: number }) => { this.resolveExitWaiters(managed.id) if (managed.disposed) { + // Why: a Windows graceful-close fallback retains the disposed owner so + // retries cannot mistake an unverified ConPTY for an absent session. + this.ptys.delete(managed.id) + this.clearPtyFlowState(managed.id) return } // Why: neutralize managed.pty.kill synchronously BEFORE anything else @@ -1027,16 +1031,15 @@ export class PtyHandler { // without this notify the renderer never learns the pane is dead // when the SIGKILL fallback fires for a SIGTERM-ignoring child. this.dispatcher.notify('pty.exit', { id, code: -1 }) - // Why: if SIGKILL's onExit never fires (kernel edge case, - // uninterruptible sleep, child wedged on a bad NFS mount), the - // fd and map entry would leak forever. Dispose synchronously so - // graceful-shutdown's SIGKILL fallback is a hard guarantee, not - // "hopefully onExit will run". The disposed guard inside - // disposeManagedPty makes a later onExit's dispose a no-op. + // Why: POSIX SIGKILL may never emit onExit for a D-state child, so + // dispose and delete it synchronously. Windows must retain the owner + // until native exit proves the already-issued ConPTY close completed. this.notifyExitListener(still) disposeManagedPty(still) - this.ptys.delete(id) - this.clearPtyFlowState(id) + if (process.platform !== 'win32' || !still.windowsImmediateKillIssued) { + this.ptys.delete(id) + this.clearPtyFlowState(id) + } } }, 5000) } @@ -1294,17 +1297,15 @@ export class PtyHandler { managed.killTimer = undefined } this.clearStartupCommandTimer(managed) - // Why: SIGKILL (not SIGTERM) before destroy. The relay process is - // exiting; any SIGTERM-ignoring remote shell (editor with unsaved - // buffers, a hung child with a bad handler, a process in - // uninterruptible sleep) would survive SIGTERM + immediate destroy() - // as an orphan on the remote host. SIGKILL is not ignorable and the - // ptmx fd release via disposeManagedPty is synchronous, so there is - // no graceful-shutdown window to preserve at this point. - try { - killPtyProcess(managed.pty, 'SIGKILL') - } catch { - /* child may already be dead */ + // Why: POSIX relay exit uses SIGKILL before destroy so resistant children + // cannot become orphans. A Windows close is already force-capable; issuing + // it again can corrupt an unrelated ConPTY pipe pairing. + if (process.platform !== 'win32' || !managed.windowsImmediateKillIssued) { + try { + killPtyProcess(managed.pty, 'SIGKILL') + } catch { + /* child may already be dead */ + } } this.notifyExitListener(managed) disposeManagedPty(managed) diff --git a/src/relay/pty-session-exit-verification.ts b/src/relay/pty-session-exit-verification.ts new file mode 100644 index 00000000000..98cf36cf1b9 --- /dev/null +++ b/src/relay/pty-session-exit-verification.ts @@ -0,0 +1,126 @@ +import { setTimeout as wait } from 'node:timers/promises' +import { PTY_SESSION_VERIFY_TIMEOUT_MS } from '../shared/terminal-teardown-timeouts' +import { isEmptyProcessSelection, type PtySessionCommandRunner } from './pty-session-membership' + +export type PtyProcessSignaler = (pid: number, signal: NodeJS.Signals | 0) => void + +const VERIFY_PID_BATCH_SIZE = 64 +const VERIFY_POLL_INITIAL_MS = 10 +const VERIFY_POLL_MAX_MS = 100 + +export async function verifyProcessesStopped( + pids: number[], + run: PtySessionCommandRunner, + killProcess: PtyProcessSignaler +): Promise { + const verificationDeadline = Date.now() + PTY_SESSION_VERIFY_TIMEOUT_MS + let pending = pids + let pollDelayMs = VERIFY_POLL_INITIAL_MS + let yieldedForReaping = false + let useFastAbsenceProbe = true + while (pending.length > 0) { + let present = pending + if (useFastAbsenceProbe) { + present = [] + for (const pid of pending) { + try { + // Why: the common SIGKILL path reaps quickly. Two signal-0 absence + // probes avoid spawning slow BSD ps, then batched ps owns later polls. + killProcess(pid, 0) + present.push(pid) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') { + return false + } + } + } + if (present.length === 0) { + return true + } + } + if (!yieldedForReaping) { + const remainingMs = verificationDeadline - Date.now() + const waitMs = Math.min(VERIFY_POLL_INITIAL_MS, remainingMs - 1) + if (waitMs <= 0) { + return false + } + // Why: node-pty reaps its child on the next event-loop turn. Give that + // callback one chance before invoking comparatively slow BSD ps. + await wait(waitMs) + yieldedForReaping = true + continue + } + useFastAbsenceProbe = false + const nextPending: number[] = [] + for (let index = 0; index < present.length; index += VERIFY_PID_BATCH_SIZE) { + const remainingMs = verificationDeadline - Date.now() + if (remainingMs <= 0) { + return false + } + const batch = present.slice(index, index + VERIFY_PID_BATCH_SIZE) + try { + const result = (await run('ps', ['-p', batch.join(','), '-o', 'pid=', '-o', 'stat='], { + timeout: remainingMs + })) as { stdout?: string | Buffer } + const livePids = readLiveProcessIds(result.stdout, batch) + if (!livePids) { + return false + } + nextPending.push(...livePids) + } catch (error) { + if (!isEmptyProcessSelection(error)) { + return false + } + } + } + pending = nextPending + if (pending.length === 0) { + return true + } + const remainingMs = verificationDeadline - Date.now() + if (remainingMs <= 0) { + return false + } + // Why: signal delivery and process reaping are asynchronous. Back off the + // still-live subset so a stuck tree cannot spawn ps every 10ms at scale. + const waitMs = Math.min(pollDelayMs, remainingMs - 1) + if (waitMs <= 0) { + return false + } + await wait(waitMs) + pollDelayMs = Math.min(pollDelayMs * 2, VERIFY_POLL_MAX_MS) + } + return true +} + +function readLiveProcessIds( + stdout: string | Buffer | undefined, + expectedPids: readonly number[] +): number[] | null { + const expected = new Set(expectedPids) + const seen = new Set() + const livePids: number[] = [] + const lines = String(stdout ?? '') + .trim() + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + for (const line of lines) { + const columns = line.split(/\s+/) + const [pidText, status] = columns + // Why: numeric coercion accepts signs, decimals, and exponents; process + // verification must accept only the canonical PID text emitted by ps. + if (columns.length !== 2 || !pidText || !/^[1-9]\d*$/.test(pidText)) { + return null + } + const pid = Number(pidText) + if (!Number.isSafeInteger(pid) || !expected.has(pid) || seen.has(pid) || !status) { + return null + } + seen.add(pid) + if (!status.startsWith('Z')) { + livePids.push(pid) + } + } + return livePids +} diff --git a/src/relay/pty-session-kill.test.ts b/src/relay/pty-session-kill.test.ts index 6941af92c8c..6544c2ac72c 100644 --- a/src/relay/pty-session-kill.test.ts +++ b/src/relay/pty-session-kill.test.ts @@ -10,12 +10,97 @@ function emptySelection(): Error & { code: number; stdout: string } { } describe('killPosixPtySession', () => { + it('kills a reparented Linux member that no longer appears in the root PPID tree', async () => { + const run = vi.fn(async (file: string, args: string[]) => { + if (file === 'ps' && args[1] === '4242' && args.includes('sid=')) { + return { stdout: '4242 pts/7\n' } + } + if (file === 'pgrep' && args[0] === '-s') { + return { stdout: '4242\n4243\n' } + } + if (file === 'pgrep' && args[0] === '-P') { + throw emptySelection() + } + if (file === 'ps' && args.includes('sid=') && args.includes('tty=')) { + return { stdout: '4243 4242 pts/7\n' } + } + if (file === 'ps' && args.includes('stat=')) { + return { stdout: '4242 Z\n4243 Z\n' } + } + throw new Error(`Unexpected command: ${file} ${args.join(' ')}`) + }) + const killProcess = vi.fn() + + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + true + ) + + expect(killProcess).toHaveBeenCalledWith(4243, 'SIGSTOP') + expect(killProcess).toHaveBeenCalledWith(4243, 'SIGKILL') + }) + + it('kills a reparented Darwin member selected by its exact controlling TTY', async () => { + const run = vi.fn(async (file: string, args: string[]) => { + if (file === 'ps' && args[1] === '4242' && args.includes('pgid=')) { + return { stdout: '4242 ttys042\n' } + } + if (file === 'pgrep' && args[0] === '-P') { + throw emptySelection() + } + if (file === 'ps' && args[0] === '-t') { + return { stdout: '4242\n4243\n' } + } + if (file === 'ps' && args.includes('tty=') && !args.includes('pgid=')) { + return { stdout: '4243 ttys042\n' } + } + if (file === 'ps' && args.includes('stat=')) { + throw emptySelection() + } + throw new Error(`Unexpected command: ${file} ${args.join(' ')}`) + }) + const killProcess = vi.fn() + + await expect( + killPosixPtySession(4242, '/dev/ttys042', 'darwin', run, killProcess) + ).resolves.toBe(true) + + expect(killProcess).toHaveBeenCalledWith(4243, 'SIGSTOP') + expect(killProcess).toHaveBeenCalledWith(4243, 'SIGKILL') + }) + + it('fails closed when the targeted membership inventory omits the frozen root', async () => { + const run = vi.fn(async (file: string, args: string[]) => { + if (file === 'ps' && args.includes('sid=')) { + return { stdout: '4242 pts/7\n' } + } + if (file === 'pgrep' && args[0] === '-P') { + throw emptySelection() + } + if (file === 'pgrep' && args[0] === '-s') { + return { stdout: '4243\n' } + } + throw new Error(`Unexpected command: ${file} ${args.join(' ')}`) + }) + const killProcess = vi.fn() + + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + false + ) + expect(killProcess.mock.calls.filter(([, signal]) => signal !== 0)).toEqual([ + [4242, 'SIGSTOP'], + [4242, 'SIGCONT'] + ]) + }) + it('freezes and kills a Linux descendant that escaped into a new session', async () => { const run = vi.fn(async (file: string, args: string[]) => { if (file === 'ps' && args.includes('sid=')) { return { stdout: '4242 pts/7\n' } } if (file === 'pgrep') { + if (args[0] === '-s') { + return { stdout: '4242\n4243\n4244\n4245\n' } + } const parents = new Set(args[1]?.split(',')) if (parents.has('4242')) { return { stdout: '4243\n4244\n' } @@ -45,7 +130,7 @@ describe('killPosixPtySession', () => { true ) - expect(killProcess.mock.calls).toEqual([ + expect(killProcess.mock.calls.filter(([, signal]) => signal !== 0)).toEqual([ [4242, 'SIGSTOP'], [4243, 'SIGSTOP'], [4244, 'SIGSTOP'], @@ -56,7 +141,7 @@ describe('killPosixPtySession', () => { [4242, 'SIGKILL'] ]) expect(run).not.toHaveBeenCalledWith('ps', expect.arrayContaining(['-e']), expect.anything()) - expect(run.mock.calls.filter(([file]) => file === 'pgrep')).toHaveLength(3) + expect(run.mock.calls.filter(([file]) => file === 'pgrep')).toHaveLength(4) }) it('validates Darwin root ownership before targeted descendant teardown', async () => { @@ -70,6 +155,9 @@ describe('killPosixPtySession', () => { } throw emptySelection() } + if (file === 'ps' && args[0] === '-t') { + return { stdout: '4242\n4243\n' } + } if (file === 'ps' && args.includes('ppid=')) { return { stdout: '4243 4242\n' } } @@ -83,7 +171,7 @@ describe('killPosixPtySession', () => { await expect( killPosixPtySession(4242, '/dev/ttys042', 'darwin', run, killProcess) ).resolves.toBe(true) - expect(run).toHaveBeenCalledTimes(6) + expect(run).toHaveBeenCalledTimes(7) expect(run).toHaveBeenNthCalledWith(1, 'ps', ['-p', '4242', '-o', 'pgid=', '-o', 'tty='], { timeout: PTY_SESSION_COMMAND_TIMEOUT_MS }) @@ -93,7 +181,7 @@ describe('killPosixPtySession', () => { expect(run).toHaveBeenLastCalledWith('ps', ['-p', '4242,4243', '-o', 'pid=', '-o', 'stat='], { timeout: expect.any(Number) }) - expect(killProcess.mock.calls).toEqual([ + expect(killProcess.mock.calls.filter(([, signal]) => signal !== 0)).toEqual([ [4242, 'SIGSTOP'], [4243, 'SIGSTOP'], [4243, 'SIGKILL'], @@ -118,6 +206,16 @@ describe('killPosixPtySession', () => { expect(killProcess).not.toHaveBeenCalled() }) + it('does not signal a root whose ownership output uses a non-canonical PID', async () => { + const run = vi.fn().mockResolvedValue({ stdout: '+4242 pts/7\n' }) + const killProcess = vi.fn() + + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + false + ) + expect(killProcess).not.toHaveBeenCalled() + }) + it('does not signal a reused Linux session leader on a different TTY', async () => { const run = vi.fn().mockResolvedValue({ stdout: '4242 pts/99\n' }) const killProcess = vi.fn() @@ -161,7 +259,7 @@ describe('killPosixPtySession', () => { .mockResolvedValueOnce({ stdout: '4242 pts/7\n' }) .mockResolvedValueOnce({ stdout: '4242 pts/7\n' }) .mockResolvedValueOnce({ stdout: '4243\n' }) - const killProcess = vi.fn((pid: number, signal: NodeJS.Signals) => { + const killProcess = vi.fn((pid: number, signal: NodeJS.Signals | 0) => { if (pid === 4243 && signal === 'SIGSTOP') { throw Object.assign(new Error('gone'), { code: 'ESRCH' }) } @@ -194,10 +292,56 @@ describe('killPosixPtySession', () => { ]) }) + it('fails closed when descendant discovery returns a malformed PID token', async () => { + const run = vi + .fn() + .mockResolvedValueOnce({ stdout: '4242 pts/7\n' }) + .mockResolvedValueOnce({ stdout: '4242 pts/7\n' }) + .mockResolvedValueOnce({ stdout: 'not-a-pid\n' }) + const killProcess = vi.fn() + + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + false + ) + expect(killProcess.mock.calls).toEqual([ + [4242, 'SIGSTOP'], + [4242, 'SIGCONT'] + ]) + }) + + it('resumes a frozen child when parent verification has extra columns', async () => { + const run = vi.fn(async (file: string, args: string[]) => { + if (file === 'ps' && args.includes('sid=')) { + return { stdout: '4242 pts/7\n' } + } + if (file === 'pgrep') { + return { stdout: '4243\n' } + } + if (file === 'ps' && args.includes('ppid=')) { + return { stdout: '4243 4242 unexpected\n' } + } + throw new Error(`Unexpected command: ${file} ${args.join(' ')}`) + }) + const killProcess = vi.fn() + + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + false + ) + expect(killProcess.mock.calls).toEqual([ + [4242, 'SIGSTOP'], + [4243, 'SIGSTOP'], + [4242, 'SIGCONT'], + [4243, 'SIGCONT'] + ]) + }) + it('fails closed when a captured process remains runnable after SIGKILL', async () => { let verificationChecks = 0 const run = vi.fn(async (file: string, args: string[]) => { if (file === 'pgrep') { + if (args[0] === '-s') { + return { stdout: '4242\n' } + } throw emptySelection() } if (file === 'ps' && args.includes('sid=')) { @@ -214,12 +358,16 @@ describe('killPosixPtySession', () => { // Initial 10ms backoff doubling to 100ms permits at most nine ps rounds // inside the shared 500ms verification deadline. expect(verificationChecks).toBeLessThanOrEqual(9) + expect(killProcess.mock.calls.filter(([, signal]) => signal === 0)).toHaveLength(2) }) it('waits for a captured process to finish exiting after SIGKILL', async () => { let verificationChecks = 0 const run = vi.fn(async (file: string, args: string[]) => { if (file === 'pgrep') { + if (args[0] === '-s') { + return { stdout: '4242\n' } + } throw emptySelection() } if (file === 'ps' && args.includes('sid=')) { @@ -239,6 +387,33 @@ describe('killPosixPtySession', () => { expect(verificationChecks).toBe(2) }) + it('lets node-pty reap killed processes before invoking final ps verification', async () => { + const run = vi.fn(async (file: string, args: string[]) => { + if (file === 'pgrep' && args[0] === '-P') { + throw emptySelection() + } + if (file === 'pgrep' && args[0] === '-s') { + return { stdout: '4242\n' } + } + if (file === 'ps' && args.includes('sid=')) { + return { stdout: '4242 pts/7\n' } + } + throw new Error(`Unexpected command: ${file} ${args.join(' ')}`) + }) + let absenceProbes = 0 + const killProcess = vi.fn((_pid: number, signal: NodeJS.Signals | 0) => { + if (signal === 0 && ++absenceProbes > 1) { + throw Object.assign(new Error('gone'), { code: 'ESRCH' }) + } + }) + + await expect(killPosixPtySession(4242, '/dev/pts/7', 'linux', run, killProcess)).resolves.toBe( + true + ) + expect(absenceProbes).toBe(2) + expect(run).not.toHaveBeenCalledWith('ps', expect.arrayContaining(['stat=']), expect.anything()) + }) + it('fails closed when verification returns a non-canonical PID token', async () => { const run = vi.fn(async (file: string, args: string[]) => { if (file === 'pgrep') { @@ -261,6 +436,9 @@ describe('killPosixPtySession', () => { it('uses bounded targeted command timeouts for ownership, children, and verification', async () => { const run = vi.fn(async (file: string, args?: string[], _options?: { timeout: number }) => { if (file === 'pgrep') { + if (args?.[0] === '-s') { + return { stdout: '4242\n' } + } throw emptySelection() } if (args?.includes('sid=')) { @@ -280,10 +458,13 @@ describe('killPosixPtySession', () => { timeout: expect.any(Number) }) expect(run.mock.calls[2]?.[2]?.timeout).toBeLessThanOrEqual(PTY_SESSION_COMMAND_TIMEOUT_MS) - expect(run).toHaveBeenNthCalledWith(4, 'ps', ['-p', '4242', '-o', 'pid=', '-o', 'stat='], { + expect(run).toHaveBeenNthCalledWith(4, 'pgrep', ['-s', '4242'], { + timeout: expect.any(Number) + }) + expect(run).toHaveBeenNthCalledWith(5, 'ps', ['-p', '4242', '-o', 'pid=', '-o', 'stat='], { timeout: expect.any(Number) }) - expect(run.mock.calls[3]?.[2]?.timeout).toBeLessThanOrEqual(PTY_SESSION_VERIFY_TIMEOUT_MS) + expect(run.mock.calls[4]?.[2]?.timeout).toBeLessThanOrEqual(PTY_SESSION_VERIFY_TIMEOUT_MS) }) it('resumes every frozen process when a stopped child was reparented', async () => { @@ -331,6 +512,9 @@ describe('killPosixPtySession', () => { } throw emptySelection() } + if (file === 'ps' && args[0] === '-t') { + return { stdout: [rootPid, ...childPids].join('\n') } + } if (file === 'ps' && args.includes('ppid=')) { return { stdout: args[1]! diff --git a/src/relay/pty-session-kill.ts b/src/relay/pty-session-kill.ts index bc56afd14bc..659db174a5e 100644 --- a/src/relay/pty-session-kill.ts +++ b/src/relay/pty-session-kill.ts @@ -1,24 +1,28 @@ import { execFile as execFileCallback } from 'node:child_process' -import { setTimeout as wait } from 'node:timers/promises' import { promisify } from 'node:util' +import { PTY_SESSION_COMMAND_TIMEOUT_MS } from '../shared/terminal-teardown-timeouts' +import { type PtyProcessSignaler, verifyProcessesStopped } from './pty-session-exit-verification' import { - PTY_SESSION_COMMAND_TIMEOUT_MS, - PTY_SESSION_VERIFY_TIMEOUT_MS -} from '../shared/terminal-teardown-timeouts' + frozenMembersStillOwnPty, + isEmptyProcessSelection, + listPtyMembers, + parseCanonicalProcessId, + parseCanonicalProcessIds, + type PtySessionCommandRunner, + rootStillOwnsPty +} from './pty-session-membership' export { PTY_SESSION_COMMAND_TIMEOUT_MS, PTY_SESSION_VERIFY_TIMEOUT_MS } from '../shared/terminal-teardown-timeouts' -type ExecFile = (file: string, args: string[], options: { timeout: number }) => Promise -type KillProcess = (pid: number, signal: NodeJS.Signals) => void +type ExecFile = PtySessionCommandRunner +type KillProcess = PtyProcessSignaler const execFile = promisify(execFileCallback) as ExecFile export const MAX_PTY_PROCESS_TREE_SIZE = 1024 const VERIFY_PID_BATCH_SIZE = 64 -const VERIFY_POLL_INITIAL_MS = 10 -const VERIFY_POLL_MAX_MS = 100 const PARENT_PID_BATCH_SIZE = 64 export async function killPosixPtySession( @@ -46,7 +50,7 @@ export async function killPosixPtySession( resumeProcesses(stopped, killProcess) return false } - const processTree = await freezeProcessTree(pid, stopped, run, killProcess) + const processTree = await freezeProcessTree(pid, ptsName, platform, stopped, run, killProcess) if (!processTree) { resumeProcesses(stopped, killProcess) return false @@ -60,40 +64,17 @@ export async function killPosixPtySession( } stopped.delete(candidate) } - return await verifyProcessesStopped(processTree, run) + return await verifyProcessesStopped(processTree, run, killProcess) } catch { resumeProcesses(stopped, killProcess) return false } } -async function rootStillOwnsPty( - pid: number, - ptsName: unknown, - platform: NodeJS.Platform, - run: ExecFile -): Promise { - if (typeof ptsName !== 'string') { - // Why: without the exact controlling TTY, a recycled session-leader PID - // is not enough authority to signal an unrelated user's process tree. - return false - } - const ownerColumn = platform === 'darwin' ? 'pgid=' : 'sid=' - const result = (await run('ps', ['-p', String(pid), '-o', ownerColumn, '-o', 'tty='], { - timeout: PTY_SESSION_COMMAND_TIMEOUT_MS - })) as { stdout?: string | Buffer } - const [ownerText, tty = ''] = String(result.stdout ?? '') - .trim() - .split(/\s+/) - if (Number(ownerText) !== pid) { - return false - } - const expectedTty = ptsName.startsWith('/dev/') ? ptsName.slice('/dev/'.length) : ptsName - return /^[A-Za-z0-9._/-]+$/.test(expectedTty) && tty === expectedTty -} - async function freezeProcessTree( rootPid: number, + ptsName: unknown, + platform: NodeJS.Platform, stopped: Set, run: ExecFile, killProcess: KillProcess @@ -102,50 +83,97 @@ async function freezeProcessTree( const seen = new Set(processTree) let frontier = [rootPid] const discoveryDeadline = Date.now() + PTY_SESSION_COMMAND_TIMEOUT_MS - while (frontier.length > 0) { - const nextFrontier: number[] = [] - for (let index = 0; index < frontier.length; index += PARENT_PID_BATCH_SIZE) { - const remainingMs = discoveryDeadline - Date.now() - if (remainingMs <= 0) { - return null - } - const parents = frontier.slice(index, index + PARENT_PID_BATCH_SIZE) - const children = await listChildren(parents, run, remainingMs) - const newlyStopped: number[] = [] - for (const child of children) { - if (seen.has(child)) { - continue - } - if (seen.size >= MAX_PTY_PROCESS_TREE_SIZE) { + while (true) { + while (frontier.length > 0) { + const nextFrontier: number[] = [] + for (let index = 0; index < frontier.length; index += PARENT_PID_BATCH_SIZE) { + const remainingMs = discoveryDeadline - Date.now() + if (remainingMs <= 0) { return null } - seen.add(child) - if (!stopProcess(child, stopped, killProcess)) { + const parents = frontier.slice(index, index + PARENT_PID_BATCH_SIZE) + const children = await listChildren(parents, run, remainingMs) + const newlyStopped: number[] = [] + for (const child of children) { + if (seen.has(child)) { + continue + } + if (seen.size >= MAX_PTY_PROCESS_TREE_SIZE) { + return null + } + seen.add(child) + if (!stopProcess(child, stopped, killProcess)) { + return null + } + newlyStopped.push(child) + } + const childVerificationRemainingMs = discoveryDeadline - Date.now() + if ( + newlyStopped.length > 0 && + (childVerificationRemainingMs <= 0 || + !(await frozenChildrenStillBelongToParents( + newlyStopped, + parents, + run, + childVerificationRemainingMs + ))) + ) { return null } - newlyStopped.push(child) + for (const child of newlyStopped) { + processTree.push(child) + nextFrontier.push(child) + } } - const childVerificationRemainingMs = discoveryDeadline - Date.now() - if ( - newlyStopped.length > 0 && - (childVerificationRemainingMs <= 0 || - !(await frozenChildrenStillBelongToParents( - newlyStopped, - parents, - run, - childVerificationRemainingMs - ))) - ) { + frontier = nextFrontier + } + + const remainingMs = discoveryDeadline - Date.now() + if (remainingMs <= 0) { + return null + } + const members = await listPtyMembers(rootPid, ptsName, platform, run, remainingMs) + if (!members.includes(rootPid)) { + // Why: the frozen, revalidated root must appear in its targeted SID/TTY + // inventory; absence means the selection cannot certify completeness. + return null + } + const newlyStoppedMembers: number[] = [] + for (const member of members) { + if (seen.has(member)) { + continue + } + if (seen.size >= MAX_PTY_PROCESS_TREE_SIZE) { return null } - for (const child of newlyStopped) { - processTree.push(child) - nextFrontier.push(child) + seen.add(member) + if (!stopProcess(member, stopped, killProcess)) { + return null } + newlyStoppedMembers.push(member) } - frontier = nextFrontier + if (newlyStoppedMembers.length === 0) { + return processTree + } + const memberVerificationRemainingMs = discoveryDeadline - Date.now() + if ( + memberVerificationRemainingMs <= 0 || + !(await frozenMembersStillOwnPty( + newlyStoppedMembers, + rootPid, + ptsName, + platform, + run, + memberVerificationRemainingMs + )) + ) { + return null + } + // Why: an intermediate shell can exit before teardown, reparenting a + // background job outside the root PPID tree while it still owns this PTY. + processTree.push(...newlyStoppedMembers) + frontier = newlyStoppedMembers } - return processTree } async function listChildren( @@ -157,11 +185,7 @@ async function listChildren( const result = (await run('pgrep', ['-P', parentPids.join(',')], { timeout })) as { stdout?: string | Buffer } - return String(result.stdout ?? '') - .trim() - .split(/\s+/) - .map(Number) - .filter((candidate) => Number.isSafeInteger(candidate) && candidate > 0) + return parseCanonicalProcessIds(result.stdout) } catch (error) { if (isEmptyProcessSelection(error)) { return [] @@ -191,10 +215,18 @@ async function frozenChildrenStillBelongToParents( for (const line of String(result.stdout ?? '') .trim() .split('\n')) { - const [pidText, parentPidText] = line.trim().split(/\s+/) - const pid = Number(pidText) - const parentPid = Number(parentPidText) - if (!batch.includes(pid) || !expectedParents.has(parentPid) || verifiedChildren.has(pid)) { + const columns = line.trim().split(/\s+/) + const [pidText, parentPidText] = columns + const pid = parseCanonicalProcessId(pidText) + const parentPid = parseCanonicalProcessId(parentPidText) + if ( + columns.length !== 2 || + pid === null || + parentPid === null || + !batch.includes(pid) || + !expectedParents.has(parentPid) || + verifiedChildren.has(pid) + ) { return false } verifiedChildren.add(pid) @@ -228,92 +260,3 @@ function resumeProcesses(stopped: Set, killProcess: KillProcess): void { } stopped.clear() } - -async function verifyProcessesStopped(pids: number[], run: ExecFile): Promise { - const verificationDeadline = Date.now() + PTY_SESSION_VERIFY_TIMEOUT_MS - let pending = pids - let pollDelayMs = VERIFY_POLL_INITIAL_MS - while (pending.length > 0) { - const nextPending: number[] = [] - for (let index = 0; index < pending.length; index += VERIFY_PID_BATCH_SIZE) { - const remainingMs = verificationDeadline - Date.now() - if (remainingMs <= 0) { - return false - } - const batch = pending.slice(index, index + VERIFY_PID_BATCH_SIZE) - try { - const result = (await run('ps', ['-p', batch.join(','), '-o', 'pid=', '-o', 'stat='], { - timeout: remainingMs - })) as { stdout?: string | Buffer } - const livePids = readLiveProcessIds(result.stdout, batch) - if (!livePids) { - return false - } - nextPending.push(...livePids) - } catch (error) { - if (!isEmptyProcessSelection(error)) { - return false - } - } - } - pending = nextPending - if (pending.length === 0) { - return true - } - const remainingMs = verificationDeadline - Date.now() - if (remainingMs <= 0) { - return false - } - // Why: signal delivery and process reaping are asynchronous. Back off the - // still-live subset so a stuck tree cannot spawn ps every 10ms at scale. - const waitMs = Math.min(pollDelayMs, remainingMs - 1) - if (waitMs <= 0) { - return false - } - await wait(waitMs) - pollDelayMs = Math.min(pollDelayMs * 2, VERIFY_POLL_MAX_MS) - } - return true -} - -function readLiveProcessIds( - stdout: string | Buffer | undefined, - expectedPids: readonly number[] -): number[] | null { - const expected = new Set(expectedPids) - const seen = new Set() - const livePids: number[] = [] - const lines = String(stdout ?? '') - .trim() - .split('\n') - .map((line) => line.trim()) - .filter(Boolean) - for (const line of lines) { - const columns = line.split(/\s+/) - const [pidText, status] = columns - // Why: numeric coercion accepts signs, decimals, and exponents; process - // verification must accept only the canonical PID text emitted by ps. - if (columns.length !== 2 || !pidText || !/^[1-9]\d*$/.test(pidText)) { - return null - } - const pid = Number(pidText) - if (!Number.isSafeInteger(pid) || !expected.has(pid) || seen.has(pid) || !status) { - return null - } - seen.add(pid) - if (!status.startsWith('Z')) { - livePids.push(pid) - } - } - return livePids -} - -function isEmptyProcessSelection(error: unknown): boolean { - if (!error || typeof error !== 'object') { - return false - } - const processError = error as { code?: unknown; stdout?: unknown } - // Why: BSD/procps ps/pgrep exit 1 for an empty selector. Only that exact - // empty result proves absence; ENOENT, timeout, and malformed output fail closed. - return processError.code === 1 && String(processError.stdout ?? '').trim() === '' -} diff --git a/src/relay/pty-session-membership.ts b/src/relay/pty-session-membership.ts new file mode 100644 index 00000000000..3594dc38042 --- /dev/null +++ b/src/relay/pty-session-membership.ts @@ -0,0 +1,151 @@ +import { PTY_SESSION_COMMAND_TIMEOUT_MS } from '../shared/terminal-teardown-timeouts' + +export type PtySessionCommandRunner = ( + file: string, + args: string[], + options: { timeout: number } +) => Promise + +const VERIFY_PID_BATCH_SIZE = 64 + +export async function rootStillOwnsPty( + pid: number, + ptsName: unknown, + platform: NodeJS.Platform, + run: PtySessionCommandRunner +): Promise { + const expectedTty = normalizePtyName(ptsName) + if (!expectedTty) { + // Why: without the exact controlling TTY, a recycled session-leader PID + // is not enough authority to signal an unrelated user's process tree. + return false + } + const ownerColumn = platform === 'darwin' ? 'pgid=' : 'sid=' + const result = (await run('ps', ['-p', String(pid), '-o', ownerColumn, '-o', 'tty='], { + timeout: PTY_SESSION_COMMAND_TIMEOUT_MS + })) as { stdout?: string | Buffer } + const columns = String(result.stdout ?? '') + .trim() + .split(/\s+/) + const [ownerText, tty] = columns + if (columns.length !== 2 || parseCanonicalProcessId(ownerText) !== pid) { + return false + } + return tty === expectedTty +} + +export async function listPtyMembers( + rootPid: number, + ptsName: unknown, + platform: NodeJS.Platform, + run: PtySessionCommandRunner, + timeout: number +): Promise { + const expectedTty = normalizePtyName(ptsName) + if (!expectedTty) { + throw new Error('Exact PTY name unavailable') + } + const file = platform === 'linux' ? 'pgrep' : 'ps' + const args = platform === 'linux' ? ['-s', String(rootPid)] : ['-t', expectedTty, '-o', 'pid='] + try { + const result = (await run(file, args, { timeout })) as { stdout?: string | Buffer } + return parseCanonicalProcessIds(result.stdout) + } catch (error) { + if (isEmptyProcessSelection(error)) { + return [] + } + throw error + } +} + +export async function frozenMembersStillOwnPty( + memberPids: readonly number[], + rootPid: number, + ptsName: unknown, + platform: NodeJS.Platform, + run: PtySessionCommandRunner, + timeout: number +): Promise { + const expectedTty = normalizePtyName(ptsName) + if (!expectedTty) { + return false + } + const verified = new Set() + const verificationDeadline = Date.now() + timeout + for (let index = 0; index < memberPids.length; index += VERIFY_PID_BATCH_SIZE) { + const remainingMs = verificationDeadline - Date.now() + if (remainingMs <= 0) { + return false + } + const batch = memberPids.slice(index, index + VERIFY_PID_BATCH_SIZE) + const ownerColumn = platform === 'linux' ? ['-o', 'sid='] : [] + const result = (await run( + 'ps', + ['-p', batch.join(','), '-o', 'pid=', ...ownerColumn, '-o', 'tty='], + { timeout: remainingMs } + )) as { stdout?: string | Buffer } + for (const line of String(result.stdout ?? '') + .trim() + .split('\n')) { + const columns = line.trim().split(/\s+/) + const [pidText, ownerOrTty, linuxTty] = columns + const pid = parseCanonicalProcessId(pidText) + const tty = platform === 'linux' ? linuxTty : ownerOrTty + if ( + columns.length !== (platform === 'linux' ? 3 : 2) || + pid === null || + !batch.includes(pid) || + verified.has(pid) || + tty !== expectedTty || + (platform === 'linux' && parseCanonicalProcessId(ownerOrTty) !== rootPid) + ) { + return false + } + verified.add(pid) + } + } + return verified.size === memberPids.length +} + +export function isEmptyProcessSelection(error: unknown): boolean { + if (!error || typeof error !== 'object') { + return false + } + const processError = error as { code?: unknown; stdout?: unknown } + // Why: BSD/procps ps/pgrep exit 1 for an empty selector. Only that exact + // empty result proves absence; ENOENT, timeout, and malformed output fail closed. + return processError.code === 1 && String(processError.stdout ?? '').trim() === '' +} + +export function parseCanonicalProcessIds(stdout: string | Buffer | undefined): number[] { + const ids: number[] = [] + const seen = new Set() + for (const token of String(stdout ?? '') + .trim() + .split(/\s+/) + .filter(Boolean)) { + const pid = parseCanonicalProcessId(token) + if (pid === null || seen.has(pid)) { + throw new Error('Malformed PTY process inventory') + } + seen.add(pid) + ids.push(pid) + } + return ids +} + +export function parseCanonicalProcessId(value: string | undefined): number | null { + if (!value || !/^[1-9]\d*$/.test(value)) { + return null + } + const pid = Number(value) + return Number.isSafeInteger(pid) ? pid : null +} + +function normalizePtyName(ptsName: unknown): string | null { + if (typeof ptsName !== 'string') { + return null + } + const expectedTty = ptsName.startsWith('/dev/') ? ptsName.slice('/dev/'.length) : ptsName + return /^[A-Za-z0-9._/-]+$/.test(expectedTty) ? expectedTty : null +} From 45b4338970d8c3cb6a28d0e3a845a1f5495ae077 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 14 Jul 2026 23:11:22 -0700 Subject: [PATCH 35/36] fix(relay): retain ConPTY liveness until native exit --- src/relay/pty-handler.test.ts | 8 +++++++- src/relay/pty-handler.ts | 22 ++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index 64adcca750c..de5626d4885 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -1239,7 +1239,7 @@ describe('PtyHandler', () => { }) }) - it('retains an unverified ConPTY across graceful-fallback teardown retries', async () => { + it('retains an unverified ConPTY across delayed teardown retries', async () => { await withWindowsPlatform(async () => { let onExitCb: ((evt: { exitCode: number }) => void) | undefined const mockKill = vi.fn() @@ -1257,6 +1257,11 @@ describe('PtyHandler', () => { await vi.advanceTimersByTimeAsync(5000) expect(handler.activePtyCount).toBe(1) + await expect(dispatcher.callRequest('pty.hasPty', { id: 'pty-1' })).resolves.toBe(true) + expect(dispatcher.notify).not.toHaveBeenCalledWith('pty.exit', { + id: 'pty-1', + code: -1 + }) const firstRetry = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true @@ -1281,6 +1286,7 @@ describe('PtyHandler', () => { expect(handler.activePtyCount).toBe(1) onExitCb!({ exitCode: 0 }) expect(handler.activePtyCount).toBe(0) + await expect(dispatcher.callRequest('pty.hasPty', { id: 'pty-1' })).resolves.toBe(false) }) }) diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index 8a27b211c38..cc6887fe5d6 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -1003,9 +1003,11 @@ export class PtyHandler { managed.windowsImmediateKillIssued = false throw error } - } else { - killPtyProcess(managed.pty, 'SIGTERM') + // Why: no stronger Windows fallback exists after ConPTY close. Retain + // liveness until native exit so deletion cannot trust a synthetic exit. + return } + killPtyProcess(managed.pty, 'SIGTERM') // Why: Some processes ignore SIGTERM (e.g. a hung child, a custom signal // handler). Without a SIGKILL fallback the PTY process would leak and the @@ -1019,12 +1021,7 @@ export class PtyHandler { managed.killTimer = setTimeout(() => { const still = this.ptys.get(id) if (still && !still.disposed) { - // Why: on Windows the graceful path already closed ConPTY, which is - // force-capable; a second close can tear down an unrelated conout - // pipe pairing. Keep the cleanup guarantee, skip the redundant kill. - if (!still.windowsImmediateKillIssued) { - killPtyProcess(still.pty, 'SIGKILL') - } + killPtyProcess(still.pty, 'SIGKILL') this.flushPtyOutput(id) // Why: emit pty.exit BEFORE disposeManagedPty sets disposed=true. // The natural onExit short-circuits on `managed.disposed`, so @@ -1032,14 +1029,11 @@ export class PtyHandler { // when the SIGKILL fallback fires for a SIGTERM-ignoring child. this.dispatcher.notify('pty.exit', { id, code: -1 }) // Why: POSIX SIGKILL may never emit onExit for a D-state child, so - // dispose and delete it synchronously. Windows must retain the owner - // until native exit proves the already-issued ConPTY close completed. + // dispose and delete it synchronously. this.notifyExitListener(still) disposeManagedPty(still) - if (process.platform !== 'win32' || !still.windowsImmediateKillIssued) { - this.ptys.delete(id) - this.clearPtyFlowState(id) - } + this.ptys.delete(id) + this.clearPtyFlowState(id) } }, 5000) } From e0185b14fbdd780961c4d239d28b3fdefb832400 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 15 Jul 2026 00:03:02 -0700 Subject: [PATCH 36/36] fix(terminals): retain teardown ownership across graceful close --- config/reliability-gates.jsonc | 31 ++++++++++- src/main/daemon/daemon-pty-router.test.ts | 47 +++++++++++++++- src/main/daemon/daemon-pty-router.ts | 8 ++- .../daemon-session-route-restoration.ts | 27 ++++++++++ .../degraded-daemon-pty-provider.test.ts | 17 ++++++ .../daemon/degraded-daemon-pty-provider.ts | 15 +++--- src/main/providers/local-pty-provider.test.ts | 24 +++++++++ src/main/providers/local-pty-provider.ts | 53 +++++++++---------- 8 files changed, 180 insertions(+), 42 deletions(-) create mode 100644 src/main/daemon/daemon-session-route-restoration.ts diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index edec03e3ee4..d498a1c122f 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -625,6 +625,24 @@ "a graceful-fallback ConPTY without an exit event remains an unverified owner across repeated teardown timeouts until native exit arrives" ] }, + { + "file": "src/main/providers/local-pty-provider.test.ts", + "assertions": [ + "a local Windows graceful close remains inventoried until native exit and an overlapping verified stop joins without issuing a second ConPTY close" + ] + }, + { + "file": "src/main/daemon/daemon-pty-router.test.ts", + "assertions": [ + "an already-paid daemon inventory restores a missing legacy route before full-session capability checks without letting a stale legacy snapshot replace a newer current owner" + ] + }, + { + "file": "src/main/daemon/degraded-daemon-pty-provider.test.ts", + "assertions": [ + "a terminating daemon session discovered by inventory cannot fall through to a successful local-fallback no-op" + ] + }, { "file": "src/main/ipc/worktrees.test.ts", "assertions": [ @@ -643,6 +661,15 @@ "result": "passed", "durationSeconds": 11.56, "summary": "15 files and 966 tests passed locally, including canonical POSIX ownership parsing, bounded exit-probe counts, ConPTY graceful-fallback retry retention, watcher-after-verification ordering, eight-worker stop bounds, and local/daemon/SSH fail-closed isolation." + }, + { + "date": "2026-07-14", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/relay/pty-session-kill.test.ts src/relay/pty-handler.test.ts src/main/runtime/worktree-pty-admission.test.ts src/main/runtime/pty-stop-concurrency.test.ts src/main/runtime/worktree-teardown.test.ts src/main/providers/local-pty-provider.test.ts src/main/providers/ssh-pty-liveness.test.ts src/main/providers/ssh-pty-provider.test.ts src/main/daemon/pty-subprocess.test.ts src/main/daemon/session.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts src/main/ipc/pty.test.ts src/main/ipc/worktrees.test.ts", + "result": "passed", + "durationSeconds": 12.02, + "summary": "15 files and 974 tests passed after merging current main, including local graceful-to-verified ConPTY overlap and missing degraded/legacy route restoration without extra provider calls or stale-route replacement." } ], "runtimeBudget": { @@ -655,11 +682,11 @@ }, "redGreenEvidence": { "status": "partial", - "evidence": "The new malformed-child-inventory assertions reject the prior silent token filtering, and the graceful-fallback retry assertion rejects the prior deletion of an unverified ConPTY owner. Saved intentional-break or CI red artifacts remain pending." + "evidence": "The new malformed-child-inventory assertions reject the prior silent token filtering, the graceful-fallback retry assertion rejects deletion of an unverified ConPTY owner, the local overlap test failed while graceful close erased its pre-exit owner, and the degraded/legacy inventory tests failed while missing routes fell through to the wrong owner. Saved CI red artifacts remain pending." }, "performanceBudget": { "required": true, - "evidence": "Destructive teardown adds no polling or scans to typing, output, focus, startup, or ordinary render paths. Fifty snapshotted PTYs use exactly fifty stops with an eight-worker ceiling; current SSH uses one targeted liveness request per PTY while a legacy relay shares one bounded inventory; POSIX discovery caps the tree at 1,024 PIDs under one 2.5-second deadline, final verification at 500 ms, PID batches at 64, and signal-0 fast probes at two per PID before batched ps polling. A final-source local victim-root-plus-child teardown completed in 99 ms against the 8,000 ms total budget while an unrelated witness PTY stayed alive." + "evidence": "Destructive teardown adds no polling or scans to typing, output, focus, startup, or ordinary render paths. Fifty snapshotted PTYs use exactly fifty stops with an eight-worker ceiling; current SSH uses one targeted liveness request per PTY while a legacy relay shares one bounded inventory; missing daemon routes are restored with an O(returned sessions) pass over that existing inventory and no additional provider calls. POSIX discovery caps the tree at 1,024 PIDs under one 2.5-second deadline, final verification at 500 ms, PID batches at 64, and signal-0 fast probes at two per PID before batched ps polling. A final-source local victim-root-plus-child teardown completed in 99 ms against the 8,000 ms total budget while an unrelated witness PTY stayed alive." }, "promotionCriteria": [ "Run the focused gate for at least 100 consecutive passes or 14 days across required CI platforms.", diff --git a/src/main/daemon/daemon-pty-router.test.ts b/src/main/daemon/daemon-pty-router.test.ts index 6e87232e59a..b76240331f2 100644 --- a/src/main/daemon/daemon-pty-router.test.ts +++ b/src/main/daemon/daemon-pty-router.test.ts @@ -1,7 +1,12 @@ import { describe, expect, it, vi } from 'vitest' import { DaemonPtyRouter } from './daemon-pty-router' import type { DaemonPtyAdapter } from './daemon-pty-adapter' -import type { PtyBackgroundStreamEvent, PtySpawnOptions, PtySpawnResult } from '../providers/types' +import type { + PtyBackgroundStreamEvent, + PtyProcessInfo, + PtySpawnOptions, + PtySpawnResult +} from '../providers/types' import { GIT_CREDENTIAL_GUARD_HOST_PROTOCOL_VERSION } from './daemon-protocol-capabilities' type AdapterMock = DaemonPtyAdapter & { @@ -193,6 +198,46 @@ describe('DaemonPtyRouter', () => { expect(router.canVerifyFullSessionTeardown('current-session')).toBe(true) }) + it('restores a missing legacy route from the teardown inventory', async () => { + const current = createAdapter('current', [], undefined, 22) + const legacy = createAdapter('legacy', [], undefined, 21) + vi.mocked(legacy.listProcesses).mockResolvedValue([ + { id: 'legacy-terminating', cwd: '', title: 'shell' } + ]) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + + await expect(router.listProcesses()).resolves.toContainEqual({ + id: 'legacy-terminating', + cwd: '', + title: 'shell' + }) + + expect(router.canVerifyFullSessionTeardown('legacy-terminating')).toBe(false) + expect(legacy.canVerifyFullSessionTeardown).toHaveBeenCalledWith('legacy-terminating') + }) + + it('does not let a stale legacy inventory replace a newer current route', async () => { + const current = createAdapter('current', [], undefined, 22) + const legacy = createAdapter('legacy', [], undefined, 21) + let resolveLegacyInventory = (_sessions: PtyProcessInfo[]): void => {} + vi.mocked(legacy.listProcesses).mockReturnValue( + new Promise((resolve) => { + resolveLegacyInventory = resolve + }) + ) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + + const listing = router.listProcesses() + await vi.waitFor(() => expect(legacy.listProcesses).toHaveBeenCalledTimes(1)) + await router.spawn({ sessionId: 'reused-session', cols: 80, rows: 24 }) + resolveLegacyInventory([{ id: 'reused-session', cwd: '', title: 'old shell' }]) + await listing + + expect(router.canVerifyFullSessionTeardown('reused-session')).toBe(true) + expect(current.canVerifyFullSessionTeardown).toHaveBeenCalledWith('reused-session') + expect(legacy.canVerifyFullSessionTeardown).not.toHaveBeenCalledWith('reused-session') + }) + it('routes background hints and authoritative snapshots to the session owner', async () => { const current = createAdapter('current') const legacy = createAdapter('legacy', ['legacy-session']) diff --git a/src/main/daemon/daemon-pty-router.ts b/src/main/daemon/daemon-pty-router.ts index 370b9e90f6c..a8287b24e8d 100644 --- a/src/main/daemon/daemon-pty-router.ts +++ b/src/main/daemon/daemon-pty-router.ts @@ -8,6 +8,7 @@ import type { PtySpawnResult } from '../providers/types' import { ShutdownVerificationOwnerCache } from './shutdown-verification-owner-cache' +import { listAndRestoreDaemonSessionRoutes } from './daemon-session-route-restoration' export class DaemonPtyRouter implements IPtyProvider { private current: DaemonPtyAdapter @@ -188,8 +189,11 @@ export class DaemonPtyRouter implements IPtyProvider { async listProcesses(): Promise { // Why: runtime exact-stop/liveness flows must fail closed if any adapter // cannot provide a trustworthy process list. - const results = await Promise.all(this.allAdapters().map((adapter) => adapter.listProcesses())) - return results.flat() + return listAndRestoreDaemonSessionRoutes( + this.allAdapters(), + this.sessionAdapters, + this.shutdownVerificationAdapters + ) } async getDefaultShell(): Promise { diff --git a/src/main/daemon/daemon-session-route-restoration.ts b/src/main/daemon/daemon-session-route-restoration.ts new file mode 100644 index 00000000000..a2884acde1d --- /dev/null +++ b/src/main/daemon/daemon-session-route-restoration.ts @@ -0,0 +1,27 @@ +import type { PtyProcessInfo } from '../providers/types' + +type DaemonSessionRouteOwner = { + listProcesses(): Promise +} + +export async function listAndRestoreDaemonSessionRoutes( + owners: readonly T[], + routes: Map, + shutdownOwners: { delete(id: string): void } +): Promise { + const inventories = await Promise.all(owners.map((owner) => owner.listProcesses())) + const sessions: PtyProcessInfo[] = [] + for (const [index, ownerSessions] of inventories.entries()) { + const owner = owners[index]! + for (const session of ownerSessions) { + sessions.push(session) + if (!routes.has(session.id)) { + // Why: graceful shutdown can retire fast liveness before native exit. + // Reuse this inventory so destructive teardown still reaches its owner. + routes.set(session.id, owner) + shutdownOwners.delete(session.id) + } + } + } + return sessions +} diff --git a/src/main/daemon/degraded-daemon-pty-provider.test.ts b/src/main/daemon/degraded-daemon-pty-provider.test.ts index eb31e610e22..04e5a896a1f 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.test.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.test.ts @@ -154,6 +154,23 @@ describe('DegradedDaemonPtyProvider', () => { expect(provider.canVerifyFullSessionTeardown(fresh.id)).toBe(true) }) + it('restores a missing daemon route from the teardown inventory', async () => { + const ptyId = 'w1@@terminating-daemon-session' + const current = createDaemonAdapter('daemon', [], 21) + vi.mocked(current.listProcesses).mockResolvedValue([{ id: ptyId, cwd: '', title: 'shell' }]) + const fallback = createProvider('fallback') + const provider = new DegradedDaemonPtyProvider({ current, legacy: [], fallback }) + + await expect(provider.listProcesses()).resolves.toContainEqual({ + id: ptyId, + cwd: '', + title: 'shell' + }) + + expect(provider.canVerifyFullSessionTeardown(ptyId)).toBe(false) + expect(current.canVerifyFullSessionTeardown).toHaveBeenCalledWith(ptyId) + }) + it('routes a previously daemon-backed id to fallback after daemon exit removes the mapping', async () => { const current = createDaemonAdapter('daemon', ['daemon-session']) const fallback = createProvider('fallback') diff --git a/src/main/daemon/degraded-daemon-pty-provider.ts b/src/main/daemon/degraded-daemon-pty-provider.ts index 7d247b08e4d..4c8a5405131 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.ts @@ -11,6 +11,7 @@ import type { import { ShutdownVerificationOwnerCache } from './shutdown-verification-owner-cache' import { reconcileDegradedDaemonSessions } from './degraded-daemon-session-reconciliation' import { subscribeToProviderReplay } from './provider-replay-subscription' +import { listAndRestoreDaemonSessionRoutes } from './daemon-session-route-restoration' type ManagedPtyProvider = IPtyProvider & { disconnectOnly?: () => Promise @@ -190,10 +191,11 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { } async listProcesses(): Promise { - const results = await Promise.all( - this.allProviders().map((provider) => provider.listProcesses()) + return listAndRestoreDaemonSessionRoutes( + this.allProviders(), + this.sessionProviders, + this.shutdownVerificationProviders ) - return results.flat() } async getDefaultShell(): Promise { @@ -320,11 +322,8 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { } private providerFor(sessionId: string): ManagedPtyProvider { - return ( - this.sessionProviders.get(sessionId) ?? - this.findProviderForExistingSession(sessionId) ?? - this.fallback - ) + const routed = this.sessionProviders.get(sessionId) + return routed ?? this.findProviderForExistingSession(sessionId) ?? this.fallback } private findProviderForExistingSession(sessionId: string): ManagedPtyProvider | null { diff --git a/src/main/providers/local-pty-provider.test.ts b/src/main/providers/local-pty-provider.test.ts index 154c5a526d8..da57f6031ca 100644 --- a/src/main/providers/local-pty-provider.test.ts +++ b/src/main/providers/local-pty-provider.test.ts @@ -1062,6 +1062,30 @@ describe('LocalPtyProvider', () => { } }) + it('keeps a gracefully closing Windows PTY visible to overlapping verified teardown', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const killSpy = vi.fn() + spawnMock.mockReturnValue({ + ...mockProc, + kill: killSpy + }) + const { id } = await provider.spawn({ cols: 80, rows: 24 }) + + const gracefulStop = provider.shutdown(id, { immediate: false }) + await Promise.resolve() + expect((await provider.listProcesses()).some((session) => session.id === id)).toBe(true) + + const verifiedStop = provider.shutdown(id, { immediate: true }) + expect(killSpy).toHaveBeenCalledTimes(1) + exitCb?.({ exitCode: -1 }) + + await expect(Promise.all([gracefulStop, verifiedStop])).resolves.toEqual([ + undefined, + undefined + ]) + expect((await provider.listProcesses()).some((session) => session.id === id)).toBe(false) + }) + it('retains the ConPTY owner when the native Windows kill throws', async () => { Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) const killError = new Error('native close failed') diff --git a/src/main/providers/local-pty-provider.ts b/src/main/providers/local-pty-provider.ts index ca79aa95bb3..8d08cfa900e 100644 --- a/src/main/providers/local-pty-provider.ts +++ b/src/main/providers/local-pty-provider.ts @@ -79,7 +79,7 @@ const ptyTerminalHandle = new Map() // to invoke/clean them up on a destroyed environment, triggering a SIGABRT. const ptyDisposables = new Map void }[]>() const ptyCleanupCallbacks = new Map void>() -const windowsImmediateKills = new WeakSet() +const windowsPtyCloseIssued = new WeakSet() // Why: node-pty can spend up to 5s enumerating a Windows console tree before // its fallback runs; leave bounded headroom for the native exit event to land. @@ -855,7 +855,7 @@ export class LocalPtyProvider implements IPtyProvider { // Why: release the master ptmx fd on the natural-exit path — without // this, a shell that exits cleanly (the common case) never releases its // fd until the next GC. See docs/fix-pty-fd-leak.md. - destroyPtyProcess(proc, { alreadyKilled: windowsImmediateKills.has(proc) }) + destroyPtyProcess(proc, { alreadyKilled: windowsPtyCloseIssued.has(proc) }) this.opts.onExit?.(id, exitCode) for (const cb of exitListeners) { cb({ id, code: exitCode }) @@ -942,42 +942,37 @@ export class LocalPtyProvider implements IPtyProvider { if (!proc) { return } - if (process.platform === 'win32' && windowsImmediateKills.has(proc)) { - if (!(await waitForPtyExit(id, proc).promise)) { - throw new Error(`Unable to verify full PTY session teardown: ${id}`) - } - return - } - let alreadyKilled = false - if (opts.immediate) { - if (process.platform === 'win32') { - // Why: ConPTY close completes asynchronously. Deletion must await the - // native exit event, while a timeout keeps the owner retryable. - const exitWaiter = waitForPtyExit(id, proc) + if (process.platform === 'win32') { + // Why: a graceful close and destructive teardown use the same ConPTY + // close. Retain its owner until native exit so overlap cannot fail open. + const exitWaiter = waitForPtyExit(id, proc) + if (!windowsPtyCloseIssued.has(proc)) { try { - windowsImmediateKills.add(proc) + windowsPtyCloseIssued.add(proc) proc.kill() } catch (error) { exitWaiter.cancel() - windowsImmediateKills.delete(proc) + windowsPtyCloseIssued.delete(proc) if (ptyProcesses.get(id) === proc) { throw error } } - if (!(await exitWaiter.promise)) { - throw new Error(`Unable to verify full PTY session teardown: ${id}`) - } - return - } else { - const killedSession = await killPosixPtySession( - proc.pid, - (proc as unknown as { ptsName?: unknown }).ptsName - ) - if (!killedSession) { - throw new Error(`Unable to verify full PTY session teardown: ${id}`) - } - alreadyKilled = true } + if (!(await exitWaiter.promise)) { + throw new Error(`Unable to verify full PTY session teardown: ${id}`) + } + return + } + let alreadyKilled = false + if (opts.immediate) { + const killedSession = await killPosixPtySession( + proc.pid, + (proc as unknown as { ptsName?: unknown }).ptsName + ) + if (!killedSession) { + throw new Error(`Unable to verify full PTY session teardown: ${id}`) + } + alreadyKilled = true } // Why: disposePtyListeners removes the onExit callback, so the natural // exit cleanup path from node-pty won't fire. Cleanup and notification