diff --git a/plugins/symphony-plus-plus-mcp/scripts/start-sympp-mcp-bridge.js b/plugins/symphony-plus-plus-mcp/scripts/start-sympp-mcp-bridge.js index 614be01145..3d5eaa28dd 100644 --- a/plugins/symphony-plus-plus-mcp/scripts/start-sympp-mcp-bridge.js +++ b/plugins/symphony-plus-plus-mcp/scripts/start-sympp-mcp-bridge.js @@ -873,7 +873,7 @@ async function bridge(identity, state, runtimeFile) { process.stdout.write(`${JSON.stringify(indeterminateToolCall(parsed.id))}\n`); continue; } - if (recovered && replayProvablyUnsent(response)) { + if (recovered && (replayProvablyUnsent(response) || !parsed || parsed.method !== "tools/call")) { try { if (!requestProtocol) await initializeSession(); response = await mcpPost(current.mcpUrl, line, sessionId, protocol, timeoutMs); diff --git a/plugins/symphony-plus-plus-mcp/scripts/sympp-mcp-process-runtime.ps1 b/plugins/symphony-plus-plus-mcp/scripts/sympp-mcp-process-runtime.ps1 index 73707f653b..2ec7cf9ea6 100644 --- a/plugins/symphony-plus-plus-mcp/scripts/sympp-mcp-process-runtime.ps1 +++ b/plugins/symphony-plus-plus-mcp/scripts/sympp-mcp-process-runtime.ps1 @@ -398,11 +398,21 @@ function Start-Frontend($Plan, [string]$BackendUrl, [string]$AssetsDir, [string] } } +function ConvertFrom-McpRequestJson([string]$Line) { + if ($PSVersionTable.PSEdition -eq "Desktop") { + Add-Type -AssemblyName System.Web.Extensions + $serializer = New-Object System.Web.Script.Serialization.JavaScriptSerializer + $serializer.MaxJsonLength = [int]::MaxValue + return $serializer.DeserializeObject($Line) + } + return $Line | ConvertFrom-Json -AsHashtable +} + function Get-RequestIdForError([string]$Line) { try { - $payload = $Line | ConvertFrom-Json - if ($payload.PSObject.Properties["id"]) { - return $payload.id + $payload = ConvertFrom-McpRequestJson $Line + if ($payload.Keys -ccontains "id") { + return $payload["id"] } } catch { } @@ -591,7 +601,10 @@ function Test-McpBackendUnavailableResponse($Response) { } function Test-McpToolCall([string]$Line) { - try { return [string](($Line | ConvertFrom-Json).method) -eq "tools/call" } catch { return $false } + try { + $payload = ConvertFrom-McpRequestJson $Line + return ($payload.Keys -ccontains "method") -and [string]$payload["method"] -ceq "tools/call" + } catch { return $true } } function Invoke-McpBackendRecovery([scriptblock]$Recover, [string]$McpUrl, [string]$ClientId, [int]$HeartbeatIntervalSec, $StdinReadState) { @@ -712,7 +725,7 @@ function Invoke-HttpMcpBridge([string]$McpUrl, [int]$TimeoutSec, [string]$Client if ($null -ne $recovered) { $McpUrl = $recovered.mcp_url; $heartbeatIntervalMs = $recovered.heartbeat_interval_ms $sessionId = $null; $protocolVersion = $null; $needsInitialize = $true - if (-not $requestMayHaveReachedBackend) { + if (-not $requestMayHaveReachedBackend -or -not (Test-McpToolCall $line)) { if (-not [string]::IsNullOrWhiteSpace($requestProtocolVersion)) { $needsInitialize = $false } else { diff --git a/plugins/symphony-plus-plus-mcp/tests/launcher/cold-start-singleton-smoke.js b/plugins/symphony-plus-plus-mcp/tests/launcher/cold-start-singleton-smoke.js index ba2babdf0d..5039bf2e09 100644 --- a/plugins/symphony-plus-plus-mcp/tests/launcher/cold-start-singleton-smoke.js +++ b/plugins/symphony-plus-plus-mcp/tests/launcher/cold-start-singleton-smoke.js @@ -87,7 +87,7 @@ function backendFixture() { return [ '"use strict";', 'const crypto=require("crypto"),fs=require("fs"),http=require("http"),path=require("path");', - 'const a=process.argv.slice(2),arg=(n)=>a[a.indexOf(n)+1],port=Number(arg("--port")),stateFile=arg("--state"),release=arg("--release"),bindRelease=arg("--bind-release"),failAfterProbe=arg("--fail-after-probe"),contract=arg("--contract"),revision=arg("--revision"),ledger=arg("--ledger");', + 'const a=process.argv.slice(2),arg=(n)=>a[a.indexOf(n)+1],port=Number(arg("--port")),stateFile=arg("--state"),release=arg("--release"),bindRelease=arg("--bind-release"),failAfterProbe=arg("--fail-after-probe"),failRead=arg("--fail-read"),contract=arg("--contract"),revision=arg("--revision"),ledger=arg("--ledger");', 'const tools=JSON.parse(Buffer.from(arg("--tools"),"base64").toString("utf8")),leases=new Set(),sessions=new Set();let failAfterProbeArmed=false;', 'let previous={};try{previous=JSON.parse(fs.readFileSync(stateFile,"utf8"));}catch(_){}', 'const state={pid:process.pid,starts:(previous.starts||0)+1,started_at:Date.now(),initialize:previous.initialize||0,tools_list:previous.tools_list||0,mutations:previous.mutations||0,attach:previous.attach||0,detach:previous.detach||0,lease_peak:previous.lease_peak||0,active_leases:0};', @@ -99,7 +99,7 @@ function backendFixture() { ' if(req.url==="/mcp/readiness"){if(release&&!fs.existsSync(release))return send(res,503,{status:"starting"});return send(res,200,{status:"ok",ledger:{reachable:true},dashboard:{ready:true},source:{revision,mcp_contract:{fingerprint:contract}}});}', ' if(req.url==="/sympp/board")return send(res,200,"Symphony++ Dashboard",{"Content-Type":"text/html"});', ' if(req.url==="/mcp/client-lease"){const p=JSON.parse(await body(req));if(p.action==="attach")state.attach++;if(p.action==="attach"||p.action==="heartbeat")leases.add(p.client_id);if(p.action==="heartbeat"&&failAfterProbe&&fs.existsSync(failAfterProbe)){fs.unlinkSync(failAfterProbe);failAfterProbeArmed=true;}if(p.action==="detach"){state.detach++;leases.delete(p.client_id);}state.lease_peak=Math.max(state.lease_peak,leases.size);save();return send(res,200,{stale_after_ms:600000});}', - ' if(req.url==="/mcp"){const p=JSON.parse(await body(req));let result,session=req.headers["mcp-session-id"];if(p.method==="initialize"){state.initialize++;session=crypto.randomUUID();sessions.add(session);result={protocolVersion:"2025-03-26",capabilities:{},serverInfo:{name:"cold-fixture",version:"1"}};}else if(!sessions.has(session))return send(res,404,{error:"missing session"});else if(p.method==="tools/list"){state.tools_list++;result={tools:tools.map(name=>({name,description:"fixture",inputSchema:{type:"object"}}))};}else if(p.method==="tools/call"&&p.params.name==="fixture.mutate"){state.mutations++;save();res.writeHead(200,{"Content-Type":"application/json"});res.write("{\\\"jsonrpc\\\":\\\"2.0\\\",\\\"id\\\":");return setTimeout(()=>{res.destroy();setTimeout(()=>process.exit(0),10);},25);}else return send(res,404,{error:"missing"});save();return send(res,200,{jsonrpc:"2.0",id:p.id,result},{"Mcp-Session-Id":session});}', + ' if(req.url==="/mcp"){const p=JSON.parse(await body(req));let result,session=req.headers["mcp-session-id"];if(p.method==="initialize"){state.initialize++;session=crypto.randomUUID();sessions.add(session);result={protocolVersion:"2025-03-26",capabilities:{},serverInfo:{name:"cold-fixture",version:"1"}};}else if(!sessions.has(session))return send(res,404,{error:"missing session"});else if(p.method==="tools/list"){state.tools_list++;if(failRead&&fs.existsSync(failRead)){fs.unlinkSync(failRead);save();res.writeHead(200,{"Content-Type":"application/json"});res.write("{\\\"jsonrpc\\\":\\\"2.0\\\",\\\"id\\\":");return setTimeout(()=>{server.close();server.closeAllConnections?.();setTimeout(()=>process.exit(0),10);},25);}result={tools:tools.map(name=>({name,description:"fixture",inputSchema:{type:"object"}}))};}else if(p.method==="tools/call"&&p.params.name==="fixture.mutate"){state.mutations++;save();res.writeHead(200,{"Content-Type":"application/json"});res.write("{\\\"jsonrpc\\\":\\\"2.0\\\",\\\"id\\\":");return setTimeout(()=>{res.destroy();setTimeout(()=>process.exit(0),10);},25);}else return send(res,404,{error:"missing"});save();return send(res,200,{jsonrpc:"2.0",id:p.id,result},{"Mcp-Session-Id":session});}', ' send(res,404,{error:"not found"});', '});', 'server.on("connection",socket=>{if(!failAfterProbeArmed)return;failAfterProbeArmed=false;socket.on("close",()=>{server.close();server.closeAllConnections?.();setTimeout(()=>process.exit(0),10);});});', @@ -107,7 +107,7 @@ function backendFixture() { ].join("\n"); } -function createArchive(root, shell, backendPort, backendState, releaseFile, bindReleaseFile, failAfterProbeFile, ledgerFile) { +function createArchive(root, shell, backendPort, backendState, releaseFile, bindReleaseFile, failAfterProbeFile, failReadFile, ledgerFile) { const source = path.join(root, "artifact-source"); const archive = path.join(root, "artifact.zip"); fs.mkdirSync(path.join(source, "dashboard"), { recursive: true }); @@ -122,7 +122,7 @@ function createArchive(root, shell, backendPort, backendState, releaseFile, bind archive, sha: sha256(fs.readFileSync(archive)), dashboardHash, - runtimeArgs: ["--port", "{port}", "--state", backendState, "--release", releaseFile, "--bind-release", bindReleaseFile, "--fail-after-probe", failAfterProbeFile, "--contract", contract, "--revision", revision, "--ledger", ledgerFile, "--tools", Buffer.from(JSON.stringify(expectedTools)).toString("base64"), "start-runtime.ps1"], + runtimeArgs: ["--port", "{port}", "--state", backendState, "--release", releaseFile, "--bind-release", bindReleaseFile, "--fail-after-probe", failAfterProbeFile, "--fail-read", failReadFile, "--contract", contract, "--revision", revision, "--ledger", ledgerFile, "--tools", Buffer.from(JSON.stringify(expectedTools)).toString("base64"), "start-runtime.ps1"], backendPort, }; } @@ -214,14 +214,18 @@ function startClient(barrier, launcher, environment, clients, latencies, readyTa return client; } -function requestClient(client, id, method, params = {}) { +function requestClientLine(client, id, line) { return new Promise((resolve, reject) => { const timeout = setTimeout(() => { client.pending.delete(id); reject(new Error(`Client request ${id} timed out. ${client.stderr}`)); }, 90000); client.pending.set(id, (response) => { clearTimeout(timeout); resolve(response); }); - client.child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`); + client.child.stdin.write(`${line}\n`); }); } +function requestClient(client, id, method, params = {}) { + return requestClientLine(client, id, JSON.stringify({ jsonrpc: "2.0", id, method, params })); +} + function assertLockFree(shell, startupLock, artifactLock) { const environment = { ...process.env, LOCK_PATHS: `${startupLock}|${artifactLock}` }; const result = spawnSync(shell, ["-NoProfile", "-NonInteractive", "-Command", "$env:LOCK_PATHS.Split('|') | ForEach-Object { $f=[IO.File]::Open($_,[IO.FileMode]::OpenOrCreate,[IO.FileAccess]::ReadWrite,[IO.FileShare]::None);$f.Dispose() }"], { env: environment, windowsHide: true, encoding: "utf8" }); @@ -241,6 +245,7 @@ async function runCase(clientCount, shell, mode = "normal") { const releaseFile = path.join(root, "backend-ready"); const bindReleaseFile = path.join(root, "backend-bind-ready"); const failAfterProbeFile = path.join(root, "fail-after-probe"); + const failReadFile = path.join(root, "fail-read"); const ledgerFile = path.join(root, "ledger", "fixture.sqlite3"); const barrier = path.join(root, "barrier"); const backendPort = await freePort(); @@ -261,7 +266,7 @@ async function runCase(clientCount, shell, mode = "normal") { writeJson(path.join(sourceRoot, "elixir", "priv", "symphony_plus_plus", "mcp_contract.json"), { mcp_contract_fingerprint: contract }); if (mode !== "backend_death") fs.writeFileSync(releaseFile, "ready"); if (mode !== "backend_prebind_death") fs.writeFileSync(bindReleaseFile, "ready"); - const artifact = createArchive(root, shell, backendPort, backendState, releaseFile, bindReleaseFile, failAfterProbeFile, ledgerFile); + const artifact = createArchive(root, shell, backendPort, backendState, releaseFile, bindReleaseFile, failAfterProbeFile, failReadFile, ledgerFile); let resolvedManifest; channel = await createChannelServer(mode, () => JSON.stringify(resolvedManifest), artifact.archive); resolvedManifest = { @@ -279,7 +284,7 @@ async function runCase(clientCount, shell, mode = "normal") { SYMPP_COLD_START_TIMEOUT_SEC: "90", SYMPP_BACKEND_STARTUP_TIMEOUT_SEC: "60", SYMPP_BACKEND_PORT_RELEASE_TIMEOUT_SEC: "1", SYMPP_ELIXIR_SETUP_TIMEOUT_SEC: "30", SYMPP_AUTOSTART_FRONTEND: "0", SYMPP_MCP_HTTP_TIMEOUT_SEC: "30", TEMP: path.join(root, "tmp"), TMP: path.join(root, "tmp") }; if (mode === "shutdown_during_recovery") environment.SYMPP_MCP_CLIENT_HEARTBEAT_SEC = "5"; - if (["powershell_fallback", "powershell_fallback_recovery", "powershell_fallback_initialize_retry"].includes(mode)) environment.SYMPP_NODE_BRIDGE = "0"; + if (mode.startsWith("powershell_fallback")) environment.SYMPP_NODE_BRIDGE = "0"; for (const name of ["SYMPP_REPO_ROOT", "SYMPP_BACKEND_URL", "SYMPP_DASHBOARD_ORIGIN", "SYMPP_DATABASE", "SYMPP_SOURCE_FALLBACK", "SYMPP_ARTIFACT_RUNTIME"]) delete environment[name]; fs.mkdirSync(environment.TEMP, { recursive: true }); @@ -323,6 +328,7 @@ async function runCase(clientCount, shell, mode = "normal") { } const firstBackend = readJson(backendState); const firstOwnerPid = Number(readJson(runtimeFile)?.publication?.owner_adapter_pid || 0); + const backendOnlyReadRecovery = mode.endsWith("backend_only_read_recovery"); let recoveryClients = clients; if (mode === "powershell_fallback_initialize_retry") { fs.writeFileSync(failAfterProbeFile, "ready"); @@ -441,19 +447,28 @@ async function runCase(clientCount, shell, mode = "normal") { await terminate(firstBackend.pid); await waitFor(() => clients.filter((client) => client.child.exitCode === null).length === clientCount - 1, "Backend owner adapter did not exit."); recoveryClients = clients.filter((client) => client.child.exitCode === null); + } else if (backendOnlyReadRecovery) { + fs.writeFileSync(failReadFile, "ready"); + const recovered = mode.startsWith("powershell_fallback") + ? await requestClientLine(clients[0], 3500, '{"jsonrpc":"2.0","id":3500,"method":"tools/list","Method":"other","params":{}}') + : await requestClient(clients[0], 3500, "tools/list"); + assert.equal(recovered.result?.tools?.length, expectedTools.length, `Read-only request was not retried after backend exit. ${JSON.stringify(recovered)}`); } if (["backend_loss", "owner_loss"].includes(mode)) { const recovered = await Promise.all(recoveryClients.map((client, index) => requestClient(client, 1000 + index, "tools/list"))); assert.ok(recovered.every((response) => response.result?.tools?.length === expectedTools.length), "Surviving adapters did not rebind tools/list."); - } else if (mode === "ambiguous_tool") { - const indeterminate = await requestClient(clients[0], 2000, "tools/call", { name: "fixture.mutate", arguments: {} }); + } else if (mode.endsWith("ambiguous_tool")) { + const indeterminate = mode.startsWith("powershell_fallback") + ? await requestClientLine(clients[0], 2000, JSON.stringify({ jsonrpc: "2.0", id: 2000, method: "tools/call", Method: "other", params: { name: "fixture.mutate", arguments: { padding: "x".repeat(2_100_000) } } })) + : await requestClient(clients[0], 2000, "tools/call", { name: "fixture.mutate", arguments: {} }); + assert.equal(indeterminate.id, 2000); assert.equal(indeterminate.error?.code, -32001); assert.equal(indeterminate.error?.data?.replayed, false); const recovered = await requestClient(clients[0], 2001, "tools/list"); assert.equal(recovered.result?.tools?.length, expectedTools.length); } - if (["backend_loss", "owner_loss", "ambiguous_tool"].includes(mode)) { - const expectedRecoveredLeases = mode === "ambiguous_tool" ? 1 : recoveryClients.length; + if (["backend_loss", "owner_loss"].includes(mode) || mode.endsWith("ambiguous_tool") || backendOnlyReadRecovery) { + const expectedRecoveredLeases = mode.endsWith("ambiguous_tool") || backendOnlyReadRecovery ? 1 : recoveryClients.length; await waitFor(() => readJson(backendState)?.active_leases === expectedRecoveredLeases, "Recovered adapters did not retain their replacement backend leases."); } const activeBackend = readJson(backendState); @@ -471,12 +486,12 @@ async function runCase(clientCount, shell, mode = "normal") { await waitFor(() => portAvailable(backendPort), "Backend listener did not stop after the final client exited."); const state = await waitFor(() => { const value = readJson(runtimeFile); return value?.backend?.status === "stopped" && value.backend.pid === null && value; }, "Runtime state did not record zero-client backend shutdown."); assert.equal(state.publication.status, "ready"); - const recoveryMode = ["backend_loss", "owner_loss", "ambiguous_tool"].includes(mode); - const recoveryRebinds = ["backend_loss", "owner_loss"].includes(mode) ? recoveryClients.length : mode === "ambiguous_tool" ? 1 : 0; + const recoveryMode = ["backend_loss", "owner_loss"].includes(mode) || mode.endsWith("ambiguous_tool") || backendOnlyReadRecovery; + const recoveryRebinds = ["backend_loss", "owner_loss"].includes(mode) ? recoveryClients.length : mode.endsWith("ambiguous_tool") || backendOnlyReadRecovery ? 1 : 0; assert.equal(backend.starts, recoveryMode ? 2 : 1); assert.equal(backend.initialize, clientCount + recoveryRebinds); - assert.equal(backend.tools_list, clientCount + recoveryRebinds); - assert.equal(backend.mutations, mode === "ambiguous_tool" ? 1 : 0); + assert.equal(backend.tools_list, clientCount + recoveryRebinds + (backendOnlyReadRecovery ? 1 : 0)); + assert.equal(backend.mutations, mode.endsWith("ambiguous_tool") ? 1 : 0); assert.equal(backend.lease_peak, clientCount); assert.equal(backend.active_leases, 0); assert.equal(fs.readdirSync(path.join(symppHome, "runtime", "codex-plugin-leases"), { withFileTypes: true }).filter((entry) => entry.isFile()).length, 0); @@ -484,7 +499,8 @@ async function runCase(clientCount, shell, mode = "normal") { assert.equal(channel.counts.archive_successes, 1); assert.equal(traceCount(traceDir, "artifact_prepare_end"), 1); assert.equal(traceCount(traceDir, "runtime_ready_published"), recoveryMode ? 2 : 1); - assert.equal(traceCount(traceDir, "backend_recovery_leader"), recoveryMode ? 1 : 0); + const recoveryLeaders = recoveryMode && mode.startsWith("powershell_fallback") ? traceCount(traceDir, "runtime_ready_published") - 1 : traceCount(traceDir, "backend_recovery_leader"); + assert.equal(recoveryLeaders, recoveryMode ? 1 : 0); if (recoveryMode) { assert.notEqual(activeBackend.pid, firstBackend.pid); } @@ -503,7 +519,7 @@ async function runCase(clientCount, shell, mode = "normal") { for (const directory of [symppHome, environment.TEMP]) if (fs.existsSync(directory)) for (const entry of fs.readdirSync(directory, { recursive: true })) if (/artifact\.zip\.tmp-|\.extracting-|codex-plugin\.json\.tmp-/.test(String(entry))) leftovers.push(entry); assert.deepEqual(leftovers, []); assert.ok(percentile(latencies, 0.95) < 60000 && Math.max(...latencies) < 90000); - return { mode, shell: path.basename(shell), clients: clientCount, p95_ms: percentile(latencies, 0.95), max_ms: Math.max(...latencies), manifest: channel.counts.manifest_successes, manifest_attempts: channel.counts.manifest_attempts, artifact: channel.counts.archive_successes, artifact_attempts: channel.counts.archive_attempts, preparations: traceCount(traceDir, "artifact_prepare_end"), backends: backend.starts, pids: recoveryMode ? 2 : 1, listeners: 0, initializes: backend.initialize, tools_list: backend.tools_list, mutations: backend.mutations, lease_peak: backend.lease_peak, leases_after: backend.active_leases, adopted: traceCount(traceDir, "backend_adopted"), recovery_leaders: traceCount(traceDir, "backend_recovery_leader") }; + return { mode, shell: path.basename(shell), clients: clientCount, p95_ms: percentile(latencies, 0.95), max_ms: Math.max(...latencies), manifest: channel.counts.manifest_successes, manifest_attempts: channel.counts.manifest_attempts, artifact: channel.counts.archive_successes, artifact_attempts: channel.counts.archive_attempts, preparations: traceCount(traceDir, "artifact_prepare_end"), backends: backend.starts, pids: recoveryMode ? 2 : 1, listeners: 0, initializes: backend.initialize, tools_list: backend.tools_list, mutations: backend.mutations, lease_peak: backend.lease_peak, leases_after: backend.active_leases, adopted: traceCount(traceDir, "backend_adopted"), recovery_leaders: recoveryLeaders }; } finally { terminateTrees(clients.filter((client) => client.child.exitCode === null).map((client) => client.child.pid)); if (!backendPid) backendPid = readJson(backendState)?.pid || 0; @@ -529,7 +545,7 @@ async function main() { const powershellFallback = await runCase(30, windowsPowerShell, "powershell_fallback"); for (const mode of ["manifest_death", "artifact_death", "backend_death", "backend_prebind_death"]) results.push(await runCase(30, pwsh, mode)); const recovery = []; - for (const mode of ["owner_loss", "backend_loss", "ambiguous_tool", "shutdown_during_recovery", "generation_changed_recovery", "cleanup_source_changed_recovery", "powershell_fallback_recovery", "powershell_fallback_initialize_retry"]) recovery.push(await runCase(["shutdown_during_recovery", "generation_changed_recovery", "cleanup_source_changed_recovery"].includes(mode) ? 3 : mode === "powershell_fallback_recovery" ? 4 : mode === "powershell_fallback_initialize_retry" ? 1 : 10, mode.startsWith("powershell_fallback") ? windowsPowerShell : pwsh, mode)); + for (const mode of ["owner_loss", "backend_loss", "backend_only_read_recovery", "ambiguous_tool", "powershell_fallback_ambiguous_tool", "shutdown_during_recovery", "generation_changed_recovery", "cleanup_source_changed_recovery", "powershell_fallback_recovery", "powershell_fallback_backend_only_read_recovery", "powershell_fallback_initialize_retry"]) recovery.push(await runCase(["shutdown_during_recovery", "generation_changed_recovery", "cleanup_source_changed_recovery"].includes(mode) ? 3 : mode === "powershell_fallback_recovery" ? 4 : mode.endsWith("backend_only_read_recovery") || mode.endsWith("ambiguous_tool") || mode === "powershell_fallback_initialize_retry" ? 1 : 10, mode.startsWith("powershell_fallback") ? windowsPowerShell : pwsh, mode)); process.stdout.write(`${JSON.stringify({ matrix: results.slice(0, 3), powershell_fallback: powershellFallback, leader_death: results.slice(3), recovery, powershell_5_1: true, pwsh: true, cleanup: true })}\n`); } diff --git a/plugins/symphony-plus-plus-mcp/tests/launcher/run-launcher-tests.ps1 b/plugins/symphony-plus-plus-mcp/tests/launcher/run-launcher-tests.ps1 index 79cc44bbc7..14e2f6e112 100644 --- a/plugins/symphony-plus-plus-mcp/tests/launcher/run-launcher-tests.ps1 +++ b/plugins/symphony-plus-plus-mcp/tests/launcher/run-launcher-tests.ps1 @@ -543,8 +543,9 @@ Assert-True ($coldExitCode -eq 0) "Installed cold-herd, leader-death, and rotati $coldSmoke = $coldSmokeJson | ConvertFrom-Json Assert-True ((@($coldSmoke.matrix.clients) -join ",") -eq "30,100,200" -and @($coldSmoke.matrix | Where-Object { $_.manifest -ne 1 -or $_.artifact -ne 1 -or $_.backends -ne 1 -or $_.listeners -ne 0 }).Count -eq 0) "30/100/200 shipped-command matrices must preserve singleton cold work and release the backend listener after the final client" Assert-True ($coldSmoke.powershell_5_1 -and $coldSmoke.pwsh -and $coldSmoke.cleanup -and @($coldSmoke.leader_death).Count -eq 4) "Cold-herd coverage must prove both PowerShell shells, cleanup, and all leader-death phases" -Assert-True (@($coldSmoke.recovery).Count -eq 8 -and @($coldSmoke.recovery | Where-Object { $_.mode -notin "shutdown_during_recovery", "powershell_fallback_initialize_retry" -and ($_.backends -ne 2 -or $_.pids -ne 2 -or $_.listeners -ne 0 -or $_.recovery_leaders -ne 1) }).Count -eq 0) "Node and PowerShell fallback recovery must each elect exactly one replacement backend and still reach zero listeners" -Assert-True (($coldSmoke.recovery | Where-Object mode -eq "ambiguous_tool").mutations -eq 1 -and @($coldSmoke.recovery | Where-Object { $_.mode -notin "shutdown_during_recovery", "generation_changed_recovery", "cleanup_source_changed_recovery", "powershell_fallback_initialize_retry" -and $_.tools_list -le $_.clients }).Count -eq 0) "Recovered adapters that issue tools/list must rebind it and never replay the ambiguous mutating tool call" +Assert-True (@($coldSmoke.recovery).Count -eq 11 -and @($coldSmoke.recovery | Where-Object { $_.mode -notin "shutdown_during_recovery", "powershell_fallback_initialize_retry" -and ($_.backends -ne 2 -or $_.pids -ne 2 -or $_.listeners -ne 0 -or $_.recovery_leaders -ne 1) }).Count -eq 0) "Node and PowerShell fallback recovery must each elect exactly one replacement backend and still reach zero listeners" +Assert-True (@($coldSmoke.recovery | Where-Object { $_.mode -like "*ambiguous_tool" -and $_.mutations -eq 1 }).Count -eq 2 -and @($coldSmoke.recovery | Where-Object { $_.mode -notin "shutdown_during_recovery", "generation_changed_recovery", "cleanup_source_changed_recovery", "powershell_fallback_initialize_retry" -and $_.tools_list -le $_.clients }).Count -eq 0) "Recovered adapters that issue tools/list must rebind it and never replay the ambiguous mutating tool call" +Assert-True (@($coldSmoke.recovery | Where-Object { $_.mode -like "*backend_only_read_recovery" -and $_.tools_list -eq 3 -and $_.mutations -eq 0 }).Count -eq 2) "Node and PowerShell fallback bridges must replay an ambiguous read-only request after backend-only recovery" Assert-True (($coldSmoke.recovery | Where-Object mode -eq "shutdown_during_recovery").shutdown_race) "Adapters must exit when STDIO closes during heartbeat recovery" Assert-True (($coldSmoke.recovery | Where-Object mode -eq "generation_changed_recovery").fatal_generation) "A replacement rejected by generation validation must detach its lease and fail the adapter closed" Assert-True (($coldSmoke.recovery | Where-Object mode -eq "cleanup_source_changed_recovery").fatal_cleanup) "A replacement rejected by cleanup-source validation must detach its lease, fail closed, and stop cleanly"