Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 34 additions & 1 deletion src/__tests__/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,20 @@
* Basic test suite for core functionality.
*/

import { describe, it, expect, beforeEach } from 'vitest';
import { describe, it, expect, beforeEach, vi } from 'vitest';
import { Arsenal, BUILTIN_TOOLS, successResult, failResult, createToolContext } from '../arsenal/index.js';
import { FRONTIER_ARSENAL_MILESTONE, SAFE_COMMANDS, TOOL_ADAPTERS, adaptersForFamily, summarizeToolCatalog } from '../arsenal/catalog.js';
import { createKnowledgeBase, createEvasionEngine, CVE_DATABASE, MITRE_TECHNIQUES } from '../stubs/index.js';
import { AGENT_PROMPT_PACKS, FOREFRONT_PRESSURE_LANES, OPERATOR_RUNBOOKS, forefrontPressureForFamily, promptPacksForFamily, runbookForFamily } from '../resources/index.js';
import { OpGeneral } from '../general/index.js';
import { LLMBackbone } from '../llm/index.js';

// Scoped to this file's new per-operator-isolation test (PR #149 review follow-up) — every other
// test here uses provider: 'mock', which never touches this module.
vi.mock('../agent/local-agents.js', () => ({ localAgentChat: vi.fn() }));
import { localAgentChat } from '../agent/local-agents.js';
const localAgentCli = vi.mocked(localAgentChat);

describe('Arsenal', () => {
let arsenal: Arsenal;

Expand Down Expand Up @@ -465,6 +471,33 @@ describe('Wedged-dispatch timeout backstop', () => {
else process.env.T3MP3ST_TASK_TIMEOUT_MS = prev;
}
}, 15000);

// PR #149 review follow-up: spawnOperator() used to hand every operator the SAME shared
// this.llm, so LocalAgentAdapter's Claude Code session-resume state (added for --resume) leaked
// across operators — one operator's session could get resumed with a different operator's
// unrelated task content. Each operator now gets its own LLMBackbone from the same config.
it('spawnOperator() gives each operator its OWN LLMBackbone, not a shared one (no cross-operator Claude session leakage)', async () => {
const mod = await import('../index.js');
const command = new mod.TempestCommand({
name: 'Isolation Op',
llm: { provider: 'local-agent', model: 'claude' },
});

const opA = command.spawnOperator('Recon-Iso', 'recon');
const opB = command.spawnOperator('Scanner-Iso', 'scanner');

expect((opA as any).llm).not.toBe((opB as any).llm);
expect((opA as any).llm).not.toBe(command.llm);

localAgentCli.mockResolvedValueOnce(JSON.stringify({ result: 'a1', session_id: 'sess-A' }));
await (opA as any).llm.chat([{ role: 'user', content: 'hello A' }]);

// Operator B has never talked to Claude before — its first call must NOT resume A's session.
localAgentCli.mockResolvedValueOnce(JSON.stringify({ result: 'b1', session_id: 'sess-B' }));
await (opB as any).llm.chat([{ role: 'user', content: 'hello B' }]);

expect(localAgentCli.mock.calls.at(-1)?.[2]).toMatchObject({ sessionId: undefined });
});
});

describe('Codex account provider', () => {
Expand Down
118 changes: 118 additions & 0 deletions src/__tests__/local-agent-path-resolution.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -283,3 +283,121 @@ describe('spawn call-sites use the resolved path (issue #78 — detected-but-uns
})).resolves.toBe('--no-tools --model openai-codex/gpt-5|long planning prompt');
});
});

/**
* REGRESSION guard (Tier 2 — #139 follow-up, session resume): confirmed empirically against the
* real Claude Code CLI that an unknown/expired --resume id is a hard, fast failure (nonzero exit,
* "No conversation found with session ID: ..." on stderr, no JSON on stdout) — the CLI never falls
* back to a fresh session on its own. localAgentChat does that fallback itself, once. These tests
* drive the REAL function (not mocked) through a fake CLI script so the retry wiring is pinned,
* not just the higher-level LocalAgentAdapter call-shape covered in local-agent-tool-calling.test.ts.
*/
const FAKE_CLI_RESUME_STALE = `#!/bin/sh
cat >/dev/null 2>&1
case "$*" in
*--resume*) echo "No conversation found with session ID: fake" >&2; exit 1 ;;
*) echo '{"result":"fresh ok","session_id":"new-session-123"}' ;;
esac
`;

// Echoes exactly what it received on stdin back to stdout on the fresh (non-resume) path — lets a
// test prove WHICH prompt string a fresh-session retry actually received, not just that it succeeded.
const FAKE_CLI_ECHO_ON_FRESH = `#!/bin/sh
prompt=$(cat)
case "$*" in
*--resume*) echo "No conversation found with session ID: fake" >&2; exit 1 ;;
*) printf '%s' "$prompt" ;;
esac
`;

// A --resume failure that is NOT the stale-session error. If a fallback retry incorrectly fired,
// the fresh (non-resume) branch below would resolve successfully — so a test asserting this
// REJECTS proves no retry happened, not just that the final error text matches.
const FAKE_CLI_NONSTALE_RESUME_ERROR = `#!/bin/sh
cat >/dev/null 2>&1
case "$*" in
*--resume*) echo "connection refused" >&2; exit 1 ;;
*) echo '{"result":"should not be reached — a retry fired for a non-stale error","session_id":"x"}' ;;
esac
`;

describe('localAgentChat — stale Claude session fallback (Tier 2)', () => {
it('retries WITHOUT --resume when the resumed session is stale, and succeeds', async () => {
const home = scratch();
putExe(join(home, '.local', 'bin'), 'claude', FAKE_CLI_RESUME_STALE);
process.env.T3MP3ST_AGENT_HOME = home;
process.env.PATH = '/usr/bin:/bin';

const out = await localAgentChat('claude', 'ping', { sessionId: 'stale-uuid', timeoutMs: 4000 });
expect(JSON.parse(out).result).toBe('fresh ok');
});

it('never adds --resume when no sessionId is supplied (no unnecessary retry path)', async () => {
const home = scratch();
putExe(join(home, '.local', 'bin'), 'claude', FAKE_CLI_RESUME_STALE);
process.env.T3MP3ST_AGENT_HOME = home;
process.env.PATH = '/usr/bin:/bin';

const out = await localAgentChat('claude', 'ping', { timeoutMs: 4000 });
expect(JSON.parse(out).result).toBe('fresh ok');
});

it('a genuine failure with no session in play still rejects (no unnecessary-retry regression)', async () => {
const home = scratch();
putExe(join(home, '.local', 'bin'), 'claude', '#!/bin/sh\ncat >/dev/null 2>&1\necho "boom" >&2\nexit 1\n');
process.env.T3MP3ST_AGENT_HOME = home;
process.env.PATH = '/usr/bin:/bin';

await expect(localAgentChat('claude', 'ping', { timeoutMs: 4000 })).rejects.toThrow(/boom/);
});

it('a non-stale failure with a session in play propagates without a fallback retry (narrowed trigger, PR #149 review)', async () => {
const home = scratch();
putExe(join(home, '.local', 'bin'), 'claude', FAKE_CLI_NONSTALE_RESUME_ERROR);
process.env.T3MP3ST_AGENT_HOME = home;
process.env.PATH = '/usr/bin:/bin';

// If the (old, unnarrowed) fallback fired here, this would RESOLVE with the fresh branch's
// success text instead of rejecting — a rejection proves no retry happened.
await expect(localAgentChat('claude', 'ping', { sessionId: 'whatever', timeoutMs: 4000 }))
.rejects.toThrow(/connection refused/);
});

it('the specific stale-session error still triggers exactly one fallback retry', async () => {
const home = scratch();
putExe(join(home, '.local', 'bin'), 'claude', FAKE_CLI_RESUME_STALE);
process.env.T3MP3ST_AGENT_HOME = home;
process.env.PATH = '/usr/bin:/bin';

const out = await localAgentChat('claude', 'ping', { sessionId: 'stale-uuid', timeoutMs: 4000 });
expect(JSON.parse(out).result).toBe('fresh ok');
});

// PR #149 review (jmagly): the fresh-session retry has no history at all, so it must receive the
// FULL transcript (fallbackPrompt) — never the delta sized for the (failed) resumed attempt.
it('the fresh-session retry receives fallbackPrompt, not the delta prompt of the failed resumed attempt', async () => {
const home = scratch();
putExe(join(home, '.local', 'bin'), 'claude', FAKE_CLI_ECHO_ON_FRESH);
process.env.T3MP3ST_AGENT_HOME = home;
process.env.PATH = '/usr/bin:/bin';

const out = await localAgentChat('claude', 'DELTA-ONLY-CONTENT', {
sessionId: 'stale-uuid',
fallbackPrompt: 'FULL-TRANSCRIPT-CONTENT',
timeoutMs: 4000,
});

expect(out).toBe('FULL-TRANSCRIPT-CONTENT');
expect(out).not.toContain('DELTA-ONLY-CONTENT');
});

it('falls back to the (delta) prompt itself when no fallbackPrompt is supplied', async () => {
const home = scratch();
putExe(join(home, '.local', 'bin'), 'claude', FAKE_CLI_ECHO_ON_FRESH);
process.env.T3MP3ST_AGENT_HOME = home;
process.env.PATH = '/usr/bin:/bin';

const out = await localAgentChat('claude', 'ONLY-PROMPT-CONTENT', { sessionId: 'stale-uuid', timeoutMs: 4000 });
expect(out).toBe('ONLY-PROMPT-CONTENT');
});
});
159 changes: 158 additions & 1 deletion src/__tests__/local-agent-tool-calling.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ vi.mock('fs/promises', async (orig) => {
return { ...actual, readFile: vi.fn() };
});

import { parseTextToolCalls, LLMBackbone } from '../llm/index.js';
import { parseTextToolCalls, LLMBackbone, type LLMMessage } from '../llm/index.js';
import { localAgentChat } from '../agent/local-agents.js';
import { readFile } from 'fs/promises';
const cli = vi.mocked(localAgentChat);
Expand Down Expand Up @@ -195,6 +195,163 @@ describe('Claude local-agent usage accounting (#139)', () => {
});
});

describe('Claude local-agent session resume (Tier 2)', () => {
it('the first call on a fresh adapter passes no sessionId', async () => {
cli.mockResolvedValueOnce(JSON.stringify({ result: 'first', session_id: 'sess-1' }));
await claudeBackbone().chat([{ role: 'user', content: 'hello' }]);
expect(cli.mock.calls.at(-1)?.[2]).toMatchObject({ sessionId: undefined });
});

it('captures session_id from the response and passes it as --resume on the next call', async () => {
const be = claudeBackbone();
cli.mockResolvedValueOnce(JSON.stringify({ result: 'first', session_id: 'sess-1' }));
await be.chat([{ role: 'user', content: 'hello' }]);

cli.mockResolvedValueOnce(JSON.stringify({ result: 'second', session_id: 'sess-1' }));
await be.chat([{ role: 'user', content: 'again' }]);

expect(cli.mock.calls.at(-1)?.[2]).toMatchObject({ sessionId: 'sess-1' });
});

it('resetLocalAgentSession() drops the tracked session so the next call starts fresh', async () => {
const be = claudeBackbone();
cli.mockResolvedValueOnce(JSON.stringify({ result: 'first', session_id: 'sess-1' }));
await be.chat([{ role: 'user', content: 'hello' }]);

be.resetLocalAgentSession();

cli.mockResolvedValueOnce(JSON.stringify({ result: 'second', session_id: 'sess-2' }));
await be.chat([{ role: 'user', content: 'again' }]);

expect(cli.mock.calls.at(-1)?.[2]).toMatchObject({ sessionId: undefined });
});

it('a separate LLMBackbone instance never inherits another instance\'s session (per-operator isolation)', async () => {
const opA = claudeBackbone();
cli.mockResolvedValueOnce(JSON.stringify({ result: 'a1', session_id: 'sess-a' }));
await opA.chat([{ role: 'user', content: 'hello' }]);

const opB = claudeBackbone();
cli.mockResolvedValueOnce(JSON.stringify({ result: 'b1', session_id: 'sess-b' }));
await opB.chat([{ role: 'user', content: 'hello' }]);

expect(cli.mock.calls.at(-1)?.[2]).toMatchObject({ sessionId: undefined });
});

it('a resumed session with no usable id in the envelope keeps the PRIOR session for the next call', async () => {
const be = claudeBackbone();
cli.mockResolvedValueOnce(JSON.stringify({ result: 'first', session_id: 'sess-1' }));
await be.chat([{ role: 'user', content: 'hello' }]);

cli.mockResolvedValueOnce(JSON.stringify({ result: 'second' })); // no session_id this time
await be.chat([{ role: 'user', content: 'again' }]);

cli.mockResolvedValueOnce(JSON.stringify({ result: 'third', session_id: 'sess-1' }));
await be.chat([{ role: 'user', content: 'once more' }]);

expect(cli.mock.calls.at(-1)?.[2]).toMatchObject({ sessionId: 'sess-1' });
});

it('non-claude local agents (codex CLI via local-agent provider) never receive a sessionId', async () => {
cli.mockResolvedValueOnce('done');
await localBackbone().chat([{ role: 'user', content: 'hello' }]);
expect(cli.mock.calls.at(-1)?.[2]).toMatchObject({ sessionId: undefined });
});

// PR #149 review (jmagly): formatPrompt() always serialized the WHOLE messages array, so a
// resumed call duplicated the transcript into a session that already had it — the exact resend
// this feature exists to eliminate. These pin the fix: same-array (same task) reuse sends only
// the NEW messages once a session exists; a different array (a new task) still gets everything.
it('a resumed call on the SAME growing messages array sends only the delta, not the full transcript', async () => {
const be = claudeBackbone();
const messages: LLMMessage[] = [
{ role: 'system', content: 'SYSTEM-PROMPT-MARKER' },
{ role: 'user', content: 'USER-TASK-MARKER' },
];

cli.mockResolvedValueOnce(JSON.stringify({
result: '{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}}]}',
session_id: 'sess-1',
}));
await be.chatWithTools(messages, TOOLS as never);

// AgentLoop's real growth pattern: push the assistant turn + tool result onto the SAME array.
messages.push({ role: 'assistant', content: 'reasoning', toolCalls: [{ id: 'c1', name: 'nmap_scan', arguments: { target: 'x' } }] });
messages.push({ role: 'tool', content: 'TOOL-RESULT-MARKER', toolCallId: 'c1', name: 'nmap_scan' });

cli.mockResolvedValueOnce(JSON.stringify({ result: 'Final debrief.', session_id: 'sess-1' }));
await be.chatWithTools(messages, TOOLS as never);

const secondPrompt = String(cli.mock.calls.at(-1)?.[1] || '');
expect(secondPrompt).not.toContain('SYSTEM-PROMPT-MARKER');
expect(secondPrompt).not.toContain('USER-TASK-MARKER');
expect(secondPrompt).toContain('TOOL-RESULT-MARKER');
expect(secondPrompt).toContain('ACTION CONTRACT'); // tool contract retained on the delta call
});

it('a NEW task (different messages array) still gets the full prompt even though the session resumes', async () => {
const be = claudeBackbone();
cli.mockResolvedValueOnce(JSON.stringify({ result: 'first', session_id: 'sess-1' }));
await be.chat([{ role: 'system', content: 'TASK-ONE-SYSTEM' }, { role: 'user', content: 'TASK-ONE-USER' }]);

// A brand-new task builds a brand-new array (AgentLoop.run() does this per task) — the
// resumed session has never seen this content, so it must go out in full despite --resume.
cli.mockResolvedValueOnce(JSON.stringify({ result: 'second', session_id: 'sess-1' }));
await be.chat([{ role: 'system', content: 'TASK-TWO-SYSTEM' }, { role: 'user', content: 'TASK-TWO-USER' }]);

const secondCall = cli.mock.calls.at(-1);
const secondPrompt = String(secondCall?.[1] || '');
expect(secondPrompt).toContain('TASK-TWO-SYSTEM');
expect(secondPrompt).toContain('TASK-TWO-USER');
expect(secondCall?.[2]).toMatchObject({ sessionId: 'sess-1', fallbackPrompt: undefined });
});

it('passes fallbackPrompt (the FULL transcript) alongside a delta send, so a stale-session retry never gets the delta', async () => {
const be = claudeBackbone();
const messages: LLMMessage[] = [{ role: 'system', content: 'S' }, { role: 'user', content: 'U' }];

cli.mockResolvedValueOnce(JSON.stringify({ result: 'first', session_id: 'sess-1' }));
await be.chat(messages);
expect(cli.mock.calls.at(-1)?.[2]).toMatchObject({ fallbackPrompt: undefined }); // nothing to fall back from yet

messages.push({ role: 'assistant', content: 'ok' });
cli.mockResolvedValueOnce(JSON.stringify({ result: 'second', session_id: 'sess-1' }));
await be.chat(messages);

const call = cli.mock.calls.at(-1);
const deltaPrompt = String(call?.[1] || '');
const fallbackPrompt = String((call?.[2] as { fallbackPrompt?: string } | undefined)?.fallbackPrompt || '');
expect(deltaPrompt).not.toContain('### SYSTEM\nS'); // delta omits the first task's system/user content
expect(deltaPrompt).not.toContain('### USER\nU');
expect(fallbackPrompt).toContain('### SYSTEM\nS'); // fallback carries the FULL transcript a fresh session would need
expect(fallbackPrompt).toContain('### USER\nU');
});

// PR #149 review follow-up: a real envelope's promptTokens on a resumed call reflects the WHOLE
// cached conversation (input + cache_creation + cache_read), not the wire size of the delta —
// confirmed empirically. The character-estimate fallback (used only when the envelope fails to
// parse) must approximate that same full-conversation size, or it silently undercounts the
// budget check by an order of magnitude on a delta call.
it('the usage estimate fallback reflects the FULL transcript, not the delta, on a mid-session parse failure', async () => {
const be = claudeBackbone();
const messages: LLMMessage[] = [
{ role: 'system', content: 'S'.repeat(2000) },
{ role: 'user', content: 'U'.repeat(2000) },
];

cli.mockResolvedValueOnce(JSON.stringify({ result: 'first', session_id: 'sess-1' }));
await be.chat(messages);

messages.push({ role: 'assistant', content: 'ok' });
cli.mockResolvedValueOnce('not json'); // envelope parse failure -> falls to the estimate
const res = await be.chat(messages);

// A delta-only estimate would be tiny (just "ok" plus the preamble/tool contract). The full
// ~4000 chars of original system/user content must show up in the estimate too.
expect(res.usage?.promptTokens).toBeGreaterThan(900);
});
});

describe('codex backbone surfaces toolCalls (guards the CodexAdapter half of the fix)', () => {
it('returns toolCalls when codex emits the contract', async () => {
fileRead.mockResolvedValueOnce('```json\n{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}}]}\n```');
Expand Down
Loading
Loading