Skip to content

Commit f256d69

Browse files
committed
Prove the conversation path end to end with a fixture CLI that edits for real
e2e/conversation-fidelity.spec.ts runs through the real app, main process and claude -p adapter against a fake Claude CLI shaped like 2.1.302 that edits a disposable repository and counts what it runs. Golden path: a reported plan, an approval, a real edit and check, the final answer, Save to Notes twice (same note, next revision), Open, and a restart that restores identical items with no re-run. Variants: a denial (bytes unchanged, nothing run), a failing check, and Stop while a check runs. Assertions are on bytes, counts, outcomes and revisions. These cover behaviour CF-1 to CF-7 built, so there was no red phase. Refs #281 Coven-Harness: Claude Code / claude-opus-5-5
1 parent f6976df commit f256d69

1 file changed

Lines changed: 179 additions & 0 deletions

File tree

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
import { test, expect, type ElectronApplication, type Page } from "@playwright/test";
2+
import fs from "node:fs";
3+
import path from "node:path";
4+
import { currentRow, items, launch, seedHome, tid } from "./support";
5+
6+
/**
7+
* The integration path for tracker #273 (docs/conversation-fidelity.md): a request runs through the
8+
* real app, main process and `claude -p` adapter against a fake Claude CLI that edits a disposable
9+
* repository for real and counts what it executes. Assertions are on file bytes, execution counts,
10+
* typed outcomes and note revisions, not on success text. Not live-provider acceptance.
11+
*/
12+
13+
type Variant = "golden" | "deny" | "fail" | "stop";
14+
const BEFORE = "hello\n";
15+
const AFTER = "hello, fidelity\n";
16+
17+
/** A stream-json CLI shaped like Claude Code 2.1.302: a TodoWrite plan, an Edit behind the stdio permission prompt, a check, an answer. */
18+
function fakeClaude(repo: string, counts: string, variant: Variant): string {
19+
const file = path.join(repo, "greeting.txt");
20+
const check = variant === "fail" ? "grep -q missing greeting.txt" : "grep -q fidelity greeting.txt";
21+
return `#!${process.execPath}
22+
const fs = require('node:fs');
23+
const { spawnSync } = require('node:child_process');
24+
if (process.argv.includes('--version')) { console.log('2.1.302 (Claude Code)'); process.exit(0); }
25+
const emit = (x) => console.log(JSON.stringify(x));
26+
const count = (key) => { const c = fs.existsSync(${JSON.stringify(counts)}) ? JSON.parse(fs.readFileSync(${JSON.stringify(counts)}, 'utf8')) : {}; c[key] = (c[key] || 0) + 1; fs.writeFileSync(${JSON.stringify(counts)}, JSON.stringify(c)); };
27+
const todos = (a, b) => emit({ type: 'assistant', message: { id: 'm-todo-' + a + b, content: [{ type: 'tool_use', id: 'todo-' + a + b, name: 'TodoWrite', input: { todos: [{ content: 'Edit greeting.txt', status: a, activeForm: 'Editing' }, { content: 'Check the file', status: b, activeForm: 'Checking' }] } }] } });
28+
let pending = null;
29+
const rl = require('node:readline').createInterface({ input: process.stdin });
30+
rl.on('close', () => process.exit(0));
31+
rl.on('line', (line) => {
32+
const msg = JSON.parse(line);
33+
if (msg.type === 'control_response' && pending) { const go = pending; pending = null; go(msg.response.response.behavior === 'allow'); return; }
34+
if (msg.type !== 'user') return;
35+
emit({ type: 'system', subtype: 'init', session_id: 'fidelity-session', claude_code_version: '2.1.302', model: 'fixture' });
36+
todos('in_progress', 'pending');
37+
emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'todo-in_progresspending', content: 'ok' }] } });
38+
emit({ type: 'assistant', message: { id: 'm-say', content: [{ type: 'text', text: 'Editing the greeting.' }] } });
39+
const edit = { file_path: ${JSON.stringify(file)}, old_string: 'hello', new_string: 'hello, fidelity' };
40+
emit({ type: 'assistant', message: { id: 'm-edit', content: [{ type: 'tool_use', id: 'edit-1', name: 'Edit', input: edit }] } });
41+
pending = (allowed) => {
42+
if (!allowed) {
43+
emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'edit-1', content: 'The user denied this action in Modex.', is_error: true }] } });
44+
emit({ type: 'assistant', message: { id: 'm-final', content: [{ type: 'text', text: 'Left the file as it was.' }] } });
45+
emit({ type: 'result', subtype: 'success', is_error: false, session_id: 'fidelity-session', result: 'Left the file as it was.', num_turns: 3 });
46+
return;
47+
}
48+
count('edits');
49+
fs.writeFileSync(${JSON.stringify(file)}, ${JSON.stringify(AFTER)});
50+
emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'edit-1', content: 'The file was updated.' }] } });
51+
todos('completed', 'in_progress');
52+
emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'todo-completedin_progress', content: 'ok' }] } });
53+
emit({ type: 'assistant', message: { id: 'm-check', content: [{ type: 'tool_use', id: 'check-1', name: 'Bash', input: { command: ${JSON.stringify(check)} } }] } });
54+
count('checksStarted');
55+
const finish = () => {
56+
const run = spawnSync('/bin/sh', ['-c', ${JSON.stringify(check)}], { cwd: ${JSON.stringify(repo)} });
57+
count('checksFinished');
58+
const ok = run.status === 0;
59+
emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'check-1', content: ok ? '' : 'exit ' + run.status, is_error: !ok }] } });
60+
if (ok) { todos('completed', 'completed'); emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'todo-completedcompleted', content: 'ok' }] } }); }
61+
const answer = ok ? 'Changed the greeting; the check passed.' : 'Changed the greeting; the check failed.';
62+
emit({ type: 'assistant', message: { id: 'm-final', content: [{ type: 'text', text: answer }] } });
63+
emit({ type: 'result', subtype: 'success', is_error: false, session_id: 'fidelity-session', result: answer, num_turns: 6 });
64+
};
65+
if (${JSON.stringify(variant)} === 'stop') setTimeout(finish, 60000); else finish();
66+
};
67+
emit({ type: 'control_request', request_id: 'perm-1', request: { subtype: 'can_use_tool', tool_name: 'Edit', tool_use_id: 'edit-1', input: edit } });
68+
});
69+
`;
70+
}
71+
72+
async function open(variant: Variant): Promise<{ app: ElectronApplication; page: Page; home: string; repo: string; counts: string; read: () => Record<string, number> }> {
73+
const { home, repo } = seedHome({ default_backend: "claude" });
74+
fs.writeFileSync(path.join(repo, "greeting.txt"), BEFORE);
75+
const counts = path.join(home, "fixture-counts.json");
76+
const cli = path.join(home, "claude-fidelity.cjs");
77+
fs.writeFileSync(cli, fakeClaude(repo, counts, variant), { mode: 0o755 });
78+
const statePath = path.join(home, "app/state.json");
79+
const state = JSON.parse(fs.readFileSync(statePath, "utf8"));
80+
state.settings.claude_bin = cli;
81+
fs.writeFileSync(statePath, JSON.stringify(state));
82+
const { app, page } = await launch(home);
83+
await page.keyboard.press("Meta+n");
84+
await expect(tid(page, "composer-input")).toBeFocused();
85+
await tid(page, "composer-input").fill("Change the greeting to mention fidelity and check it");
86+
await page.keyboard.press("Meta+Enter");
87+
await expect(tid(page, "thread-view")).toBeVisible({ timeout: 15_000 });
88+
const read = () => (fs.existsSync(counts) ? JSON.parse(fs.readFileSync(counts, "utf8")) as Record<string, number> : {});
89+
return { app, page, home, repo, counts, read };
90+
}
91+
92+
const approval = (page: Page) => items(page, "approval").filter({ hasText: "Edit" });
93+
const close = async (run: { app: ElectronApplication; home: string; repo: string }) => {
94+
await run.app.close().catch(() => {});
95+
fs.rmSync(run.home, { recursive: true, force: true });
96+
fs.rmSync(run.repo, { recursive: true, force: true });
97+
};
98+
99+
test("golden path: plan, approval, a real edit and check, the answer, Save to Notes, Open, and a restart that restores without re-running", async () => {
100+
let run = await open("golden");
101+
try {
102+
await approval(run.page).getByRole("button", { name: "Approve" }).click();
103+
await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 });
104+
expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(AFTER);
105+
expect(run.read()).toEqual({ edits: 1, checksStarted: 1, checksFinished: 1 });
106+
// Typed outcomes, the reported plan and the final answer, as recorded rather than as prose says.
107+
await expect(tid(items(run.page, "plan"), "plan-step")).toHaveText(["Reported done Edit greeting.txt", "Reported done Check the file"]);
108+
await tid(run.page, "turn-label").last().click();
109+
await expect(tid(items(run.page, "tool").filter({ hasText: "edit" }).first(), "tool-meta")).toContainText("done");
110+
const check = items(run.page, "tool").filter({ hasText: "grep -q fidelity" });
111+
await tid(check, "item-toggle").click();
112+
await expect(tid(check, "tool-exit")).toHaveText("Exit code not reported");
113+
await expect(run.page.locator('.turn-content > div[data-final="true"]').last()).toContainText("Changed the greeting; the check passed.");
114+
// Save to Notes, again (same note, next revision), then Open.
115+
await tid(run.page, "save-task-note").last().click();
116+
await expect(tid(run.page, "task-note-saved").last()).toContainText("Saved to Notes · Task:");
117+
const notes = async () => (await run.page.evaluate(() => window.modex!.invoke("space:list", undefined))) as { id: string; title: string; markdown: string; revision: number }[];
118+
const first = (await notes()).find((p) => p.title.startsWith("Task:"))!;
119+
expect(first.markdown).toContain("- [x] Edit greeting.txt (reported done)");
120+
expect(first.markdown).toContain("`greeting.txt`");
121+
expect(first.markdown).toContain("`grep -q fidelity greeting.txt` — succeeded, exit code not reported");
122+
expect(first.markdown).not.toContain(run.repo);
123+
await tid(run.page, "save-task-note").last().click();
124+
await expect(tid(run.page, "task-note-saved").last()).toContainText("Updated in Notes · Task:");
125+
const second = (await notes()).filter((p) => p.title.startsWith("Task:"));
126+
expect(second.map((p) => [p.id, p.revision])).toEqual([[first.id, first.revision + 1]]);
127+
// Open the note: the record reads in Notes as saved.
128+
const threadId = (await currentRow(run.page).getAttribute("data-thread-id"))!;
129+
const before = await run.page.evaluate((id) => window.modex!.invoke("thread:items", { threadId: id }), threadId);
130+
await tid(run.page, "task-note-saved").last().getByRole("button", { name: "Open" }).click();
131+
await expect(run.page.getByText(`From modex thread ${threadId}.`)).toBeVisible();
132+
// Restart: the same items come back, and nothing runs again.
133+
await run.app.close();
134+
const relaunched = await launch(run.home);
135+
run = { ...run, app: relaunched.app, page: relaunched.page };
136+
const after = await run.page.evaluate((id) => window.modex!.invoke("thread:items", { threadId: id }), threadId);
137+
expect(after).toEqual(before);
138+
expect(run.read()).toEqual({ edits: 1, checksStarted: 1, checksFinished: 1 });
139+
expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(AFTER);
140+
} finally { await close(run); }
141+
});
142+
143+
test("a denial leaves the file as it was and runs nothing", async () => {
144+
const run = await open("deny");
145+
try {
146+
await approval(run.page).getByRole("button", { name: "Deny" }).click();
147+
await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 });
148+
expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(BEFORE);
149+
expect(run.read()).toEqual({});
150+
await tid(run.page, "turn-label").last().click();
151+
await expect(tid(items(run.page, "tool").filter({ hasText: "edit" }).first(), "tool-meta")).toContainText("denied");
152+
} finally { await close(run); }
153+
});
154+
155+
test("a failing check is a failed command, whatever the reply says", async () => {
156+
const run = await open("fail");
157+
try {
158+
await approval(run.page).getByRole("button", { name: "Approve" }).click();
159+
await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 });
160+
const check = items(run.page, "tool").filter({ hasText: "grep -q missing" });
161+
await expect(check).toBeVisible();
162+
await expect(tid(check, "tool-meta")).toContainText("failed");
163+
await expect(tid(items(run.page, "plan"), "plan-step")).toHaveText(["Reported done Edit greeting.txt", "In progress Check the file"]);
164+
} finally { await close(run); }
165+
});
166+
167+
test("Stop while a check runs ends it as stopped; the check never reports, and nothing is retried", async () => {
168+
const run = await open("stop");
169+
try {
170+
await approval(run.page).getByRole("button", { name: "Approve" }).click();
171+
await expect.poll(() => run.read().checksStarted ?? 0).toBe(1);
172+
await tid(run.page, "stop").click();
173+
await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 });
174+
const check = items(run.page, "tool").filter({ hasText: "grep -q fidelity" });
175+
await expect(tid(check, "tool-meta")).toContainText("stopped");
176+
expect(run.read()).toEqual({ edits: 1, checksStarted: 1 });
177+
await expect(run.page.getByText("Stopped.")).toBeVisible();
178+
} finally { await close(run); }
179+
});

0 commit comments

Comments
 (0)