|
| 1 | +import { test, expect, type ElectronApplication, type Page } from "@playwright/test"; |
| 2 | +import fs from "node:fs"; |
| 3 | +import path from "node:path"; |
| 4 | +import { currentRow, items, launch, seedHome, tid } from "./support"; |
| 5 | + |
| 6 | +/** |
| 7 | + * The integration path for tracker #273 (docs/conversation-fidelity.md): a request runs through the |
| 8 | + * real app, main process and `claude -p` adapter against a fake Claude CLI that edits a disposable |
| 9 | + * repository for real and counts what it executes. Assertions are on file bytes, execution counts, |
| 10 | + * typed outcomes and note revisions, not on success text. Not live-provider acceptance. |
| 11 | + */ |
| 12 | + |
| 13 | +type Variant = "golden" | "deny" | "fail" | "stop"; |
| 14 | +const BEFORE = "hello\n"; |
| 15 | +const AFTER = "hello, fidelity\n"; |
| 16 | + |
| 17 | +/** A stream-json CLI shaped like Claude Code 2.1.302: a TodoWrite plan, an Edit behind the stdio permission prompt, a check, an answer. */ |
| 18 | +function fakeClaude(repo: string, counts: string, variant: Variant): string { |
| 19 | + const file = path.join(repo, "greeting.txt"); |
| 20 | + const check = variant === "fail" ? "grep -q missing greeting.txt" : "grep -q fidelity greeting.txt"; |
| 21 | + return `#!${process.execPath} |
| 22 | +const fs = require('node:fs'); |
| 23 | +const { spawnSync } = require('node:child_process'); |
| 24 | +if (process.argv.includes('--version')) { console.log('2.1.302 (Claude Code)'); process.exit(0); } |
| 25 | +const emit = (x) => console.log(JSON.stringify(x)); |
| 26 | +const count = (key) => { const c = fs.existsSync(${JSON.stringify(counts)}) ? JSON.parse(fs.readFileSync(${JSON.stringify(counts)}, 'utf8')) : {}; c[key] = (c[key] || 0) + 1; fs.writeFileSync(${JSON.stringify(counts)}, JSON.stringify(c)); }; |
| 27 | +const todos = (a, b) => emit({ type: 'assistant', message: { id: 'm-todo-' + a + b, content: [{ type: 'tool_use', id: 'todo-' + a + b, name: 'TodoWrite', input: { todos: [{ content: 'Edit greeting.txt', status: a, activeForm: 'Editing' }, { content: 'Check the file', status: b, activeForm: 'Checking' }] } }] } }); |
| 28 | +let pending = null; |
| 29 | +const rl = require('node:readline').createInterface({ input: process.stdin }); |
| 30 | +rl.on('close', () => process.exit(0)); |
| 31 | +rl.on('line', (line) => { |
| 32 | + const msg = JSON.parse(line); |
| 33 | + if (msg.type === 'control_response' && pending) { const go = pending; pending = null; go(msg.response.response.behavior === 'allow'); return; } |
| 34 | + if (msg.type !== 'user') return; |
| 35 | + emit({ type: 'system', subtype: 'init', session_id: 'fidelity-session', claude_code_version: '2.1.302', model: 'fixture' }); |
| 36 | + todos('in_progress', 'pending'); |
| 37 | + emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'todo-in_progresspending', content: 'ok' }] } }); |
| 38 | + emit({ type: 'assistant', message: { id: 'm-say', content: [{ type: 'text', text: 'Editing the greeting.' }] } }); |
| 39 | + const edit = { file_path: ${JSON.stringify(file)}, old_string: 'hello', new_string: 'hello, fidelity' }; |
| 40 | + emit({ type: 'assistant', message: { id: 'm-edit', content: [{ type: 'tool_use', id: 'edit-1', name: 'Edit', input: edit }] } }); |
| 41 | + pending = (allowed) => { |
| 42 | + if (!allowed) { |
| 43 | + emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'edit-1', content: 'The user denied this action in Modex.', is_error: true }] } }); |
| 44 | + emit({ type: 'assistant', message: { id: 'm-final', content: [{ type: 'text', text: 'Left the file as it was.' }] } }); |
| 45 | + emit({ type: 'result', subtype: 'success', is_error: false, session_id: 'fidelity-session', result: 'Left the file as it was.', num_turns: 3 }); |
| 46 | + return; |
| 47 | + } |
| 48 | + count('edits'); |
| 49 | + fs.writeFileSync(${JSON.stringify(file)}, ${JSON.stringify(AFTER)}); |
| 50 | + emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'edit-1', content: 'The file was updated.' }] } }); |
| 51 | + todos('completed', 'in_progress'); |
| 52 | + emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'todo-completedin_progress', content: 'ok' }] } }); |
| 53 | + emit({ type: 'assistant', message: { id: 'm-check', content: [{ type: 'tool_use', id: 'check-1', name: 'Bash', input: { command: ${JSON.stringify(check)} } }] } }); |
| 54 | + count('checksStarted'); |
| 55 | + const finish = () => { |
| 56 | + const run = spawnSync('/bin/sh', ['-c', ${JSON.stringify(check)}], { cwd: ${JSON.stringify(repo)} }); |
| 57 | + count('checksFinished'); |
| 58 | + const ok = run.status === 0; |
| 59 | + emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'check-1', content: ok ? '' : 'exit ' + run.status, is_error: !ok }] } }); |
| 60 | + if (ok) { todos('completed', 'completed'); emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'todo-completedcompleted', content: 'ok' }] } }); } |
| 61 | + const answer = ok ? 'Changed the greeting; the check passed.' : 'Changed the greeting; the check failed.'; |
| 62 | + emit({ type: 'assistant', message: { id: 'm-final', content: [{ type: 'text', text: answer }] } }); |
| 63 | + emit({ type: 'result', subtype: 'success', is_error: false, session_id: 'fidelity-session', result: answer, num_turns: 6 }); |
| 64 | + }; |
| 65 | + if (${JSON.stringify(variant)} === 'stop') setTimeout(finish, 60000); else finish(); |
| 66 | + }; |
| 67 | + emit({ type: 'control_request', request_id: 'perm-1', request: { subtype: 'can_use_tool', tool_name: 'Edit', tool_use_id: 'edit-1', input: edit } }); |
| 68 | +}); |
| 69 | +`; |
| 70 | +} |
| 71 | + |
| 72 | +async function open(variant: Variant): Promise<{ app: ElectronApplication; page: Page; home: string; repo: string; counts: string; read: () => Record<string, number> }> { |
| 73 | + const { home, repo } = seedHome({ default_backend: "claude" }); |
| 74 | + fs.writeFileSync(path.join(repo, "greeting.txt"), BEFORE); |
| 75 | + const counts = path.join(home, "fixture-counts.json"); |
| 76 | + const cli = path.join(home, "claude-fidelity.cjs"); |
| 77 | + fs.writeFileSync(cli, fakeClaude(repo, counts, variant), { mode: 0o755 }); |
| 78 | + const statePath = path.join(home, "app/state.json"); |
| 79 | + const state = JSON.parse(fs.readFileSync(statePath, "utf8")); |
| 80 | + state.settings.claude_bin = cli; |
| 81 | + fs.writeFileSync(statePath, JSON.stringify(state)); |
| 82 | + const { app, page } = await launch(home); |
| 83 | + await page.keyboard.press("Meta+n"); |
| 84 | + await expect(tid(page, "composer-input")).toBeFocused(); |
| 85 | + await tid(page, "composer-input").fill("Change the greeting to mention fidelity and check it"); |
| 86 | + await page.keyboard.press("Meta+Enter"); |
| 87 | + await expect(tid(page, "thread-view")).toBeVisible({ timeout: 15_000 }); |
| 88 | + const read = () => (fs.existsSync(counts) ? JSON.parse(fs.readFileSync(counts, "utf8")) as Record<string, number> : {}); |
| 89 | + return { app, page, home, repo, counts, read }; |
| 90 | +} |
| 91 | + |
| 92 | +const approval = (page: Page) => items(page, "approval").filter({ hasText: "Edit" }); |
| 93 | +const close = async (run: { app: ElectronApplication; home: string; repo: string }) => { |
| 94 | + await run.app.close().catch(() => {}); |
| 95 | + fs.rmSync(run.home, { recursive: true, force: true }); |
| 96 | + fs.rmSync(run.repo, { recursive: true, force: true }); |
| 97 | +}; |
| 98 | + |
| 99 | +test("golden path: plan, approval, a real edit and check, the answer, Save to Notes, Open, and a restart that restores without re-running", async () => { |
| 100 | + let run = await open("golden"); |
| 101 | + try { |
| 102 | + await approval(run.page).getByRole("button", { name: "Approve" }).click(); |
| 103 | + await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 }); |
| 104 | + expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(AFTER); |
| 105 | + expect(run.read()).toEqual({ edits: 1, checksStarted: 1, checksFinished: 1 }); |
| 106 | + // Typed outcomes, the reported plan and the final answer, as recorded rather than as prose says. |
| 107 | + await expect(tid(items(run.page, "plan"), "plan-step")).toHaveText(["Reported done Edit greeting.txt", "Reported done Check the file"]); |
| 108 | + await tid(run.page, "turn-label").last().click(); |
| 109 | + await expect(tid(items(run.page, "tool").filter({ hasText: "edit" }).first(), "tool-meta")).toContainText("done"); |
| 110 | + const check = items(run.page, "tool").filter({ hasText: "grep -q fidelity" }); |
| 111 | + await tid(check, "item-toggle").click(); |
| 112 | + await expect(tid(check, "tool-exit")).toHaveText("Exit code not reported"); |
| 113 | + await expect(run.page.locator('.turn-content > div[data-final="true"]').last()).toContainText("Changed the greeting; the check passed."); |
| 114 | + // Save to Notes, again (same note, next revision), then Open. |
| 115 | + await tid(run.page, "save-task-note").last().click(); |
| 116 | + await expect(tid(run.page, "task-note-saved").last()).toContainText("Saved to Notes · Task:"); |
| 117 | + const notes = async () => (await run.page.evaluate(() => window.modex!.invoke("space:list", undefined))) as { id: string; title: string; markdown: string; revision: number }[]; |
| 118 | + const first = (await notes()).find((p) => p.title.startsWith("Task:"))!; |
| 119 | + expect(first.markdown).toContain("- [x] Edit greeting.txt (reported done)"); |
| 120 | + expect(first.markdown).toContain("`greeting.txt`"); |
| 121 | + expect(first.markdown).toContain("`grep -q fidelity greeting.txt` — succeeded, exit code not reported"); |
| 122 | + expect(first.markdown).not.toContain(run.repo); |
| 123 | + await tid(run.page, "save-task-note").last().click(); |
| 124 | + await expect(tid(run.page, "task-note-saved").last()).toContainText("Updated in Notes · Task:"); |
| 125 | + const second = (await notes()).filter((p) => p.title.startsWith("Task:")); |
| 126 | + expect(second.map((p) => [p.id, p.revision])).toEqual([[first.id, first.revision + 1]]); |
| 127 | + // Open the note: the record reads in Notes as saved. |
| 128 | + const threadId = (await currentRow(run.page).getAttribute("data-thread-id"))!; |
| 129 | + const before = await run.page.evaluate((id) => window.modex!.invoke("thread:items", { threadId: id }), threadId); |
| 130 | + await tid(run.page, "task-note-saved").last().getByRole("button", { name: "Open" }).click(); |
| 131 | + await expect(run.page.getByText(`From modex thread ${threadId}.`)).toBeVisible(); |
| 132 | + // Restart: the same items come back, and nothing runs again. |
| 133 | + await run.app.close(); |
| 134 | + const relaunched = await launch(run.home); |
| 135 | + run = { ...run, app: relaunched.app, page: relaunched.page }; |
| 136 | + const after = await run.page.evaluate((id) => window.modex!.invoke("thread:items", { threadId: id }), threadId); |
| 137 | + expect(after).toEqual(before); |
| 138 | + expect(run.read()).toEqual({ edits: 1, checksStarted: 1, checksFinished: 1 }); |
| 139 | + expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(AFTER); |
| 140 | + } finally { await close(run); } |
| 141 | +}); |
| 142 | + |
| 143 | +test("a denial leaves the file as it was and runs nothing", async () => { |
| 144 | + const run = await open("deny"); |
| 145 | + try { |
| 146 | + await approval(run.page).getByRole("button", { name: "Deny" }).click(); |
| 147 | + await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 }); |
| 148 | + expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(BEFORE); |
| 149 | + expect(run.read()).toEqual({}); |
| 150 | + await tid(run.page, "turn-label").last().click(); |
| 151 | + await expect(tid(items(run.page, "tool").filter({ hasText: "edit" }).first(), "tool-meta")).toContainText("denied"); |
| 152 | + } finally { await close(run); } |
| 153 | +}); |
| 154 | + |
| 155 | +test("a failing check is a failed command, whatever the reply says", async () => { |
| 156 | + const run = await open("fail"); |
| 157 | + try { |
| 158 | + await approval(run.page).getByRole("button", { name: "Approve" }).click(); |
| 159 | + await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 }); |
| 160 | + const check = items(run.page, "tool").filter({ hasText: "grep -q missing" }); |
| 161 | + await expect(check).toBeVisible(); |
| 162 | + await expect(tid(check, "tool-meta")).toContainText("failed"); |
| 163 | + await expect(tid(items(run.page, "plan"), "plan-step")).toHaveText(["Reported done Edit greeting.txt", "In progress Check the file"]); |
| 164 | + } finally { await close(run); } |
| 165 | +}); |
| 166 | + |
| 167 | +test("Stop while a check runs ends it as stopped; the check never reports, and nothing is retried", async () => { |
| 168 | + const run = await open("stop"); |
| 169 | + try { |
| 170 | + await approval(run.page).getByRole("button", { name: "Approve" }).click(); |
| 171 | + await expect.poll(() => run.read().checksStarted ?? 0).toBe(1); |
| 172 | + await tid(run.page, "stop").click(); |
| 173 | + await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 }); |
| 174 | + const check = items(run.page, "tool").filter({ hasText: "grep -q fidelity" }); |
| 175 | + await expect(tid(check, "tool-meta")).toContainText("stopped"); |
| 176 | + expect(run.read()).toEqual({ edits: 1, checksStarted: 1 }); |
| 177 | + await expect(run.page.getByText("Stopped.")).toBeVisible(); |
| 178 | + } finally { await close(run); } |
| 179 | +}); |
0 commit comments