Skip to content

Commit 2c7a68f

Browse files
committed
Merge main (CF-8 first part #294) into codex-websearch-done
Coven-Harness: Claude Code / claude-opus-5-5
2 parents d325b60 + d3dfe76 commit 2c7a68f

4 files changed

Lines changed: 313 additions & 2 deletions

File tree

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
import { test, expect, type ElectronApplication, type Page } from "@playwright/test";
2+
import fs from "node:fs";
3+
import path from "node:path";
4+
import { currentRow, items, launch, seedHome, tid } from "./support";
5+
6+
/**
7+
* The integration path for tracker #273 (docs/conversation-fidelity.md): a request runs through the
8+
* real app, main process and `claude -p` adapter against a fake Claude CLI that edits a disposable
9+
* repository for real and counts what it executes. Assertions are on file bytes, execution counts,
10+
* typed outcomes and note revisions, not on success text. Not live-provider acceptance.
11+
*/
12+
13+
type Variant = "golden" | "deny" | "fail" | "stop";
14+
const BEFORE = "hello\n";
15+
const AFTER = "hello, fidelity\n";
16+
17+
/** A stream-json CLI shaped like Claude Code 2.1.302: a TodoWrite plan, an Edit behind the stdio permission prompt, a check, an answer. */
18+
function fakeClaude(repo: string, counts: string, variant: Variant): string {
19+
const file = path.join(repo, "greeting.txt");
20+
const check = variant === "fail" ? "grep -q missing greeting.txt" : "grep -q fidelity greeting.txt";
21+
return `#!${process.execPath}
22+
const fs = require('node:fs');
23+
const { spawnSync } = require('node:child_process');
24+
if (process.argv.includes('--version')) { console.log('2.1.302 (Claude Code)'); process.exit(0); }
25+
const emit = (x) => console.log(JSON.stringify(x));
26+
const count = (key) => { const c = fs.existsSync(${JSON.stringify(counts)}) ? JSON.parse(fs.readFileSync(${JSON.stringify(counts)}, 'utf8')) : {}; c[key] = (c[key] || 0) + 1; fs.writeFileSync(${JSON.stringify(counts)}, JSON.stringify(c)); };
27+
const todos = (a, b) => emit({ type: 'assistant', message: { id: 'm-todo-' + a + b, content: [{ type: 'tool_use', id: 'todo-' + a + b, name: 'TodoWrite', input: { todos: [{ content: 'Edit greeting.txt', status: a, activeForm: 'Editing' }, { content: 'Check the file', status: b, activeForm: 'Checking' }] } }] } });
28+
let pending = null;
29+
const rl = require('node:readline').createInterface({ input: process.stdin });
30+
rl.on('close', () => process.exit(0));
31+
rl.on('line', (line) => {
32+
const msg = JSON.parse(line);
33+
if (msg.type === 'control_response' && pending) { const go = pending; pending = null; go(msg.response.response.behavior === 'allow'); return; }
34+
if (msg.type !== 'user') return;
35+
emit({ type: 'system', subtype: 'init', session_id: 'fidelity-session', claude_code_version: '2.1.302', model: 'fixture' });
36+
todos('in_progress', 'pending');
37+
emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'todo-in_progresspending', content: 'ok' }] } });
38+
emit({ type: 'assistant', message: { id: 'm-say', content: [{ type: 'text', text: 'Editing the greeting.' }] } });
39+
const edit = { file_path: ${JSON.stringify(file)}, old_string: 'hello', new_string: 'hello, fidelity' };
40+
emit({ type: 'assistant', message: { id: 'm-edit', content: [{ type: 'tool_use', id: 'edit-1', name: 'Edit', input: edit }] } });
41+
pending = (allowed) => {
42+
if (!allowed) {
43+
emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'edit-1', content: 'The user denied this action in Modex.', is_error: true }] } });
44+
emit({ type: 'assistant', message: { id: 'm-final', content: [{ type: 'text', text: 'Left the file as it was.' }] } });
45+
emit({ type: 'result', subtype: 'success', is_error: false, session_id: 'fidelity-session', result: 'Left the file as it was.', num_turns: 3 });
46+
return;
47+
}
48+
count('edits');
49+
fs.writeFileSync(${JSON.stringify(file)}, ${JSON.stringify(AFTER)});
50+
emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'edit-1', content: 'The file was updated.' }] } });
51+
todos('completed', 'in_progress');
52+
emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'todo-completedin_progress', content: 'ok' }] } });
53+
emit({ type: 'assistant', message: { id: 'm-check', content: [{ type: 'tool_use', id: 'check-1', name: 'Bash', input: { command: ${JSON.stringify(check)} } }] } });
54+
count('checksStarted');
55+
const finish = () => {
56+
const run = spawnSync('/bin/sh', ['-c', ${JSON.stringify(check)}], { cwd: ${JSON.stringify(repo)} });
57+
count('checksFinished');
58+
const ok = run.status === 0;
59+
emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'check-1', content: ok ? '' : 'exit ' + run.status, is_error: !ok }] } });
60+
if (ok) { todos('completed', 'completed'); emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'todo-completedcompleted', content: 'ok' }] } }); }
61+
const answer = ok ? 'Changed the greeting; the check passed.' : 'Changed the greeting; the check failed.';
62+
emit({ type: 'assistant', message: { id: 'm-final', content: [{ type: 'text', text: answer }] } });
63+
emit({ type: 'result', subtype: 'success', is_error: false, session_id: 'fidelity-session', result: answer, num_turns: 6 });
64+
};
65+
if (${JSON.stringify(variant)} === 'stop') setTimeout(finish, 60000); else finish();
66+
};
67+
emit({ type: 'control_request', request_id: 'perm-1', request: { subtype: 'can_use_tool', tool_name: 'Edit', tool_use_id: 'edit-1', input: edit } });
68+
});
69+
`;
70+
}
71+
72+
async function open(variant: Variant): Promise<{ app: ElectronApplication; page: Page; home: string; repo: string; counts: string; read: () => Record<string, number> }> {
73+
const { home, repo } = seedHome({ default_backend: "claude" });
74+
fs.writeFileSync(path.join(repo, "greeting.txt"), BEFORE);
75+
const counts = path.join(home, "fixture-counts.json");
76+
const cli = path.join(home, "claude-fidelity.cjs");
77+
fs.writeFileSync(cli, fakeClaude(repo, counts, variant), { mode: 0o755 });
78+
const statePath = path.join(home, "app/state.json");
79+
const state = JSON.parse(fs.readFileSync(statePath, "utf8"));
80+
state.settings.claude_bin = cli;
81+
fs.writeFileSync(statePath, JSON.stringify(state));
82+
const { app, page } = await launch(home);
83+
await page.keyboard.press("Meta+n");
84+
await expect(tid(page, "composer-input")).toBeFocused();
85+
await tid(page, "composer-input").fill("Change the greeting to mention fidelity and check it");
86+
await page.keyboard.press("Meta+Enter");
87+
await expect(tid(page, "thread-view")).toBeVisible({ timeout: 15_000 });
88+
const read = () => (fs.existsSync(counts) ? JSON.parse(fs.readFileSync(counts, "utf8")) as Record<string, number> : {});
89+
return { app, page, home, repo, counts, read };
90+
}
91+
92+
const approval = (page: Page) => items(page, "approval").filter({ hasText: "Edit" });
93+
const close = async (run: { app: ElectronApplication; home: string; repo: string }) => {
94+
await run.app.close().catch(() => {});
95+
fs.rmSync(run.home, { recursive: true, force: true });
96+
fs.rmSync(run.repo, { recursive: true, force: true });
97+
};
98+
99+
test("golden path: plan, approval, a real edit and check, the answer, Save to Notes, Open, and a restart that restores without re-running", async () => {
100+
let run = await open("golden");
101+
try {
102+
await approval(run.page).getByRole("button", { name: "Approve" }).click();
103+
await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 });
104+
expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(AFTER);
105+
expect(run.read()).toEqual({ edits: 1, checksStarted: 1, checksFinished: 1 });
106+
// Typed outcomes, the reported plan and the final answer, as recorded rather than as prose says.
107+
await expect(tid(items(run.page, "plan"), "plan-step")).toHaveText(["Reported done Edit greeting.txt", "Reported done Check the file"]);
108+
await tid(run.page, "turn-label").last().click();
109+
await expect(tid(items(run.page, "tool").filter({ hasText: "edit" }).first(), "tool-meta")).toContainText("done");
110+
const check = items(run.page, "tool").filter({ hasText: "grep -q fidelity" });
111+
await tid(check, "item-toggle").click();
112+
await expect(tid(check, "tool-exit")).toHaveText("Exit code not reported");
113+
await expect(run.page.locator('.turn-content > div[data-final="true"]').last()).toContainText("Changed the greeting; the check passed.");
114+
// Save to Notes, again (same note, next revision), then Open.
115+
await tid(run.page, "save-task-note").last().click();
116+
await expect(tid(run.page, "task-note-saved").last()).toContainText("Saved to Notes · Task:");
117+
const notes = async () => (await run.page.evaluate(() => window.modex!.invoke("space:list", undefined))) as { id: string; title: string; markdown: string; revision: number }[];
118+
const first = (await notes()).find((p) => p.title.startsWith("Task:"))!;
119+
expect(first.markdown).toContain("- [x] Edit greeting.txt (reported done)");
120+
expect(first.markdown).toContain("`greeting.txt`");
121+
expect(first.markdown).toContain("`grep -q fidelity greeting.txt` — succeeded, exit code not reported");
122+
expect(first.markdown).not.toContain(run.repo);
123+
await tid(run.page, "save-task-note").last().click();
124+
await expect(tid(run.page, "task-note-saved").last()).toContainText("Updated in Notes · Task:");
125+
const second = (await notes()).filter((p) => p.title.startsWith("Task:"));
126+
expect(second.map((p) => [p.id, p.revision])).toEqual([[first.id, first.revision + 1]]);
127+
// Open the note: the record reads in Notes as saved.
128+
const threadId = (await currentRow(run.page).getAttribute("data-thread-id"))!;
129+
const before = await run.page.evaluate((id) => window.modex!.invoke("thread:items", { threadId: id }), threadId);
130+
await tid(run.page, "task-note-saved").last().getByRole("button", { name: "Open" }).click();
131+
await expect(run.page.getByText(`From modex thread ${threadId}.`)).toBeVisible();
132+
// Restart: the same items come back, and nothing runs again.
133+
await run.app.close();
134+
const relaunched = await launch(run.home);
135+
run = { ...run, app: relaunched.app, page: relaunched.page };
136+
const after = await run.page.evaluate((id) => window.modex!.invoke("thread:items", { threadId: id }), threadId);
137+
expect(after).toEqual(before);
138+
expect(run.read()).toEqual({ edits: 1, checksStarted: 1, checksFinished: 1 });
139+
expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(AFTER);
140+
} finally { await close(run); }
141+
});
142+
143+
test("a denial leaves the file as it was and runs nothing", async () => {
144+
const run = await open("deny");
145+
try {
146+
await approval(run.page).getByRole("button", { name: "Deny" }).click();
147+
await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 });
148+
expect(fs.readFileSync(path.join(run.repo, "greeting.txt"), "utf8")).toBe(BEFORE);
149+
expect(run.read()).toEqual({});
150+
await tid(run.page, "turn-label").last().click();
151+
await expect(tid(items(run.page, "tool").filter({ hasText: "edit" }).first(), "tool-meta")).toContainText("denied");
152+
} finally { await close(run); }
153+
});
154+
155+
test("a failing check is a failed command, whatever the reply says", async () => {
156+
const run = await open("fail");
157+
try {
158+
await approval(run.page).getByRole("button", { name: "Approve" }).click();
159+
await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 });
160+
const check = items(run.page, "tool").filter({ hasText: "grep -q missing" });
161+
await expect(check).toBeVisible();
162+
await expect(tid(check, "tool-meta")).toContainText("failed");
163+
await expect(tid(items(run.page, "plan"), "plan-step")).toHaveText(["Reported done Edit greeting.txt", "In progress Check the file"]);
164+
} finally { await close(run); }
165+
});
166+
167+
test("Stop while a check runs ends it as stopped; the check never reports, and nothing is retried", async () => {
168+
const run = await open("stop");
169+
try {
170+
await approval(run.page).getByRole("button", { name: "Approve" }).click();
171+
await expect.poll(() => run.read().checksStarted ?? 0).toBe(1);
172+
await tid(run.page, "stop").click();
173+
await expect(currentRow(run.page)).toHaveAttribute("data-status", "idle", { timeout: 20_000 });
174+
const check = items(run.page, "tool").filter({ hasText: "grep -q fidelity" });
175+
await expect(tid(check, "tool-meta")).toContainText("stopped");
176+
expect(run.read()).toEqual({ edits: 1, checksStarted: 1 });
177+
await expect(run.page.getByText("Stopped.")).toBeVisible();
178+
} finally { await close(run); }
179+
});

‎docs/conversation-fidelity.md‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Conversation fidelity
2+
3+
What Modex shows about a coding turn, where each fact comes from, and what it does not claim. It is
4+
the user-facing summary of tracker [#273](https://github.com/TypeSafeAI/modex/issues/273). The
5+
contracts behind it are [conversation events](conversation-events.md),
6+
[evidence](conversation-evidence.md), [task plans and records](task-context.md) and the
7+
[event journal](event-journal.md). The acceptance matrix and its open gates are in
8+
[the acceptance receipt](reviews/2026-10-18-conversation-fidelity-acceptance.md).
9+
10+
Coding still runs only through `codex app-server` and `claude -p`. Nothing described here executes,
11+
replays or re-sends anything by being shown, reloaded or reconnected.
12+
13+
## Supported
14+
15+
| Feature | Codex (app-server, codex-cli 0.163.0) | Claude (`claude -p`, Claude Code 2.1.302) |
16+
| --- | --- | --- |
17+
| Message identity and phase | Native item ids; `commentary` and `final_answer` phases | Message id and block index; no phase (unspecified) |
18+
| Final answer | The latest attempt's `final` message | The latest attempt's last reply |
19+
| Reasoning | Summary parts; reasoning text kept apart; empty reasoning is "not shared" | Thinking text when `thinking_display` is `summarized` (summary) or other (provider text); signature-only or redacted is "not shared" |
20+
| Tool outcome | From native status: succeeded, failed, denied (declined), unknown | Succeeded, failed, or denied when Modex refused it |
21+
| Exit code | Reported | Not reported |
22+
| Citations | Memory citations | None; links in the reply are labelled as the model's |
23+
| Plan | `turn/plan/updated` | `TodoWrite` |
24+
| Save to Notes | Yes | Yes |
25+
| Stop | Native interrupt | Process stop (CF-2, [#275](https://github.com/TypeSafeAI/modex/issues/275), refines this) |
26+
27+
## Fallbacks
28+
29+
- **History from before these changes** reads unchanged. It keeps the old last-reply heuristic and
30+
gains no invented provenance.
31+
- **A value a CLI does not report** stays unspecified, unknown or not shared. Modex never guesses
32+
one.
33+
- **Work a dead process left running** becomes unknown everywhere it is shown, not running and not
34+
failed, and is not run again.
35+
- **A phone that predates these fields** shows a tool's outcome in its title, and a plan as a line of
36+
text.
37+
- **Journal off,** the default, keeps the earlier reload behaviour. Journal on replays the same items
38+
as the transcript.
39+
40+
## Privacy limits
41+
42+
- **Opaque reasoning** (signatures, redacted thinking) is never read, stored or shown.
43+
- **Task records** leave out reasoning, command output and paths outside the project.
44+
- **Streamer Mode** masks what the window shows. It does not sanitize the clipboard, accessibility
45+
text, images or arbitrary secrets in transcripts. Copy and Save to Notes write what the transcript
46+
holds.
47+
48+
## Migration and rollback
49+
50+
Every new item field is optional and additive. An older build reading a newer transcript ignores the
51+
fields it does not know and shows the item as before. A plan item shows as text on older phones; an
52+
older desktop ignores it. Nothing migrates state files in place. The journal stores events
53+
unchanged, so rolling back to an older build leaves its replay working.
54+
55+
## Reference captures
56+
57+
The CF-4 typed-turn spec captures a typed turn at 900×600 in Graphite, Jev and OpenCoven
58+
(`MODEX_SCREENSHOTS=<dir>` keeps them). They are test captures from the mock engine and injected
59+
items, not provider screenshots. No reference images are committed yet; the receipt lists that as
60+
open.

0 commit comments

Comments
 (0)