|
| 1 | +import { test, expect, type ElectronApplication, type Page } from "@playwright/test"; |
| 2 | +import fs from "node:fs"; |
| 3 | +import path from "node:path"; |
| 4 | +import { currentRow, items, launch, resizeWindow, seedHome, tid } from "./support"; |
| 5 | + |
| 6 | +/** |
| 7 | + * Turns from the conversation-event contract (docs/conversation-events.md) render from their own |
| 8 | + * fields: the final answer is the `final` message, every running item is live, only work that |
| 9 | + * succeeded folds, reasoning says what kind it is, and a retry shows as a new attempt. The items are |
| 10 | + * sent to the window as main would send them; no CLI runs. |
| 11 | + */ |
| 12 | +let app: ElectronApplication; |
| 13 | +let page: Page; |
| 14 | +let home: string; |
| 15 | +let repo: string; |
| 16 | +let threadId: string; |
| 17 | + |
| 18 | +const at = () => new Date().toISOString(); |
| 19 | +const origin = (attempt: string, item: string) => ({ thread: threadId, turn: "u-typed", attempt, backend: "codex", native: { runtime: "R", thread: "nt", turn: "nu", item } }); |
| 20 | +const say = (id: string, text: string, phase: string, outcome = "succeeded", attempt = "A1") => ({ id, kind: "assistant", text, phase, outcome, origin: origin(attempt, id), at: at() }); |
| 21 | +const tool = (id: string, title: string, outcome: string, attempt = "A1") => ({ id, kind: "tool", name: "shell", title, args: { command: title.slice(2) }, status: outcome === "running" ? "running" : "done", outcome, |
| 22 | + ...(outcome === "succeeded" ? { ok: true } : outcome === "failed" || outcome === "denied" ? { ok: false } : {}), output: "", durationMs: 1200, origin: origin(attempt, id), at: at() }); |
| 23 | +const think = (id: string, reasoning: string, text: string, status = "done") => ({ id, kind: "thinking", text, reasoning, status, outcome: status === "running" ? "running" : "succeeded", durationMs: 3000, origin: origin("A1", id), at: at() }); |
| 24 | + |
| 25 | +async function send(events: Record<string, unknown>[]): Promise<void> { |
| 26 | + await app.evaluate(({ BrowserWindow }, list) => { |
| 27 | + for (const event of list) BrowserWindow.getAllWindows()[0]!.webContents.send("thread:event", event); |
| 28 | + }, events); |
| 29 | +} |
| 30 | +const item = (event: Record<string, unknown>) => ({ type: "item", threadId, item: event }); |
| 31 | +const status = (value: string) => ({ type: "status", threadId, status: value }); |
| 32 | +/** |
| 33 | + * Marks the thread running the way main does before a turn streams. The window reloads a thread's |
| 34 | + * items from main when it starts running, so items sent before that reload finishes would be |
| 35 | + * replaced by main's (which never had them): wait for the reload first. |
| 36 | + */ |
| 37 | +async function running(): Promise<void> { |
| 38 | + await send([status("running")]); |
| 39 | + await expect(currentRow(page)).toHaveAttribute("data-status", "running"); |
| 40 | + // A probe that is still shown after a moment was sent after the reload landed. |
| 41 | + for (let attempt = 0; ; attempt++) { |
| 42 | + const id = `probe-${Date.now()}-${attempt}`; |
| 43 | + await send([item({ id, kind: "notice", level: "info", text: id, at: at() })]); |
| 44 | + await page.waitForTimeout(150); |
| 45 | + if (await page.getByText(id).isVisible()) return; |
| 46 | + if (attempt === 20) throw new Error("the window kept reloading the running thread's items"); |
| 47 | + } |
| 48 | +} |
| 49 | + |
| 50 | +test.beforeAll(async () => { |
| 51 | + ({ home, repo } = seedHome()); |
| 52 | + ({ app, page } = await launch(home)); |
| 53 | + const script = path.join(home, "typed-turns-script.json"); |
| 54 | + fs.writeFileSync(script, JSON.stringify([{ content: "Ready." }])); |
| 55 | + await page.evaluate((mock_script) => window.modex!.invoke("settings:update", { mock_script }), script); |
| 56 | + await page.keyboard.press("Meta+n"); |
| 57 | + await expect(tid(page, "composer-input")).toBeFocused(); |
| 58 | + await tid(page, "composer-input").fill("start"); |
| 59 | + await page.keyboard.press("Meta+Enter"); |
| 60 | + await expect(tid(page, "thread-view")).toBeVisible({ timeout: 15_000 }); |
| 61 | + await expect(currentRow(page)).toHaveAttribute("data-status", "idle"); |
| 62 | + threadId = (await currentRow(page).getAttribute("data-thread-id"))!; |
| 63 | +}); |
| 64 | + |
| 65 | +test.afterAll(async () => { |
| 66 | + await app?.close(); |
| 67 | + fs.rmSync(home, { recursive: true, force: true }); |
| 68 | + fs.rmSync(repo, { recursive: true, force: true }); |
| 69 | +}); |
| 70 | + |
| 71 | +test("a typed turn answers with its final message, folds only work that succeeded, and labels reasoning honestly", async () => { |
| 72 | + await send([ |
| 73 | + item({ id: "u-typed", kind: "user", text: "typed turn", at: at() }), |
| 74 | + item(think("r-summary", "summary", "Plan: check twice.")), |
| 75 | + item(say("c1", "Checking one.", "commentary")), |
| 76 | + item(tool("t1", "$ echo one", "succeeded")), |
| 77 | + item(tool("t2", "$ touch denied.txt", "denied")), |
| 78 | + item(tool("t3", "$ curl example.com", "unknown")), |
| 79 | + item(think("r-hidden", "unavailable", "")), |
| 80 | + item(say("f", "Done ha ha ha.", "final")), |
| 81 | + item(say("after", "One more note.", "commentary")), |
| 82 | + status("idle"), |
| 83 | + ]); |
| 84 | + const answer = page.locator('.turn-content > div[data-final="true"]').last(); |
| 85 | + await expect(answer).toContainText("Done ha ha ha."); |
| 86 | + // Succeeded work and commentary before the answer fold; the denial and the unknown stay in view. |
| 87 | + await expect(items(page, "assistant").filter({ hasText: "Checking one." })).toBeHidden(); |
| 88 | + await expect(items(page, "tool").filter({ hasText: "$ echo one" })).toBeHidden(); |
| 89 | + await expect(items(page, "tool").filter({ hasText: "$ touch denied.txt" })).toBeVisible(); |
| 90 | + await expect(tid(items(page, "tool").filter({ hasText: "$ touch denied.txt" }), "tool-meta")).toHaveText("denied · 1.2s"); |
| 91 | + await expect(tid(items(page, "tool").filter({ hasText: "$ curl example.com" }), "tool-meta")).toHaveText("status unavailable · 1.2s"); |
| 92 | + // A commentary message after the answer is not the answer. |
| 93 | + await expect(items(page, "assistant").filter({ hasText: "One more note." })).toHaveAttribute("data-phase", "commentary"); |
| 94 | + // Expanding the header shows the folded work, reasoning labelled by what it is. |
| 95 | + await tid(page, "turn-label").last().click(); |
| 96 | + await expect(items(page, "tool").filter({ hasText: "$ echo one" })).toBeVisible(); |
| 97 | + await expect(tid(page.locator('[data-reasoning="summary"]'), "reasoning-tag")).toHaveText("Summary"); |
| 98 | + const hidden = page.locator('[data-reasoning="unavailable"]'); |
| 99 | + await expect(tid(hidden, "reasoning-tag")).toHaveText("Not shared"); |
| 100 | + await tid(hidden, "item-toggle").click(); |
| 101 | + await expect(tid(hidden, "item-body")).toHaveText("The model reasoned here, but its reasoning was not shared."); |
| 102 | +}); |
| 103 | + |
| 104 | +test("every running item is live at once, and a stopped message says it stopped", async () => { |
| 105 | + await running(); |
| 106 | + await send([ |
| 107 | + item({ id: "u-live", kind: "user", text: "live turn", at: at() }), |
| 108 | + item({ ...tool("l-tool", "$ npm test", "running"), origin: { ...origin("L1", "l-tool"), turn: "u-live" } }), |
| 109 | + item({ ...say("l-say", "Running the tests", "commentary", "running"), origin: { ...origin("L1", "l-say"), turn: "u-live" } }), |
| 110 | + ]); |
| 111 | + await expect(page.locator('.turn-content > div[data-live="true"]')).toHaveCount(2); |
| 112 | + await send([ |
| 113 | + item({ ...say("l-say", "Running the tests", "commentary", "interrupted"), origin: { ...origin("L1", "l-say"), turn: "u-live" } }), |
| 114 | + item({ ...tool("l-tool", "$ npm test", "interrupted"), origin: { ...origin("L1", "l-tool"), turn: "u-live" } }), |
| 115 | + status("idle"), |
| 116 | + ]); |
| 117 | + await expect(page.locator('.turn-content > div[data-live="true"]')).toHaveCount(0); |
| 118 | + const stopped = items(page, "assistant").filter({ hasText: "Running the tests" }); |
| 119 | + await expect(stopped).toHaveAttribute("data-outcome", "interrupted"); |
| 120 | + await expect(tid(stopped, "message-status")).toHaveText("Stopped before it finished"); |
| 121 | + await expect(tid(items(page, "tool").filter({ hasText: "$ npm test" }), "tool-meta")).toHaveText("stopped · 1.2s"); |
| 122 | +}); |
| 123 | + |
| 124 | +test("a retried turn shows its new attempt, and only that attempt answers", async () => { |
| 125 | + await send([ |
| 126 | + item({ id: "u-retry", kind: "user", text: "retry turn", at: at() }), |
| 127 | + item({ ...say("old", "Stale answer.", "final", "succeeded", "R1"), origin: { ...origin("R1", "old"), turn: "u-retry" } }), |
| 128 | + item({ id: "fail", kind: "notice", level: "error", text: "The turn failed.", at: at() }), |
| 129 | + item({ ...say("new", "Fresh answer.", "final", "succeeded", "R2"), origin: { ...origin("R2", "new"), turn: "u-retry" } }), |
| 130 | + status("idle"), |
| 131 | + ]); |
| 132 | + await expect(tid(page, "attempt-divider").last()).toHaveText("Attempt 2"); |
| 133 | + await expect(page.locator('.turn-content > div[data-final="true"]').last()).toContainText("Fresh answer."); |
| 134 | + await expect(items(page, "assistant").filter({ hasText: "Stale answer." })).toBeHidden(); |
| 135 | + await expect(page.getByText("The turn failed.")).toBeVisible(); |
| 136 | +}); |
| 137 | + |
| 138 | +test("scrolling up while output streams shows Jump to latest; it returns to the bottom and resumes following", async () => { |
| 139 | + const transcript = tid(page, "transcript"); |
| 140 | + await send([item({ id: "u-long", kind: "user", text: "long turn", at: at() }), |
| 141 | + ...Array.from({ length: 40 }, (_, i) => item({ ...say(`long-${i}`, `Line ${i}\n\n${"text ".repeat(40)}`, "commentary"), origin: { ...origin("J1", `long-${i}`), turn: "u-long" } }))]); |
| 142 | + await expect(tid(page, "jump-latest")).toBeHidden(); |
| 143 | + await transcript.evaluate((el) => { el.scrollTop = 0; el.dispatchEvent(new Event("scroll")); }); |
| 144 | + await expect(tid(page, "jump-latest")).toBeVisible(); |
| 145 | + // New output does not pull the reader back down. |
| 146 | + await send([item({ ...say("long-new", "Newest line", "commentary"), origin: { ...origin("J1", "long-new"), turn: "u-long" } })]); |
| 147 | + await expect.poll(() => transcript.evaluate((el) => el.scrollTop)).toBe(0); |
| 148 | + await tid(page, "jump-latest").click(); |
| 149 | + await expect.poll(() => transcript.evaluate((el) => el.scrollHeight - el.scrollTop - el.clientHeight)).toBeLessThanOrEqual(80); |
| 150 | + await expect(tid(page, "jump-latest")).toBeHidden(); |
| 151 | + await send([item({ ...say("final-long", "All lines done.", "final"), origin: { ...origin("J1", "final-long"), turn: "u-long" } })]); |
| 152 | + await expect(page.getByText("All lines done.")).toBeInViewport(); |
| 153 | +}); |
| 154 | + |
| 155 | +test("typed turns stay readable in every theme at the 900×600 minimum", async ({}, testInfo) => { |
| 156 | + // Its own turn: an earlier test's running() reload may have replaced what was sent before it. |
| 157 | + const o = (attempt: string, id: string) => ({ ...origin(attempt, id), turn: "u-themes" }); |
| 158 | + await send([ |
| 159 | + item({ id: "u-themes", kind: "user", text: "theme turn", at: at() }), |
| 160 | + item({ ...say("th-old", "Superseded answer.", "final", "succeeded", "T1"), origin: o("T1", "th-old") }), |
| 161 | + item({ ...think("th-summary", "summary", "Plan: check the themes."), origin: o("T2", "th-summary") }), |
| 162 | + item({ ...tool("th-denied", "$ rm -rf build", "denied", "T2"), origin: o("T2", "th-denied") }), |
| 163 | + item({ ...say("th-final", "Themes checked.", "final", "succeeded", "T2"), origin: o("T2", "th-final") }), |
| 164 | + ]); |
| 165 | + await resizeWindow(app, page, 900, 600); |
| 166 | + const turn = page.locator(".turn-content").filter({ has: page.getByText("Themes checked.") }); |
| 167 | + await tid(page, "turn-label").last().click(); |
| 168 | + for (const theme of ["graphite", "jev", "coven"]) { |
| 169 | + await page.evaluate((value) => window.modex!.invoke("settings:update", { theme: value }), theme); |
| 170 | + await turn.scrollIntoViewIfNeeded(); |
| 171 | + const transcript = tid(page, "transcript"); |
| 172 | + expect(await transcript.evaluate((el) => el.scrollWidth - el.clientWidth), `${theme}: no sideways scroll`).toBeLessThanOrEqual(0); |
| 173 | + await expect(turn.getByText("Themes checked.")).toBeVisible(); |
| 174 | + await expect(turn.getByText("$ rm -rf build")).toBeVisible(); |
| 175 | + await expect(tid(turn.locator('[data-reasoning="summary"]'), "reasoning-tag")).toBeVisible(); |
| 176 | + await expect(tid(turn, "attempt-divider")).toHaveText("Attempt 2"); |
| 177 | + const shot = await page.screenshot(); |
| 178 | + await testInfo.attach(`typed-turn-${theme}-900x600`, { body: shot, contentType: "image/png" }); |
| 179 | + // MODEX_SCREENSHOTS=<dir> keeps them for review evidence. |
| 180 | + if (process.env.MODEX_SCREENSHOTS) fs.writeFileSync(path.join(process.env.MODEX_SCREENSHOTS, `typed-turn-${theme}-900x600.png`), shot); |
| 181 | + } |
| 182 | +}); |
0 commit comments