Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 120 additions & 1 deletion packages/app/e2e/perf/perf-probe.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import fs from "node:fs/promises"
import path from "node:path"
import { raw } from "../../../opencode/test/lib/llm-server"
import { test, expect } from "../fixtures"
import { waitTerminalFocusIdle, withSession } from "../actions"
import { cleanupSession, waitSessionIdle, waitSessionSaved, waitTerminalFocusIdle, withSession } from "../actions"
import {
promptSelector,
sessionMessageItemSelector,
Expand All @@ -11,6 +11,7 @@ import {
terminalSelector,
} from "../selectors"
import { sessionPath, terminalToggleKey } from "../utils"
import type { createSdk } from "../utils"
import { installPerfProbe, resetPerfProbe, snapshotPerfProbe, summarizeScenarioRuns } from "./probe"
import { applyPerfProfile, readPerfProfile, shouldRunScenario, type PerfScenarioName } from "./profiles"
import { seedTimelineRecomputeSession } from "./timeline-fixture"
Expand All @@ -37,8 +38,22 @@ const longMarkdown = [
...Array.from({ length: 80 }, (_, index) => `Paragraph ${index + 1}: ${"streaming markdown content ".repeat(8)}`),
].join("\n")

const heavyBashCommand =
'node -e \'for (let i = 0; i < 900; i++) console.log(String(i).padStart(4, "0") + " " + "heavy bash output ".repeat(8))\''

const scenarioResults: ReturnType<typeof summarizeScenarioRuns>[] = []

type PerfSdk = ReturnType<typeof createSdk>
type PerfProject = {
directory: string
url: string
sdk: PerfSdk
trackSession: (sessionID: string) => void
}
type PerfLlm = {
tool: (name: string, input: unknown) => Promise<void>
}

const chatChunk = (delta: Record<string, unknown>, input?: { finish?: string; usage?: { input: number; output: number } }) => ({
id: "chatcmpl-test",
object: "chat.completion.chunk",
Expand Down Expand Up @@ -149,6 +164,82 @@ async function scrollTimelineTo(page: Parameters<typeof snapshotPerfProbe>[0], t
expect(found).toBe(true)
}

async function enableShellToolPartsExpanded(page: Parameters<typeof snapshotPerfProbe>[0]) {
const apply = () => {
const raw = localStorage.getItem("settings.v3")
const current = (() => {
if (!raw) return {}
try {
return JSON.parse(raw) as Record<string, unknown>
} catch {
return {}
}
})()
const general = current.general && typeof current.general === "object" ? current.general : {}
localStorage.setItem(
"settings.v3",
JSON.stringify({
...current,
general: {
...general,
shellToolPartsExpanded: true,
},
}),
)
}

await page.addInitScript(apply)
await page.evaluate(apply)
}

async function seedHeavyBashSession(input: {
project: PerfProject
llm: PerfLlm
run: number
}) {
const session = await input.project.sdk.session
.create({
title: `perf heavy bash ${Date.now()}-${input.run}`,
permission: [{ permission: "bash", pattern: "*", action: "allow" }],
})
.then((result) => result.data)
if (!session?.id) throw new Error("Session create did not return an id")
input.project.trackSession(session.id)

await input.llm.tool("bash", {
command: heavyBashCommand,
description: "Prints heavy deterministic output",
})
await input.project.sdk.session.promptAsync({
sessionID: session.id,
agent: "build",
parts: [{ type: "text", text: "Run the heavy bash perf fixture." }],
})
await waitSessionIdle(input.project.sdk, session.id, 90_000)
await waitSessionSaved(input.project.directory, session.id, 90_000, input.project.url)

await expect
.poll(
async () => {
const messages = await input.project.sdk.session.messages({ sessionID: session.id, limit: 20 })
return (messages.data ?? []).some((message) =>
message.parts.some(
(part) =>
part.type === "tool" &&
part.tool === "bash" &&
part.state.status === "completed" &&
typeof part.state.output === "string" &&
part.state.output.includes("heavy bash output"),
),
)
},
{ timeout: 30_000 },
)
.toBe(true)

return session
}

function skipUnlessScenario(name: PerfScenarioName) {
test.skip(!shouldRunScenario(PERF_PROFILE, name), `${PERF_PROFILE} profile does not run ${name}`)
}
Expand Down Expand Up @@ -262,6 +353,34 @@ test.describe("PR0.1 perf probe baseline", () => {
scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: PERF_PROFILE, scenario: "tool-call-expand", runs }))
})

test("tool-default-open-heavy-bash emits a 3-run JSON baseline", async ({ page, project, llm }) => {
skipUnlessScenario("tool-default-open-heavy-bash")
await installPerfProbe(page)
await applyPerfProfile(page, PERF_PROFILE)
await project.open()
await enableShellToolPartsExpanded(page)

const runs = []
for (let run = 0; run < 3; run += 1) {
const session = await seedHeavyBashSession({ project, llm, run })
try {
await page.goto(sessionPath(project.directory, session.id))
const trigger = page.locator('[data-slot="collapsible-trigger"]').filter({ has: page.locator('[data-component="tool-trigger"]') }).first()
await expect(trigger).toBeVisible({ timeout: 30_000 })
await expect(trigger).toHaveAttribute("aria-expanded", "true")
await settleFrames(page, 2)
runs.push(await snapshotPerfProbe(page))
} finally {
await cleanupSession({ sdk: project.sdk, sessionID: session.id }).catch(() => undefined)
}
if (run < 2) await cooldownAfterRun(page)
}

scenarioResults.push(
summarizeScenarioRuns({ branch: perfBranch, profile: PERF_PROFILE, scenario: "tool-default-open-heavy-bash", runs }),
)
})

test("terminal-side-panel-open emits a 3-run JSON baseline", async ({ page, project }) => {
skipUnlessScenario("terminal-side-panel-open")
await installPerfProbe(page)
Expand Down
9 changes: 9 additions & 0 deletions packages/app/e2e/perf/profiles.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import { expect, test } from "bun:test"
import { shouldRunScenario, type PerfScenarioName } from "./profiles"

test("default profile runs heavy default-open bash perf coverage", () => {
const scenario = "tool-default-open-heavy-bash" as PerfScenarioName

expect(shouldRunScenario("default", scenario)).toBe(true)
expect(shouldRunScenario("low-end", scenario)).toBe(false)
})
2 changes: 2 additions & 0 deletions packages/app/e2e/perf/profiles.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ export type PerfScenarioName =
| "homepage-cold"
| "session-streaming-long"
| "tool-call-expand"
| "tool-default-open-heavy-bash"
| "terminal-side-panel-open"
| "session-scroll-reading"
| "session-timeline-recompute"
Expand All @@ -13,6 +14,7 @@ const defaultScenarios = new Set<PerfScenarioName>([
"homepage-cold",
"session-streaming-long",
"tool-call-expand",
"tool-default-open-heavy-bash",
"terminal-side-panel-open",
"session-scroll-reading",
])
Expand Down
136 changes: 136 additions & 0 deletions packages/ui/src/components/basic-tool-defer.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
import { afterAll, beforeAll, beforeEach, expect, test } from "bun:test"
import { GlobalRegistrator } from "@happy-dom/global-registrator"
import { createRequire } from "module"
import { createServer, type ViteDevServer } from "vite"
import solidPlugin from "vite-plugin-solid"
import { basicToolInitialReady } from "./basic-tool"

const require = createRequire(import.meta.url)
const solidWeb = require.resolve("solid-js/web/dist/web.js")
const solidCore = require.resolve("solid-js/dist/solid.js")
const solidStore = require.resolve("solid-js/store/dist/store.js")

let server: ViteDevServer | undefined
let rafCallbacks: Array<FrameRequestCallback | undefined> = []
let registeredDom = false

const flushAnimationFrames = () => {
const callbacks = rafCallbacks
rafCallbacks = []
callbacks.forEach((callback) => callback?.(performance.now()))
}

beforeAll(async () => {
if (typeof document === "undefined" || typeof window === "undefined") {
GlobalRegistrator.register()
registeredDom = true
}
window.requestAnimationFrame = globalThis.requestAnimationFrame = ((callback: FrameRequestCallback) => {
rafCallbacks.push(callback)
return rafCallbacks.length
}) as typeof requestAnimationFrame
window.cancelAnimationFrame = globalThis.cancelAnimationFrame = ((id: number) => {
rafCallbacks[id - 1] = undefined
}) as typeof cancelAnimationFrame

server = await createServer({
root: new URL("../..", import.meta.url).pathname,
Comment thread
Astro-Han marked this conversation as resolved.
configFile: false,
plugins: [solidPlugin({ solid: { generate: "dom" } })],
resolve: {
alias: {
"solid-js/web": solidWeb,
"solid-js/store": solidStore,
"solid-js": solidCore,
},
},
server: { middlewareMode: true },
appType: "custom",
logLevel: "silent",
ssr: { noExternal: ["@kobalte/core", "solid-js"] },
})
})

beforeEach(() => {
document.body.textContent = ""
rafCallbacks = []
})

afterAll(async () => {
await server?.close()
if (registeredDom) GlobalRegistrator.unregister()
})

async function loadFixture(): Promise<typeof import("../../test/fixtures/basic-tool-render.fixture")> {
if (!server) throw new Error("Vite server not initialized")
return (await server.ssrLoadModule(
"/test/fixtures/basic-tool-render.fixture.tsx",
)) as typeof import("../../test/fixtures/basic-tool-render.fixture")
}

test("deferred default-open tools do not mount details immediately", () => {
expect(basicToolInitialReady({ defaultOpen: true, defer: true })).toBe(false)
})

test("deferred default-open tools do not resolve details until the next frame", async () => {
const { mountBasicTool } = await loadFixture()
const tool = mountBasicTool({ defaultOpen: true, defer: true })

await Promise.resolve()

expect(tool.detailsRenderCount()).toBe(0)
expect(tool.details()).toBeNull()

flushAnimationFrames()
await Promise.resolve()

expect(tool.detailsRenderCount()).toBeGreaterThan(0)
expect(tool.details()?.textContent).toBe("details")

tool.dispose()
})

test("non-deferred default-open tools keep rendering details synchronously", async () => {
const { mountBasicTool } = await loadFixture()
const tool = mountBasicTool({ defaultOpen: true, defer: false })

expect(tool.detailsRenderCount()).toBeGreaterThan(0)
expect(tool.details()?.textContent).toBe("details")

tool.dispose()
})

test("deferred tools reset details when closed", async () => {
const { mountBasicTool } = await loadFixture()
const tool = mountBasicTool({ defaultOpen: false, defer: true })

expect(tool.detailsRenderCount()).toBe(0)
expect(tool.details()).toBeNull()

tool.trigger()?.click()
await Promise.resolve()
flushAnimationFrames()
await Promise.resolve()

expect(tool.detailsRenderCount()).toBeGreaterThan(0)
expect(tool.details()?.textContent).toBe("details")

tool.trigger()?.click()
await Promise.resolve()

expect(tool.details()).toBeNull()

tool.dispose()
})

test("non-deferred default-open tools keep the previous immediate details behavior", () => {
expect(basicToolInitialReady({ defaultOpen: true })).toBe(true)
expect(basicToolInitialReady({ defaultOpen: true, defer: false })).toBe(true)
})

test("closed tools start without mounted details", () => {
expect(basicToolInitialReady({ defaultOpen: false, defer: true })).toBe(false)
expect(basicToolInitialReady({ defaultOpen: false })).toBe(false)
expect(basicToolInitialReady({ defer: true })).toBe(false)
expect(basicToolInitialReady({})).toBe(false)
})
Loading
Loading