diff --git a/packages/app/src/agent-stream/render-strategy.test.ts b/packages/app/src/agent-stream/render-strategy.test.ts index ef6478455c..128829fb75 100644 --- a/packages/app/src/agent-stream/render-strategy.test.ts +++ b/packages/app/src/agent-stream/render-strategy.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import type { StreamItem } from "@/types/stream"; import { collectAssistantTurnContentForStreamRenderStrategy, + collectAssistantTurnSpeechForStreamRenderStrategy, getBottomOffsetForStreamRenderStrategy, getFrameChildOrderForStreamRenderStrategy, getHistoryLiveBoundaryIndexForStreamRenderStrategy, @@ -38,6 +39,24 @@ function assistantMessage(id: string, text: string, seed: number): StreamItem { }; } +function toolCall(id: string, seed: number): StreamItem { + return { + kind: "tool_call", + id, + timestamp: createTimestamp(seed), + payload: { + source: "orchestrator", + data: { + toolCallId: id, + toolName: "bash", + arguments: "cmd", + result: null, + status: "completed", + }, + }, + }; +} + describe("resolveStreamRenderStrategy", () => { it("uses forward_stream on web", () => { const strategy = resolveStreamRenderStrategy({ @@ -196,6 +215,99 @@ describe("neighbor and traversal semantics", () => { ).toBe("assistant-1\n\nassistant-2"); }); + it("speaks only the prose after the last tool call", () => { + const chronological: StreamItem[] = [ + userMessage("u1", "user-1", 1), + assistantMessage("a1", "before the tool", 2), + toolCall("t1", 3), + assistantMessage("a2", "after the tool", 4), + ]; + + const forward = resolveStreamRenderStrategy({ platform: "web", isMobileBreakpoint: false }); + const startIndex = chronological.findIndex((item) => item.id === "a2"); + + // Copy takes the whole turn; speech stops at the tool call. Reading a long + // turn from the top would replay narration the user already watched. + expect( + collectAssistantTurnContentForStreamRenderStrategy({ + strategy: forward, + items: chronological, + startIndex, + }), + ).toBe("before the tool\n\nafter the tool"); + expect( + collectAssistantTurnSpeechForStreamRenderStrategy({ + strategy: forward, + items: chronological, + startIndex, + }), + ).toBe("after the tool"); + }); + + it("speaks the whole turn when it has no tool calls", () => { + const chronological: StreamItem[] = [ + userMessage("u1", "user-1", 1), + assistantMessage("a1", "assistant-1", 2), + assistantMessage("a2", "assistant-2", 3), + ]; + + const forward = resolveStreamRenderStrategy({ platform: "web", isMobileBreakpoint: false }); + + expect( + collectAssistantTurnSpeechForStreamRenderStrategy({ + strategy: forward, + items: chronological, + startIndex: chronological.findIndex((item) => item.id === "a2"), + }), + ).toBe("assistant-1\n\nassistant-2"); + }); + + it("speaks nothing when the turn ends on a tool call", () => { + const chronological: StreamItem[] = [ + userMessage("u1", "user-1", 1), + assistantMessage("a1", "before the tool", 2), + toolCall("t1", 3), + ]; + + const forward = resolveStreamRenderStrategy({ platform: "web", isMobileBreakpoint: false }); + + // The button hides on an empty result rather than synthesizing silence. + expect( + collectAssistantTurnSpeechForStreamRenderStrategy({ + strategy: forward, + items: chronological, + startIndex: chronological.findIndex((item) => item.id === "t1"), + }), + ).toBe(""); + }); + + it("stops at the last tool call in both traversal directions", () => { + const chronological: StreamItem[] = [ + userMessage("u1", "user-1", 1), + assistantMessage("a1", "before the tool", 2), + toolCall("t1", 3), + assistantMessage("a2", "after the tool", 4), + ]; + + const inverted = resolveStreamRenderStrategy({ + platform: "android", + isMobileBreakpoint: false, + }); + const invertedItems = orderTailForStreamRenderStrategy({ + strategy: inverted, + streamItems: chronological, + }); + + // Native renders an inverted list, so the walk runs the other way. Same text. + expect( + collectAssistantTurnSpeechForStreamRenderStrategy({ + strategy: inverted, + items: invertedItems, + startIndex: invertedItems.findIndex((item) => item.id === "a2"), + }), + ).toBe("after the tool"); + }); + it("returns undefined neighbor when index would be out of bounds", () => { const forward = resolveStreamRenderStrategy({ platform: "web", diff --git a/packages/app/src/agent-stream/strategy.ts b/packages/app/src/agent-stream/strategy.ts index 4077bb025e..a5f90e54f2 100644 --- a/packages/app/src/agent-stream/strategy.ts +++ b/packages/app/src/agent-stream/strategy.ts @@ -99,6 +99,7 @@ export interface StreamStrategy { relation: NeighborRelation, ) => StreamItem | undefined; collectAssistantTurnContent: (items: StreamItem[], startIndex: number) => string; + collectAssistantTurnSpeech: (items: StreamItem[], startIndex: number) => string; isNearBottom: (input: StreamNearBottomInput) => boolean; getBottomOffset: (metrics: StreamViewportMetrics) => number; getEdgeSlotProps: ( @@ -180,6 +181,34 @@ export function createStreamStrategy(config: StreamStrategyConfig): StreamStrate } return messages.toReversed().join("\n\n"); }, + /** + * The turn's closing prose — what the agent said after its last tool call. + * + * Unlike `collectAssistantTurnContent`, which copies the whole turn, this + * stops at the first `tool_call` walking backward. Reading a long turn aloud + * from the top would replay narration the user watched scroll by; the part + * worth hearing is the summary at the end. + * + * Empty when the turn ends on a tool call with nothing after it — the caller + * hides the button rather than synthesizing silence. + */ + collectAssistantTurnSpeech: (items, startIndex) => { + const messages: string[] = []; + for ( + let index = startIndex; + index >= 0 && index < items.length; + index += config.assistantTurnTraversalStep + ) { + const currentItem = items[index]; + if (currentItem.kind === "user_message" || currentItem.kind === "tool_call") { + break; + } + if (currentItem.kind === "assistant_message") { + messages.push(currentItem.text); + } + } + return messages.toReversed().join("\n\n"); + }, isNearBottom: (input) => config.isNearBottom(input), getBottomOffset: (metrics) => config.getBottomOffset(metrics), getEdgeSlotProps: (component, gapSize) => { @@ -281,6 +310,14 @@ export function collectAssistantTurnContentForStreamRenderStrategy(params: { return params.strategy.collectAssistantTurnContent(params.items, params.startIndex); } +export function collectAssistantTurnSpeechForStreamRenderStrategy(params: { + strategy: StreamStrategy; + items: StreamItem[]; + startIndex: number; +}): string { + return params.strategy.collectAssistantTurnSpeech(params.items, params.startIndex); +} + export function isNearBottomForStreamRenderStrategy( params: StreamNearBottomInput & { strategy: StreamStrategy }, ): boolean { diff --git a/packages/app/src/agent-stream/turn-footer.tsx b/packages/app/src/agent-stream/turn-footer.tsx index fae2eedd87..c745684996 100644 --- a/packages/app/src/agent-stream/turn-footer.tsx +++ b/packages/app/src/agent-stream/turn-footer.tsx @@ -7,6 +7,7 @@ import type { TurnTiming } from "@/timeline/turn-time"; import type { StreamItem } from "@/types/stream"; import { collectAssistantTurnContentForStreamRenderStrategy, + collectAssistantTurnSpeechForStreamRenderStrategy, type StreamStrategy, } from "./strategy"; import { resolveAssistantTurnForkBoundary, type AssistantTurnForkBoundary } from "./turn-boundary"; @@ -185,6 +186,16 @@ function CompletedTurnFooter({ }), [strategy, items, startIndex], ); + const getSpeech = useCallback( + () => + collectAssistantTurnSpeechForStreamRenderStrategy({ + strategy, + items, + startIndex, + }), + [strategy, items, startIndex], + ); + const turnId = items[startIndex]?.id ?? null; const boundary = resolveAssistantTurnForkBoundary({ items, startIndex, @@ -203,6 +214,8 @@ function CompletedTurnFooter({ + {children} ); @@ -881,6 +883,11 @@ function FaviconStatusSync() { return null; } +function ReadAloudRouteGuard() { + useReadAloudRouteGuard(); + return null; +} + const ROOT_STACK_SCREEN_OPTIONS = { headerShown: false, animation: "none" as const, diff --git a/packages/app/src/components/message.tsx b/packages/app/src/components/message.tsx index 2cc7cb4029..49dffda64a 100644 --- a/packages/app/src/components/message.tsx +++ b/packages/app/src/components/message.tsx @@ -105,6 +105,7 @@ import { RewindMenu, type RewindMode } from "@/components/rewind/rewind-menu"; import { useRewindAgentMutation } from "@/components/rewind/use-rewind-agent-mutation"; import { AssistantForkMenu, type AssistantForkTarget } from "@/components/assistant-fork-menu"; import { useRetainedPanelActive } from "@/components/retained-panel"; +import { TurnReadAloudButton } from "@/read-aloud/turn-read-aloud-button"; import { markdownCopyDataSet, markdownCopyOrderedListDataSet, @@ -566,6 +567,10 @@ export const UserMessage = memo(function UserMessage({ interface AssistantTurnFooterProps { getContent: () => string; + /** The turn's closing prose, for read aloud. Omitted where speech isn't offered. */ + getSpeech?: () => string; + /** Assistant message id, identifying this turn as the read-aloud owner. */ + turnId?: string | null; completedAt?: Date; durationMs?: number; onFork?: (target: AssistantForkTarget) => Promise | void; @@ -611,6 +616,8 @@ const TIMESTAMP_REVEAL_MS = 3000; */ export const AssistantTurnFooter = memo(function AssistantTurnFooter({ getContent, + getSpeech, + turnId, completedAt, durationMs, onFork, @@ -667,6 +674,7 @@ export const AssistantTurnFooter = memo(function AssistantTurnFooter({ getContent={getContent} containerStyle={assistantTurnFooterStylesheet.copyButton} /> + {getSpeech && turnId ? : null} {canFork ? : null} {durationLabel ? ( boolean; +}): Promise { + throw new Error("Read aloud is not supported on this platform"); +} + +export function stopReadAloudAudio(): void { + // No playback to stop. +} diff --git a/packages/app/src/read-aloud/read-aloud-audio.web.ts b/packages/app/src/read-aloud/read-aloud-audio.web.ts new file mode 100644 index 0000000000..7e35ac6e56 --- /dev/null +++ b/packages/app/src/read-aloud/read-aloud-audio.web.ts @@ -0,0 +1,75 @@ +// Not a browser global — the bundle has no `Buffer` unless it is imported, and +// `@types/node` makes the bare reference typecheck while failing at runtime. +import { Buffer } from "buffer"; + +import { createAudioEngine } from "@/voice/audio-engine"; +import type { AudioEngine } from "@/voice/audio-engine-types"; + +/** + * Read-aloud playback on web. + * + * Reuses the voice-mode audio engine because on web its `initialize()` only + * opens a playback `AudioContext` — capture (and therefore the microphone + * permission prompt) is behind the separate `startCapture()` call, which read + * aloud never makes. `play()` already queues sequentially, which is exactly the + * ordering read-aloud segments need. + */ +export const isReadAloudAudioSupported = true; + +let engine: AudioEngine | null = null; + +function getEngine(): AudioEngine { + if (!engine) { + engine = createAudioEngine( + { + onCaptureData: () => undefined, + onVolumeLevel: () => undefined, + }, + { traceLabel: "read-aloud" }, + ); + } + return engine; +} + +function toMimeType(format: string): string { + if (format === "mp3") { + return "audio/mpeg"; + } + return format.startsWith("audio/") ? format : `audio/${format}`; +} + +export async function playReadAloudSegment(params: { + audioBase64: string; + format: string; + /** + * Whether this segment has been superseded. Checked after `initialize()`: + * a stop landing during that await is forgotten by the time `play()` runs, + * because `play()` then captures the engine's post-stop generation and + * proceeds as if it were a fresh request. + */ + isCancelled?: () => boolean; +}): Promise { + const bytes = Buffer.from(params.audioBase64, "base64"); + const active = getEngine(); + await active.initialize(); + if (params.isCancelled?.()) { + return; + } + await active.play({ + size: bytes.byteLength, + type: toMimeType(params.format), + async arrayBuffer() { + return Uint8Array.from(bytes).buffer; + }, + }); +} + +export function stopReadAloudAudio(): void { + if (!engine) { + return; + } + // Order matters: drain the queue first so `stop()` does not let a queued + // segment start playing after the user asked for silence. + engine.clearQueue(); + engine.stop(); +} diff --git a/packages/app/src/read-aloud/read-aloud-store.test.ts b/packages/app/src/read-aloud/read-aloud-store.test.ts new file mode 100644 index 0000000000..de79e45a10 --- /dev/null +++ b/packages/app/src/read-aloud/read-aloud-store.test.ts @@ -0,0 +1,127 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { DaemonClient } from "@getpaseo/client"; + +const playReadAloudSegment = vi.fn().mockResolvedValue(undefined); +const stopReadAloudAudio = vi.fn(); + +vi.mock("@/read-aloud/read-aloud-audio", () => ({ + isReadAloudAudioSupported: true, + playReadAloudSegment: (params: { audioBase64: string; format: string }) => + playReadAloudSegment(params), + stopReadAloudAudio: () => stopReadAloudAudio(), +})); + +const { getReadAloudSnapshot, startReadAloud, stopReadAloud } = + await import("@/read-aloud/read-aloud-store"); + +/** Minimal stand-in: the store only ever calls `startReadAloud` on the client. */ +function createClient(): DaemonClient { + return { + startReadAloud: vi.fn().mockReturnValue({ requestId: "req-1", cancel: vi.fn() }), + } as unknown as DaemonClient; +} + +describe("read aloud playback", () => { + beforeEach(() => { + stopReadAloud(); + vi.clearAllMocks(); + }); + + it("names the turn that owns playback, so other footers stay idle", () => { + startReadAloud({ client: createClient(), text: "hello", ownerId: "turn-1", serverId: "srv-1" }); + + expect(getReadAloudSnapshot().ownerId).toBe("turn-1"); + }); + + it("hands the slot to the turn that started last", () => { + startReadAloud({ client: createClient(), text: "first", ownerId: "turn-1", serverId: "srv-1" }); + startReadAloud({ + client: createClient(), + text: "second", + ownerId: "turn-2", + serverId: "srv-1", + }); + + // Only one voice at a time: the second press supersedes the first rather + // than leaving two footers both rendering themselves as speaking. + expect(getReadAloudSnapshot().ownerId).toBe("turn-2"); + }); + + it("releases the slot on stop", () => { + startReadAloud({ client: createClient(), text: "hello", ownerId: "turn-1", serverId: "srv-1" }); + stopReadAloud(); + + expect(getReadAloudSnapshot().ownerId).toBeNull(); + }); + + it("releases the slot when the stream ends and every segment has played", async () => { + const client = createClient(); + vi.mocked(client.startReadAloud).mockImplementation((params) => { + params.onSegment({ + audioBase64: "AAA", + format: "pcm_s16le_24000", + segmentIndex: 0, + segmentCount: 1, + }); + params.onEnd(); + return { requestId: "req-1", cancel: vi.fn() }; + }); + + startReadAloud({ client, text: "hello", ownerId: "turn-1", serverId: "srv-1" }); + await vi.waitFor(() => expect(getReadAloudSnapshot().status).toBe("idle")); + + // A finished read must free the slot too, not just an explicit stop — + // otherwise the button stays stuck showing a stop square forever. + expect(getReadAloudSnapshot().ownerId).toBeNull(); + }); + + it("does not take the slot when the platform has no audio engine", async () => { + vi.resetModules(); + vi.doMock("@/read-aloud/read-aloud-audio", () => ({ + isReadAloudAudioSupported: false, + playReadAloudSegment: vi.fn(), + stopReadAloudAudio: vi.fn(), + })); + const unsupported = await import("@/read-aloud/read-aloud-store"); + + unsupported.startReadAloud({ + client: createClient(), + text: "hello", + ownerId: "turn-1", + serverId: "srv-1", + }); + + const snapshot = unsupported.getReadAloudSnapshot(); + expect(snapshot.failure?.code).toBe("unsupported_platform"); + expect(snapshot.ownerId).toBeNull(); + vi.doUnmock("@/read-aloud/read-aloud-audio"); + }); + + it("records the host that started playback, so a route change can stop it", () => { + startReadAloud({ client: createClient(), text: "hello", ownerId: "turn-1", serverId: "srv-1" }); + + // The Stop control lives in a turn footer. Navigate away and it unmounts, + // so playback has to be attributable to a host or it becomes unstoppable. + expect(getReadAloudSnapshot().ownerServerId).toBe("srv-1"); + }); + + it("clears the owning host on stop", () => { + startReadAloud({ client: createClient(), text: "hello", ownerId: "turn-1", serverId: "srv-1" }); + stopReadAloud(); + + expect(getReadAloudSnapshot().ownerServerId).toBeNull(); + }); + + it("surfaces a daemon failure on the snapshot", () => { + const client = createClient(); + vi.mocked(client.startReadAloud).mockImplementation((params) => { + params.onError({ code: "tts_unavailable", message: "no tts" }); + return { requestId: "req-1", cancel: vi.fn() }; + }); + + startReadAloud({ client, text: "hello", ownerId: "turn-1", serverId: "srv-1" }); + + const snapshot = getReadAloudSnapshot(); + expect(snapshot.failure?.code).toBe("tts_unavailable"); + }); +}); diff --git a/packages/app/src/read-aloud/read-aloud-store.ts b/packages/app/src/read-aloud/read-aloud-store.ts new file mode 100644 index 0000000000..9236bb9f63 --- /dev/null +++ b/packages/app/src/read-aloud/read-aloud-store.ts @@ -0,0 +1,187 @@ +import { useSyncExternalStore } from "react"; +import type { DaemonClient, ReadAloudHandle } from "@getpaseo/client"; + +import { + isReadAloudAudioSupported, + playReadAloudSegment, + stopReadAloudAudio, +} from "@/read-aloud/read-aloud-audio"; + +export interface ReadAloudFailure { + /** Daemon-side code (`tts_unavailable`, `text_too_long`, …) or a client code. */ + code: string; + /** Raw daemon message, used only when the code has no translated copy. */ + message: string; +} + +export interface ReadAloudSnapshot { + /** `loading` = synthesizing, no audio yet. `speaking` = audio is playing. */ + status: "idle" | "loading" | "speaking"; + failure: ReadAloudFailure | null; + /** + * Who asked for this read — the assistant message id of the turn whose button + * was pressed. Playback is a single module-level slot, but the button lives in + * every turn footer, so a footer subscribing to this snapshot has to know + * whether it is the one speaking or a bystander. `null` when idle. + */ + ownerId: string | null; + /** + * The host whose route started this read. The footer button unmounts when you + * navigate away, taking the Stop control with it, so playback has to be tied + * to the route that owns it rather than left running unreachable. + */ + ownerServerId: string | null; +} + +let snapshot: ReadAloudSnapshot = { + status: "idle", + failure: null, + ownerId: null, + ownerServerId: null, +}; +const listeners = new Set<() => void>(); + +/** + * Bumped by every start/stop so callbacks from a superseded request are ignored + * instead of resurrecting state for audio the user already dismissed. + */ +let generation = 0; +let handle: ReadAloudHandle | null = null; +let pendingSegmentPlaybacks = 0; +let streamEnded = false; + +/** The turn currently holding the playback slot. Cleared when playback ends. */ +let ownerId: string | null = null; +/** The host that turn belongs to. Cleared with `ownerId`. */ +let ownerServerId: string | null = null; + +// Takes everything but the module-owned fields, so no call site has to remember +// to carry `ownerId` or `ownerServerId` forward. +function setSnapshot(next: Omit): void { + snapshot = { ...next, ownerId, ownerServerId }; + for (const listener of listeners) { + listener(); + } +} + +function finishIfDone(token: number): void { + if (token !== generation) { + return; + } + if (!streamEnded || pendingSegmentPlaybacks > 0) { + return; + } + handle = null; + ownerId = null; + ownerServerId = null; + setSnapshot({ status: "idle", failure: snapshot.failure }); +} + +export function getReadAloudSnapshot(): ReadAloudSnapshot { + return snapshot; +} + +export function subscribeReadAloud(listener: () => void): () => void { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +} + +export function useReadAloudSnapshot(): ReadAloudSnapshot { + return useSyncExternalStore(subscribeReadAloud, getReadAloudSnapshot, getReadAloudSnapshot); +} + +/** Stop playback and cancel daemon-side synthesis. Safe to call when idle. */ +export function stopReadAloud(): void { + generation += 1; + handle?.cancel(); + handle = null; + ownerId = null; + ownerServerId = null; + pendingSegmentPlaybacks = 0; + streamEnded = false; + stopReadAloudAudio(); + setSnapshot({ status: "idle", failure: null }); +} + +export function startReadAloud(params: { + client: DaemonClient; + text: string; + /** Assistant message id of the turn being read; surfaces as `snapshot.ownerId`. */ + ownerId: string; + /** Host the turn belongs to; playback stops when the route leaves it. */ + serverId: string; +}): void { + stopReadAloud(); + + if (!isReadAloudAudioSupported) { + setSnapshot({ + status: "idle", + failure: { code: "unsupported_platform", message: "Read aloud is not available here" }, + }); + return; + } + + // Set after the guard above: a rejected start never takes the slot, so a + // failure surfaces without a bystander footer rendering itself as the owner. + ownerId = params.ownerId; + ownerServerId = params.serverId; + + const token = generation; + setSnapshot({ status: "loading", failure: null }); + + handle = params.client.startReadAloud({ + text: params.text, + onSegment: (segment) => { + if (token !== generation) { + return; + } + pendingSegmentPlaybacks += 1; + if (snapshot.status === "loading") { + setSnapshot({ status: "speaking", failure: null }); + } + const settleSegment = (error?: unknown) => { + if (token !== generation) { + return; + } + pendingSegmentPlaybacks = Math.max(0, pendingSegmentPlaybacks - 1); + // A stop is not a failure — `stopReadAloud` already bumped `generation`, + // so anything reaching here is a genuine decode/playback problem. Report + // it: silently swallowing meant a broken segment looked like success + // with no audio, which is indistinguishable from "the feature is dead". + if (error !== undefined) { + setSnapshot({ + status: snapshot.status, + failure: { + code: "playback_failed", + message: error instanceof Error ? error.message : String(error), + }, + }); + } + finishIfDone(token); + }; + void playReadAloudSegment({ + audioBase64: segment.audioBase64, + format: segment.format, + isCancelled: () => token !== generation, + }).then( + () => settleSegment(), + (error: unknown) => settleSegment(error ?? new Error("Playback failed")), + ); + }, + onError: (error) => { + if (token !== generation) { + return; + } + setSnapshot({ status: snapshot.status, failure: { ...error } }); + }, + onEnd: () => { + if (token !== generation) { + return; + } + streamEnded = true; + finishIfDone(token); + }, + }); +} diff --git a/packages/app/src/read-aloud/turn-read-aloud-button.tsx b/packages/app/src/read-aloud/turn-read-aloud-button.tsx new file mode 100644 index 0000000000..d3a242c77c --- /dev/null +++ b/packages/app/src/read-aloud/turn-read-aloud-button.tsx @@ -0,0 +1,176 @@ +import { memo, useCallback, useMemo, type ReactNode } from "react"; +import { Pressable, type StyleProp, type ViewStyle } from "react-native"; +import { useTranslation } from "react-i18next"; +import { StyleSheet } from "react-native-unistyles"; +import { CircleStop, Volume2, VolumeX } from "lucide-react-native"; + +import { LoadingSpinner } from "@/components/ui/loading-spinner"; +import { isReadAloudAudioSupported } from "@/read-aloud/read-aloud-audio"; +import { + startReadAloud, + stopReadAloud, + useReadAloudSnapshot, + type ReadAloudFailure, + type ReadAloudSnapshot, +} from "@/read-aloud/read-aloud-store"; +import { useReadAloudServerId } from "@/read-aloud/use-read-aloud-host"; +import { useHostRuntimeClient } from "@/runtime/host-runtime"; + +const turnReadAloudButtonStylesheet = StyleSheet.create((theme) => ({ + container: { + alignSelf: "center", + padding: theme.spacing[1], + marginTop: 0, + }, + iconColor: { + color: theme.colors.foregroundMuted, + }, + iconHoveredColor: { + color: theme.colors.foreground, + }, + iconFailedColor: { + color: theme.colors.destructive, + }, +})); + +interface TurnReadAloudButtonProps { + /** Assistant message id — identifies this turn as the playback owner. */ + turnId: string; + /** The turn's closing prose. An empty string hides the button. */ + getSpeech: () => string; + containerStyle?: StyleProp; +} + +function renderIcon(params: { + status: ReadAloudSnapshot["status"]; + failed: boolean; + color: string; +}): ReactNode { + if (params.status === "loading") { + return ; + } + if (params.status === "speaking") { + // Outlined circle around the stop square: a filled square alone reads as a + // solid blob at footer size, next to the outlined copy and fork icons. + return ; + } + if (params.failed) { + return ; + } + return ; +} + +// `hovered` is optional: Pressable's render prop types it as `boolean | undefined` +// on platforms with no pointer. +function resolveIconColor(params: { + failed: boolean; + hovered: boolean | undefined; + status: ReadAloudSnapshot["status"]; +}): string { + if (params.failed) { + return turnReadAloudButtonStylesheet.iconFailedColor.color; + } + // Playback keeps the button at full strength so the stop affordance reads as + // active without the pointer sitting on it. + if (params.hovered || params.status !== "idle") { + return turnReadAloudButtonStylesheet.iconHoveredColor.color; + } + return turnReadAloudButtonStylesheet.iconColor.color; +} + +/** + * Why a read failed, in the user's language. + * + * The daemon's own `failure.message` is deliberately not shown: it is English, + * untranslated, and written for a log. Unknown codes fall back to the generic + * string rather than leaking it. + */ +function useFailureLabel(failure: ReadAloudFailure | null): string | null { + const { t } = useTranslation(); + if (!failure) { + return null; + } + switch (failure.code) { + case "tts_unavailable": + return t("readAloud.errors.ttsUnavailable"); + case "text_too_long": + return t("readAloud.errors.tooLong"); + case "empty_text": + return t("readAloud.errors.empty"); + case "unsupported_platform": + return t("readAloud.errors.unsupported"); + default: + return t("readAloud.errors.failed"); + } +} + +/** + * Speak the end of an assistant turn. + * + * Playback is a single app-wide slot, so this compares the store's `ownerId` + * against its own turn: pressing a second turn's button supersedes the first + * rather than stacking two voices. + * + * Hidden when the turn has nothing to say after its last tool call, when the + * route host doesn't advertise the capability, and on platforms with no audio + * engine — a button that cannot make sound is worse than no button. + */ +export const TurnReadAloudButton = memo(function TurnReadAloudButton({ + turnId, + getSpeech, + containerStyle, +}: TurnReadAloudButtonProps) { + const { t } = useTranslation(); + const snapshot = useReadAloudSnapshot(); + const serverId = useReadAloudServerId(); + const client = useHostRuntimeClient(serverId ?? ""); + + const isOwner = snapshot.ownerId === turnId; + const status = isOwner ? snapshot.status : "idle"; + const failure = isOwner ? snapshot.failure : null; + const failed = failure !== null; + const failureLabel = useFailureLabel(failure); + + const handlePress = useCallback(() => { + if (status !== "idle") { + stopReadAloud(); + return; + } + const text = getSpeech(); + if (!text || !client || !serverId) { + return; + } + startReadAloud({ client, text, ownerId: turnId, serverId }); + }, [client, getSpeech, serverId, status, turnId]); + + const pressableStyle = useMemo( + () => [turnReadAloudButtonStylesheet.container, containerStyle], + [containerStyle], + ); + + // Called during render rather than in the press handler so a turn ending on a + // tool call never shows a button that would do nothing. + const hasSpeech = getSpeech().length > 0; + if (!hasSpeech || !serverId || !isReadAloudAudioSupported) { + return null; + } + + return ( + + {({ hovered }) => + renderIcon({ status, failed, color: resolveIconColor({ failed, hovered, status }) }) + } + + ); +}); diff --git a/packages/app/src/read-aloud/use-read-aloud-host.ts b/packages/app/src/read-aloud/use-read-aloud-host.ts new file mode 100644 index 0000000000..c3eb15a413 --- /dev/null +++ b/packages/app/src/read-aloud/use-read-aloud-host.ts @@ -0,0 +1,41 @@ +import { useMemo } from "react"; +import { useLocalSearchParams, usePathname } from "expo-router"; + +import { useHostFeatureMap } from "@/runtime/host-features"; +import { parseActiveWorkspaceSelection } from "@/stores/navigation-active-workspace-store/navigation"; + +function useRouteServerId(): string | null { + const params = useLocalSearchParams<{ + serverId?: string | string[]; + workspaceId?: string | string[]; + }>(); + const pathname = usePathname(); + // Read-only: unlike `useActiveWorkspaceSelection`, this must not also record + // the workspace as "last visited" — that stays owned by the workspace screen. + return parseActiveWorkspaceSelection({ pathname, params })?.serverId ?? null; +} + +/** + * Which host synthesizes the speech. + * + * The turn is spoken by the host that owns the route it came from, never another + * paired daemon. The text being read *is* workspace content — code, agent output + * — so sending it to a different host would disclose it across an independently + * paired daemon boundary. There is deliberately no fallback: a route host that + * doesn't advertise the capability shows no button, and so does a route with no + * host at all (settings, history). + * + * Reachability is not checked here: the daemon client resolves to `null` before + * it has a transport, and a host that drops mid-request surfaces the failure on + * the button. + */ +export function useReadAloudServerId(): string | null { + const routeServerId = useRouteServerId(); + const serverIds = useMemo(() => (routeServerId ? [routeServerId] : []), [routeServerId]); + const featureByServerId = useHostFeatureMap(serverIds, "readAloud"); + + if (!routeServerId) { + return null; + } + return featureByServerId.get(routeServerId) === true ? routeServerId : null; +} diff --git a/packages/app/src/read-aloud/use-read-aloud-route-guard.ts b/packages/app/src/read-aloud/use-read-aloud-route-guard.ts new file mode 100644 index 0000000000..a314a96b1b --- /dev/null +++ b/packages/app/src/read-aloud/use-read-aloud-route-guard.ts @@ -0,0 +1,39 @@ +import { useEffect } from "react"; + +import { + getReadAloudSnapshot, + stopReadAloud, + useReadAloudSnapshot, +} from "@/read-aloud/read-aloud-store"; +import { useReadAloudServerId } from "@/read-aloud/use-read-aloud-host"; + +/** + * Stop playback when the route leaves the host that started it. + * + * The Stop control lives in a turn footer, so navigating to another workspace + * unmounts it while the module-level playback keeps going — audio with no way + * to stop it, and every button on the new route showing idle because none of + * them owns the old turn id. + * + * Mounted once at the app root. Per-button cleanup would be wrong: footers + * unmount during ordinary list virtualization, which must not stop playback. + */ +export function useReadAloudRouteGuard(): void { + const routeServerId = useReadAloudServerId(); + const { ownerServerId } = useReadAloudSnapshot(); + + useEffect(() => { + if (!ownerServerId) { + return; + } + if (routeServerId === ownerServerId) { + return; + } + // Re-read rather than trusting the closed-over value: the effect can run a + // tick after a read already finished on its own. + if (getReadAloudSnapshot().ownerServerId !== ownerServerId) { + return; + } + stopReadAloud(); + }, [ownerServerId, routeServerId]); +} diff --git a/packages/app/src/voice/audio-engine.web.test.ts b/packages/app/src/voice/audio-engine.web.test.ts new file mode 100644 index 0000000000..1e1a4d7786 --- /dev/null +++ b/packages/app/src/voice/audio-engine.web.test.ts @@ -0,0 +1,102 @@ +// @vitest-environment jsdom +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { createAudioEngine } from "@/voice/audio-engine"; + +/** + * A segment decodes across two awaits before it registers as the active + * playback. A stop landing in that window has nothing to cancel, so without a + * generation check the segment starts playing after the user asked for silence. + */ +const startedSources: string[] = []; + +class FakeBufferSource { + buffer: unknown = null; + playbackRate = { value: 1 }; + connect = vi.fn(); + stop = vi.fn(); + addEventListener = vi.fn(); + start = vi.fn(() => { + startedSources.push("started"); + }); +} + +/** Resolves only when the test says so, so a stop can land mid-decode. */ +let releaseDecode: (() => void) | null = null; + +class FakeAudioContext { + state = "running"; + destination = {}; + sampleRate = 24000; + resume = vi.fn().mockResolvedValue(undefined); + close = vi.fn().mockResolvedValue(undefined); + createBufferSource = vi.fn(() => new FakeBufferSource()); + createGain = vi.fn(() => ({ connect: vi.fn(), gain: { value: 1 } })); + createBuffer = vi.fn(() => ({ + duration: 1, + getChannelData: () => new Float32Array(24000), + copyToChannel: vi.fn(), + })); + decodeAudioData = vi.fn( + () => + new Promise((resolve) => { + releaseDecode = () => + resolve({ duration: 1, getChannelData: () => new Float32Array(24000) }); + }), + ); +} + +beforeEach(() => { + startedSources.length = 0; + releaseDecode = null; + // The engine reads `window.AudioContext`, not the bare global. + vi.stubGlobal("AudioContext", FakeAudioContext); + (window as unknown as { AudioContext: unknown }).AudioContext = FakeAudioContext; +}); + +/** Playback never invokes these — only capture does — but the type requires them. */ +function playbackOnlyCallbacks() { + return { onCaptureData: vi.fn(), onVolumeLevel: vi.fn() }; +} + +function mp3Segment() { + return { + size: 8, + type: "audio/mpeg", + async arrayBuffer() { + return new ArrayBuffer(8); + }, + }; +} + +describe("read-aloud playback cancellation", () => { + it("drops a segment that finishes decoding after stop", async () => { + const engine = createAudioEngine(playbackOnlyCallbacks()); + await engine.initialize(); + + const playback = engine.play(mp3Segment()).catch(() => "rejected"); + // Let the queue reach decodeAudioData, which is now pending. + await vi.waitFor(() => expect(releaseDecode).not.toBeNull()); + + engine.stop(); + releaseDecode?.(); + await playback; + + // The decode completed, but the segment must never reach the output. + expect(startedSources).toHaveLength(0); + }); + + it("drops a segment that finishes decoding after clearQueue", async () => { + const engine = createAudioEngine(playbackOnlyCallbacks()); + await engine.initialize(); + + const playback = engine.play(mp3Segment()).catch(() => "rejected"); + await vi.waitFor(() => expect(releaseDecode).not.toBeNull()); + + engine.clearQueue(); + releaseDecode?.(); + await playback; + + expect(startedSources).toHaveLength(0); + }); +}); diff --git a/packages/app/src/voice/audio-engine.web.ts b/packages/app/src/voice/audio-engine.web.ts index 0453c0ae33..4d68a71ad8 100644 --- a/packages/app/src/voice/audio-engine.web.ts +++ b/packages/app/src/voice/audio-engine.web.ts @@ -106,6 +106,13 @@ export function createAudioEngine( reject: (error: Error) => void; settled: boolean; } | null; + /** + * Bumped by every `stop()`/`clearQueue()`. A segment decodes across two + * awaits before it registers as `activePlayback`, so a stop landing in that + * window has nothing to cancel — without this the segment would start + * playing after the user asked for silence. + */ + playbackGeneration: number; } = { playbackContext: null, captureContext: null, @@ -118,6 +125,7 @@ export function createAudioEngine( queue: [], processingQueue: false, activePlayback: null, + playbackGeneration: 0, }; async function ensurePlaybackContext(): Promise { @@ -163,6 +171,7 @@ export function createAudioEngine( } async function playAudio(audio: AudioPlaybackSource): Promise { + const generation = refs.playbackGeneration; const context = await ensurePlaybackContext(); const arrayBuffer = await audio.arrayBuffer(); const type = (audio.type || "").toLowerCase(); @@ -174,10 +183,16 @@ export function createAudioEngine( ) : await decodeAudioData(context, arrayBuffer); - const durationSec = audioBuffer.duration; + // Re-check after the awaits above: a stop during context init or decode + // could not reach this segment, so it has to bow out on its own. + if (generation !== refs.playbackGeneration) { + throw new Error("Playback stopped"); + } + const source = context.createBufferSource(); source.buffer = audioBuffer; source.connect(context.destination); + const durationSec = audioBuffer.duration; return await new Promise((resolve, reject) => { refs.activePlayback = { source, resolve, reject, settled: false }; @@ -380,6 +395,7 @@ export function createAudioEngine( }, stop() { + refs.playbackGeneration += 1; if (refs.activePlayback) { const active = refs.activePlayback; refs.activePlayback = null; @@ -396,6 +412,7 @@ export function createAudioEngine( }, clearQueue() { + refs.playbackGeneration += 1; while (refs.queue.length > 0) { refs.queue.shift()!.reject(new Error("Playback stopped")); } diff --git a/packages/client/src/daemon-client.ts b/packages/client/src/daemon-client.ts index 72b0cb61ae..dd5783ead5 100644 --- a/packages/client/src/daemon-client.ts +++ b/packages/client/src/daemon-client.ts @@ -437,6 +437,24 @@ export interface FileUploadInput { requestId?: string; chunkSize?: number; } + +export interface ReadAloudSegment { + segmentIndex: number; + segmentCount: number; + audioBase64: string; + /** Provider-reported format, e.g. `pcm;rate=24000` or `mp3`. */ + format: string; +} + +export interface ReadAloudError { + code: string; + message: string; +} + +export interface ReadAloudHandle { + requestId: string; + cancel(): void; +} export type FileUploadResult = FileUploadResponse["payload"]; type FileDownloadTokenPayload = FileDownloadTokenResponse["payload"]; type ListProviderFeaturesPayload = ListProviderFeaturesResponseMessage["payload"]; @@ -3545,6 +3563,81 @@ export class DaemonClient { this.sendSessionMessage({ type: "audio_played", id }); } + /** + * Synthesize `text` on the daemon and stream the audio back segment by + * segment, in order. Segments arrive as they finish synthesizing so playback + * can start on the first sentence; the caller is responsible for queueing + * them. Returns a handle whose `cancel()` stops synthesis and detaches the + * listener. + */ + startReadAloud(params: { + text: string; + onSegment: (segment: ReadAloudSegment) => void; + onError: (error: ReadAloudError) => void; + onEnd: () => void; + }): ReadAloudHandle { + const requestId = this.createRequestId(); + let settled = false; + + const unsubscribe = this.on("speech.tts.read_aloud.response", (message) => { + const payload = message.payload; + if (payload.requestId !== requestId || settled) { + return; + } + + if (payload.error) { + settled = true; + unsubscribe(); + params.onError(payload.error); + params.onEnd(); + return; + } + + if (payload.audio !== undefined && payload.format !== undefined) { + params.onSegment({ + segmentIndex: payload.segmentIndex, + segmentCount: payload.segmentCount, + audioBase64: payload.audio, + format: payload.format, + }); + } + + if (payload.isLast) { + settled = true; + unsubscribe(); + params.onEnd(); + } + }); + + try { + this.sendSessionMessageStrict({ + type: "speech.tts.read_aloud.request", + requestId, + text: params.text, + }); + } catch (error) { + settled = true; + unsubscribe(); + params.onError({ + code: "transport_unavailable", + message: error instanceof Error ? error.message : String(error), + }); + params.onEnd(); + } + + return { + requestId, + cancel: () => { + if (settled) { + return; + } + settled = true; + unsubscribe(); + this.sendSessionMessage({ type: "speech.tts.cancel_read_aloud.request", requestId }); + }, + }; + } + // ============================================================================ // Git Operations // ============================================================================ diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index a0fdbd5e77..b45352b32e 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -32,6 +32,9 @@ export type { DaemonEvent, BrowserAutomationExecuteRequestMessage, BrowserAutomationExecuteResponseMessage, + ReadAloudError, + ReadAloudHandle, + ReadAloudSegment, WebSocketFactory, WebSocketLike, } from "./daemon-client.js"; diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index 1b011bc2db..679ff80172 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -1272,6 +1272,24 @@ export const DictationStreamCancelMessageSchema = z.object({ dictationId: z.string(), }); +// ============================================================================ +// Read aloud (on-demand TTS for arbitrary client-selected text) +// ============================================================================ + +export const ReadAloudRequestMessageSchema = z.object({ + type: z.literal("speech.tts.read_aloud.request"), + requestId: z.string(), + text: z.string(), +}); + +// Fire-and-forget: cancelling read-aloud has no `.response`. The daemon stops +// synthesizing and stops emitting `speech.tts.read_aloud.response` segments for +// this requestId; the client has already torn down its own playback. +export const ReadAloudCancelRequestMessageSchema = z.object({ + type: z.literal("speech.tts.cancel_read_aloud.request"), + requestId: z.string(), +}); + const GitSetupOptionsSchema = z.object({ baseBranch: z.string().optional(), createNewBranch: z.boolean().optional(), @@ -2531,6 +2549,8 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ HubExecutionControlRequestSchema, BrowserAutomationExecuteResponseSchema, VoiceAudioChunkMessageSchema, + ReadAloudRequestMessageSchema, + ReadAloudCancelRequestMessageSchema, AbortRequestMessageSchema, AudioPlayedMessageSchema, FetchAgentsRequestMessageSchema, @@ -2795,6 +2815,29 @@ export const DictationStreamErrorMessageSchema = z.object({ }), }); +// One request fans out to N of these, in segment order. Audio is segmented so a +// long message starts playing after the first sentence instead of after the +// whole synthesis, and so no single message carries minutes of raw PCM. +export const ReadAloudResponseMessageSchema = z.object({ + type: z.literal("speech.tts.read_aloud.response"), + payload: z.object({ + requestId: z.string(), + segmentIndex: z.number().int().nonnegative(), + segmentCount: z.number().int().nonnegative(), + isLast: z.boolean(), + // Base64 audio for this segment. Absent on the terminal error message. + audio: z.string().optional(), + // Provider-reported audio format, e.g. "pcm;rate=24000" or "mp3". + format: z.string().optional(), + error: z + .object({ + code: z.string(), + message: z.string(), + }) + .optional(), + }), +}); + export const ServerCapabilityStateSchema = z.object({ enabled: z.boolean(), reason: z.string(), @@ -2898,6 +2941,8 @@ export const ServerInfoStatusPayloadSchema = z agentDetach: z.boolean().optional(), // COMPAT(agentThinkingUpdate): added in v0.2.4, remove gate after 2027-01-28. agentThinkingUpdate: z.boolean().optional(), + // COMPAT(readAloud): added in v0.2.5, drop the gate when floor >= v0.2.5. + readAloud: z.boolean().optional(), // COMPAT(daemonDiagnostics): added in v0.1.100, remove gate after 2026-12-25 once daemon floor >= v0.1.100. daemonDiagnostics: z.boolean().optional(), // COMPAT(daemonSelfUpdate): added in v0.1.93, remove gate after 2026-12-13. @@ -5346,6 +5391,7 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [ ActivityLogMessageSchema, AssistantChunkMessageSchema, AudioOutputMessageSchema, + ReadAloudResponseMessageSchema, TranscriptionResultMessageSchema, VoiceInputStateMessageSchema, DictationStreamAckMessageSchema, @@ -5701,6 +5747,10 @@ export type DictationStreamStartMessage = z.infer; export type DictationStreamFinishMessage = z.infer; export type DictationStreamCancelMessage = z.infer; +export type ReadAloudRequestMessage = z.infer; +export type ReadAloudCancelRequestMessage = z.infer; +export type ReadAloudResponseMessage = z.infer; +export type ReadAloudResponsePayload = ReadAloudResponseMessage["payload"]; export type CreateAgentRequestMessage = z.infer; export type AgentAttachment = z.infer; export type ForgeChangeRequestAttachment = z.infer; diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 13c58ce0d6..dea144f372 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -553,7 +553,12 @@ async function createControlledInterruptFixture(options: { })(); await manager.waitForAgentRunStart(agent.id); }, - cleanup: () => rmSync(workdir, { recursive: true, force: true }), + // Retry the teardown: AgentManager can flush an agent snapshot into + // `agents/` while this runs, and on Windows an open handle makes rmdir + // fail with ENOTEMPTY, which `force` does not cover. Same options as + // spawn.launch-regression.test.ts and run-git-command.windows-shell.test.ts. + cleanup: () => + rmSync(workdir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }), }; } diff --git a/packages/server/src/server/agent/tts-manager.ts b/packages/server/src/server/agent/tts-manager.ts index ee6c747019..b343cf6751 100644 --- a/packages/server/src/server/agent/tts-manager.ts +++ b/packages/server/src/server/agent/tts-manager.ts @@ -3,6 +3,7 @@ import type { Readable } from "node:stream"; import { v4 as uuidv4 } from "uuid"; import type { TextToSpeechProvider } from "../speech/speech-provider.js"; import { toResolver, type Resolvable } from "../speech/provider-resolver.js"; +import { splitTextForTts, type TtsSegment } from "../speech/tts-text-splitter.js"; import type { SessionOutboundMessage } from "../messages.js"; interface PendingPlayback { @@ -12,11 +13,6 @@ interface PendingPlayback { streamEnded: boolean; } -interface TtsSegment { - index: number; - text: string; -} - type PreparedTtsSegment = TtsSegment & { format: string; stream: Readable; @@ -27,103 +23,9 @@ type PreparedSegmentResult = | { kind: "aborted" } | { kind: "error"; error: unknown }; -const MAX_TTS_SEGMENT_CHARS = 260; const TTS_PREFETCH_SEGMENTS = 2; const CLOSED_AUDIO_ID_TTL_MS = 10_000; -function splitOversizedFragment(fragment: string, maxChars: number): string[] { - const trimmed = fragment.trim(); - if (!trimmed) { - return []; - } - - if (trimmed.length <= maxChars) { - return [trimmed]; - } - - const clauseChunks = trimmed.split(/(?<=[,;:])\s+/); - if (clauseChunks.length > 1) { - const parts: string[] = []; - let current = ""; - - const pushCurrent = () => { - const value = current.trim(); - if (value) { - parts.push(value); - } - current = ""; - }; - - for (const clause of clauseChunks) { - const clauseText = clause.trim(); - if (!clauseText) { - continue; - } - - if (clauseText.length > maxChars) { - pushCurrent(); - parts.push(...splitOversizedFragment(clauseText, maxChars)); - continue; - } - - if (!current) { - current = clauseText; - continue; - } - - const candidate = `${current} ${clauseText}`; - if (candidate.length <= maxChars) { - current = candidate; - continue; - } - - pushCurrent(); - current = clauseText; - } - - pushCurrent(); - if (parts.length > 1 || parts[0] !== trimmed) { - return parts; - } - } - - const parts: string[] = []; - let remaining = trimmed; - while (remaining.length > maxChars) { - let idx = remaining.lastIndexOf(" ", maxChars); - if (idx < Math.floor(maxChars * 0.5)) { - idx = maxChars; - } - parts.push(remaining.slice(0, idx).trim()); - remaining = remaining.slice(idx).trim(); - } - if (remaining.length > 0) { - parts.push(remaining); - } - return parts; -} - -function splitTextForTts(text: string): TtsSegment[] { - const normalized = text.trim().replace(/\s+/g, " "); - if (!normalized) { - throw new Error("Cannot synthesize empty text"); - } - - const sentences = normalized.split(/(?<=[.!?])\s+/); - const parts: TtsSegment[] = []; - let segmentIndex = 0; - - for (const sentence of sentences) { - const fragments = splitOversizedFragment(sentence, MAX_TTS_SEGMENT_CHARS); - for (const fragment of fragments) { - parts.push({ index: segmentIndex, text: fragment }); - segmentIndex += 1; - } - } - - return parts; -} - /** * Per-session TTS manager * Handles TTS audio generation and playback confirmation tracking diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 0ae81a21c3..82a76d5229 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -1864,6 +1864,16 @@ export class Session { case "dictation_stream_cancel": this.voiceSession.handleDictationCancel(msg.dictationId); return undefined; + case "speech.tts.read_aloud.request": + // Not awaited: this spans the whole synthesis (seconds to a minute for a + // long message), and the caller measures the awaited duration as + // request latency. Read aloud is a stream of `.response` segments, not a + // request/response, and the controller reports its own failures. + void this.voiceSession.handleReadAloudRequest(msg); + return undefined; + case "speech.tts.cancel_read_aloud.request": + this.voiceSession.handleReadAloudCancel(msg); + return undefined; case "restart_server_request": return this.handleRestartServerRequest(msg.requestId, msg.reason); case "shutdown_server_request": diff --git a/packages/server/src/server/session/voice/read-aloud-controller.test.ts b/packages/server/src/server/session/voice/read-aloud-controller.test.ts new file mode 100644 index 0000000000..1c488caf3c --- /dev/null +++ b/packages/server/src/server/session/voice/read-aloud-controller.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it } from "vitest"; +import pino from "pino"; +import { Readable } from "node:stream"; + +import { MAX_READ_ALOUD_CHARS, ReadAloudController } from "./read-aloud-controller.js"; +import type { SessionOutboundMessage } from "../../messages.js"; +import type { TextToSpeechProvider } from "../../speech/speech-provider.js"; + +type ReadAloudResponse = Extract< + SessionOutboundMessage, + { type: "speech.tts.read_aloud.response" } +>; + +interface Harness { + controller: ReadAloudController; + responses: ReadAloudResponse[]; + synthesized: string[]; +} + +function createHarness( + tts: TextToSpeechProvider | null, + hooks?: { + onResponse?: (response: ReadAloudResponse, harness: Harness) => void; + }, +): Harness { + const responses: ReadAloudResponse[] = []; + const synthesized: string[] = []; + + const wrapped: TextToSpeechProvider | null = tts + ? { + async synthesizeSpeech(text: string) { + synthesized.push(text); + return tts.synthesizeSpeech(text); + }, + } + : null; + + const harness: Harness = { + controller: new ReadAloudController({ + sessionId: "session-1", + logger: pino({ level: "silent" }), + tts: wrapped, + emit: (message) => { + if (message.type !== "speech.tts.read_aloud.response") { + return; + } + responses.push(message); + hooks?.onResponse?.(message, harness); + }, + }), + responses, + synthesized, + }; + + return harness; +} + +function fakeTts(audio = "sound"): TextToSpeechProvider { + return { + async synthesizeSpeech() { + return { stream: Readable.from([Buffer.from(audio)]), format: "pcm;rate=24000" }; + }, + }; +} + +describe("ReadAloudController", () => { + it("streams one audio segment per sentence, in order, marking only the last", async () => { + const { controller, responses, synthesized } = createHarness(fakeTts("abc")); + + await controller.handleRequest({ + requestId: "req-1", + text: "First sentence. Second sentence. Third sentence.", + }); + + expect(synthesized).toEqual(["First sentence.", "Second sentence.", "Third sentence."]); + expect(responses.map((message) => message.payload.segmentIndex)).toEqual([0, 1, 2]); + expect(responses.map((message) => message.payload.isLast)).toEqual([false, false, true]); + + for (const message of responses) { + expect(message.payload.requestId).toBe("req-1"); + expect(message.payload.segmentCount).toBe(3); + expect(message.payload.format).toBe("pcm;rate=24000"); + expect(message.payload.audio).toBe(Buffer.from("abc").toString("base64")); + expect(message.payload.error).toBeUndefined(); + } + }); + + it("never sends wrapper markup to the provider", async () => { + const { controller, synthesized } = createHarness(fakeTts()); + + await controller.handleRequest({ + requestId: "req-1", + text: [ + "", + "Are you working?", + "", + "This message was spoken by the user.", + ].join("\n"), + }); + + expect(synthesized).toEqual(["Are you working?", "This message was spoken by the user."]); + }); + + it("rejects a selection that is nothing but markup", async () => { + const { controller, responses, synthesized } = createHarness(fakeTts()); + + await controller.handleRequest({ requestId: "req-1", text: "" }); + + expect(synthesized).toEqual([]); + expect(responses[0].payload.error?.code).toBe("empty_text"); + }); + + it("reports tts_unavailable without emitting audio when no provider is configured", async () => { + const { controller, responses } = createHarness(null); + + await controller.handleRequest({ requestId: "req-1", text: "Read me." }); + + expect(responses).toHaveLength(1); + expect(responses[0].payload.error?.code).toBe("tts_unavailable"); + expect(responses[0].payload.isLast).toBe(true); + expect(responses[0].payload.audio).toBeUndefined(); + }); + + it("rejects text over the length cap instead of synthesizing it", async () => { + const { controller, responses, synthesized } = createHarness(fakeTts()); + + await controller.handleRequest({ + requestId: "req-1", + text: "word ".repeat(MAX_READ_ALOUD_CHARS), + }); + + expect(synthesized).toEqual([]); + expect(responses).toHaveLength(1); + expect(responses[0].payload.error?.code).toBe("text_too_long"); + }); + + it("rejects a whitespace-only selection", async () => { + const { controller, responses, synthesized } = createHarness(fakeTts()); + + await controller.handleRequest({ requestId: "req-1", text: " \n " }); + + expect(synthesized).toEqual([]); + expect(responses[0].payload.error?.code).toBe("empty_text"); + }); + + it("reports synth_failed as a terminal error when the provider throws", async () => { + const failing: TextToSpeechProvider = { + async synthesizeSpeech() { + throw new Error("model not loaded"); + }, + }; + const { controller, responses } = createHarness(failing); + + await controller.handleRequest({ requestId: "req-1", text: "Read me." }); + + expect(responses).toHaveLength(1); + expect(responses[0].payload.error).toEqual({ + code: "synth_failed", + message: "model not loaded", + }); + expect(responses[0].payload.isLast).toBe(true); + }); + + it("stops emitting further segments once the client cancels mid-stream", async () => { + const { controller, responses } = createHarness(fakeTts(), { + onResponse: (_response, harness) => { + if (harness.responses.length === 1) { + harness.controller.cancel("req-1"); + } + }, + }); + + await controller.handleRequest({ requestId: "req-1", text: "One. Two. Three." }); + + expect(responses).toHaveLength(1); + expect(responses[0].payload.segmentIndex).toBe(0); + expect(responses[0].payload.isLast).toBe(false); + }); + + it("supersedes an in-flight request when a new one arrives", async () => { + let releaseFirst: (() => void) | null = null; + const firstBlocked = new Promise((resolve) => { + releaseFirst = resolve; + }); + + const slowTts: TextToSpeechProvider = { + async synthesizeSpeech(text: string) { + if (text === "Slow.") { + await firstBlocked; + } + return { stream: Readable.from([Buffer.from("x")]), format: "pcm;rate=24000" }; + }, + }; + + const { controller, responses } = createHarness(slowTts); + + const first = controller.handleRequest({ requestId: "req-1", text: "Slow." }); + const second = controller.handleRequest({ requestId: "req-2", text: "Fast." }); + releaseFirst?.(); + await Promise.all([first, second]); + + expect(responses.map((message) => message.payload.requestId)).toEqual(["req-2"]); + }); +}); diff --git a/packages/server/src/server/session/voice/read-aloud-controller.ts b/packages/server/src/server/session/voice/read-aloud-controller.ts new file mode 100644 index 0000000000..c4a9653aa5 --- /dev/null +++ b/packages/server/src/server/session/voice/read-aloud-controller.ts @@ -0,0 +1,275 @@ +import type pino from "pino"; +import type { Readable } from "node:stream"; + +import type { SessionOutboundMessage } from "../../messages.js"; +import { toResolver, type Resolvable } from "../../speech/provider-resolver.js"; +import type { TextToSpeechProvider } from "../../speech/speech-provider.js"; +import { hasSpeakableContent, sanitizeTextForReadAloud } from "../../speech/read-aloud-text.js"; +import { splitTextForTts, type TtsSegment } from "../../speech/tts-text-splitter.js"; + +/** + * Upper bound on a single read-aloud request. Guards the daemon against a + * "select all" that would otherwise queue minutes of synthesis work. + */ +export const MAX_READ_ALOUD_CHARS = 4000; + +/** How many segments are synthesized ahead of the one currently being emitted. */ +const READ_ALOUD_PREFETCH_SEGMENTS = 2; + +export type ReadAloudErrorCode = + | "empty_text" + | "text_too_long" + | "tts_unavailable" + | "synth_failed"; + +interface ActiveRequest { + abortController: AbortController; +} + +type PreparedSegment = TtsSegment & { + format: string; + stream: Readable; +}; + +export interface ReadAloudControllerOptions { + sessionId: string; + logger: pino.Logger; + tts: Resolvable; + emit: (message: SessionOutboundMessage) => void; +} + +/** + * On-demand text-to-speech for client-selected text ("Read aloud"). + * + * Deliberately separate from `TTSManager`: voice-mode audio carries an + * `isVoiceMode` drift flag and gates an agent turn on playback confirmation, + * neither of which applies here. Read aloud is fire-and-forget streaming — + * segments are pushed as they finish synthesizing and the client queues them. + */ +export class ReadAloudController { + private readonly logger: pino.Logger; + private readonly resolveTts: () => TextToSpeechProvider | null; + private readonly emit: (message: SessionOutboundMessage) => void; + private readonly active = new Map(); + + constructor(options: ReadAloudControllerOptions) { + this.logger = options.logger.child({ + module: "speech", + component: "read-aloud", + sessionId: options.sessionId, + }); + this.resolveTts = toResolver(options.tts); + this.emit = options.emit; + } + + public async handleRequest(params: { requestId: string; text: string }): Promise { + const { requestId } = params; + + // Only one read aloud plays at a time, so a new request supersedes whatever + // is still synthesizing rather than competing with it for the TTS worker. + this.cancelAll("superseded by a newer read aloud request"); + + // Cap on what the user selected, before sanitizing, so the limit means the + // same thing to them whether or not the message was full of markup. + const rawLength = params.text.trim().length; + const text = sanitizeTextForReadAloud(params.text); + if (!hasSpeakableContent(text)) { + this.emitError(requestId, "empty_text", "Nothing to read aloud"); + return; + } + + if (rawLength > MAX_READ_ALOUD_CHARS) { + this.emitError( + requestId, + "text_too_long", + `Message is too long to read aloud (${rawLength} of ${MAX_READ_ALOUD_CHARS} characters)`, + ); + return; + } + + const tts = this.resolveTts(); + if (!tts) { + this.emitError(requestId, "tts_unavailable", "Text-to-speech is not configured on this host"); + return; + } + + const abortController = new AbortController(); + this.active.set(requestId, { abortController }); + const abortSignal = abortController.signal; + + // Re-index after dropping unspeakable fragments so `segmentIndex` stays a + // dense 0..n-1 run and the last segment really is the last one. + const segments: TtsSegment[] = []; + for (const segment of splitTextForTts(text)) { + if (hasSpeakableContent(segment.text)) { + segments.push({ index: segments.length, text: segment.text }); + } + } + if (segments.length === 0) { + this.emitError(requestId, "empty_text", "Nothing to read aloud"); + return; + } + const startedAtMs = Date.now(); + this.logger.debug( + { requestId, chars: text.length, segmentCount: segments.length }, + "Read aloud started", + ); + + const inflight = new Map>(); + let nextToSchedule = 0; + + const scheduleAhead = () => { + while ( + nextToSchedule < segments.length && + inflight.size < READ_ALOUD_PREFETCH_SEGMENTS && + !abortSignal.aborted + ) { + const segment = segments[nextToSchedule]; + inflight.set(segment.index, this.synthesizeSegment(tts, segment)); + nextToSchedule += 1; + } + }; + + scheduleAhead(); + + try { + for (const segment of segments) { + const pending = inflight.get(segment.index); + if (!pending) { + break; + } + + let prepared: PreparedSegment; + try { + prepared = await pending; + } finally { + inflight.delete(segment.index); + } + + if (abortSignal.aborted) { + destroyStream(prepared.stream); + return; + } + + scheduleAhead(); + + const audio = await collectStream(prepared.stream); + if (abortSignal.aborted) { + return; + } + + this.emit({ + type: "speech.tts.read_aloud.response", + payload: { + requestId, + segmentIndex: segment.index, + segmentCount: segments.length, + isLast: segment.index === segments.length - 1, + audio: audio.toString("base64"), + format: prepared.format, + }, + }); + } + + this.logger.debug( + { requestId, totalMs: Date.now() - startedAtMs, segmentCount: segments.length }, + "Read aloud completed", + ); + } catch (error) { + if (abortSignal.aborted) { + this.logger.debug({ requestId }, "Read aloud aborted during synthesis"); + return; + } + this.logger.warn({ requestId, err: error }, "Read aloud synthesis failed"); + this.emitError( + requestId, + "synth_failed", + error instanceof Error ? error.message : String(error), + { segmentIndex: segments.length, segmentCount: segments.length }, + ); + } finally { + this.active.delete(requestId); + discardPrefetched(inflight); + } + } + + public cancel(requestId: string): void { + const request = this.active.get(requestId); + if (!request) { + return; + } + this.active.delete(requestId); + request.abortController.abort(); + this.logger.debug({ requestId }, "Read aloud cancelled by client"); + } + + public dispose(): void { + this.cancelAll("session closed"); + } + + private cancelAll(reason: string): void { + if (this.active.size === 0) { + return; + } + for (const [requestId, request] of this.active.entries()) { + this.active.delete(requestId); + request.abortController.abort(); + this.logger.debug({ requestId, reason }, "Read aloud cancelled"); + } + } + + private synthesizeSegment( + tts: TextToSpeechProvider, + segment: TtsSegment, + ): Promise { + return tts + .synthesizeSpeech(segment.text) + .then(({ stream, format }) => ({ ...segment, stream, format })); + } + + private emitError( + requestId: string, + code: ReadAloudErrorCode, + message: string, + position?: { segmentIndex: number; segmentCount: number }, + ): void { + this.emit({ + type: "speech.tts.read_aloud.response", + payload: { + requestId, + segmentIndex: position?.segmentIndex ?? 0, + segmentCount: position?.segmentCount ?? 0, + isLast: true, + error: { code, message }, + }, + }); + } +} + +function destroyStream(stream: Readable): void { + if (typeof stream.destroy === "function" && !stream.destroyed) { + stream.destroy(); + } +} + +function discardPrefetched(inflight: Map>): void { + for (const pending of inflight.values()) { + void pending.then( + (prepared) => destroyStream(prepared.stream), + () => undefined, + ); + } + inflight.clear(); +} + +async function collectStream(stream: Readable): Promise { + const chunks: Buffer[] = []; + try { + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + } finally { + destroyStream(stream); + } + return Buffer.concat(chunks); +} diff --git a/packages/server/src/server/session/voice/voice-session.ts b/packages/server/src/server/session/voice/voice-session.ts index f6331dba3f..0e52afd488 100644 --- a/packages/server/src/server/session/voice/voice-session.ts +++ b/packages/server/src/server/session/voice/voice-session.ts @@ -14,6 +14,7 @@ import { type DictationStreamOutboundMessage, } from "../../dictation/dictation-stream-manager.js"; import { createVoiceTurnController, type VoiceTurnController } from "./voice-turn-controller.js"; +import { ReadAloudController } from "./read-aloud-controller.js"; import { buildVoiceModeSystemPrompt, stripVoiceModeSystemPrompt } from "../../voice-config.js"; import type { VoiceCallerContext, VoiceSpeakHandler } from "../../voice-types.js"; import type { ManagedAgent } from "../../agent/agent-manager.js"; @@ -191,6 +192,7 @@ export class VoiceSession { private readonly ttsManager: TTSManager; private readonly sttManager: STTManager; + private readonly readAloudController: ReadAloudController; private readonly registerVoiceSpeakHandler?: ( agentId: string, @@ -224,6 +226,12 @@ export class VoiceSession { this.getSpeechReadiness = dictation?.getSpeechReadiness; this.ttsManager = new TTSManager(this.sessionId, this.sessionLogger, tts); + this.readAloudController = new ReadAloudController({ + sessionId: this.sessionId, + logger: this.sessionLogger, + tts, + emit: (msg) => this.emit(msg), + }); this.sttManager = new STTManager(this.sessionId, this.sessionLogger, stt, { language: sttLanguage, }); @@ -258,6 +266,18 @@ export class VoiceSession { this.dictationStreamManager.handleCancel(dictationId); } + handleReadAloudRequest( + msg: Extract, + ): Promise { + return this.readAloudController.handleRequest({ requestId: msg.requestId, text: msg.text }); + } + + handleReadAloudCancel( + msg: Extract, + ): void { + this.readAloudController.cancel(msg.requestId); + } + async handleDictationStreamStart( msg: Extract, ): Promise { @@ -1302,6 +1322,7 @@ export class VoiceSession { this.ttsManager.cleanup(); this.sttManager.cleanup(); this.dictationStreamManager.cleanupAll(); + this.readAloudController.dispose(); await this.disableVoiceModeForActiveAgent(true); this.isVoiceMode = false; diff --git a/packages/server/src/server/speech/read-aloud-text.test.ts b/packages/server/src/server/speech/read-aloud-text.test.ts new file mode 100644 index 0000000000..4df04fba02 --- /dev/null +++ b/packages/server/src/server/speech/read-aloud-text.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; + +import { hasSpeakableContent, sanitizeTextForReadAloud } from "./read-aloud-text.js"; + +describe("sanitizeTextForReadAloud", () => { + it("drops Paseo's spoken-input wrapper and speaks only the message", () => { + const selection = [ + "", + "Are you working?", + "", + "This message was spoken by the user.", + ].join("\n"); + + expect(sanitizeTextForReadAloud(selection)).toBe( + "Are you working? This message was spoken by the user.", + ); + }); + + it("keeps comparisons that are not tags", () => { + expect(sanitizeTextForReadAloud("if a < b and c > d")).toBe("if a < b and c > d"); + }); + + it("strips markdown emphasis and heading markers but keeps the words", () => { + expect(sanitizeTextForReadAloud("## The **bold** and _quiet_ `code` part")).toBe( + "The bold and quiet code part", + ); + }); + + it("speaks a link's label, not its URL", () => { + expect(sanitizeTextForReadAloud("See [the docs](https://example.com/a/b) for more")).toBe( + "See the docs for more", + ); + }); + + it("drops code fences while keeping the code inside", () => { + const selection = ["Run this:", "```bash", "npm run dev", "```"].join("\n"); + expect(sanitizeTextForReadAloud(selection)).toBe("Run this: npm run dev"); + }); + + it("strips list bullets and numbering", () => { + expect(sanitizeTextForReadAloud("- first\n- second\n1. third")).toBe("first second third"); + }); + + it("collapses a markup-only selection to nothing", () => { + expect(sanitizeTextForReadAloud("\n")).toBe(""); + }); +}); + +describe("hasSpeakableContent", () => { + it("accepts anything containing a letter or digit", () => { + expect(hasSpeakableContent("hi")).toBe(true); + expect(hasSpeakableContent("42")).toBe(true); + expect(hasSpeakableContent("こんにちは")).toBe(true); + }); + + it("rejects punctuation-only fragments that would synthesize to silence", () => { + expect(hasSpeakableContent("--- ...")).toBe(false); + expect(hasSpeakableContent("")).toBe(false); + }); +}); diff --git a/packages/server/src/server/speech/read-aloud-text.ts b/packages/server/src/server/speech/read-aloud-text.ts new file mode 100644 index 0000000000..46d41115c4 --- /dev/null +++ b/packages/server/src/server/speech/read-aloud-text.ts @@ -0,0 +1,48 @@ +/** + * Turn an agent's message into something worth hearing. + * + * Agent output is raw screen text: it carries markdown syntax, and agent + * messages carry Paseo's own wrapper tags (``, ``). + * Synthesized verbatim those become spoken noise — "spoken input, are you + * working" — so they are stripped before they reach the TTS provider. + * + * Deliberately conservative: this removes markup, it does not reflow or + * summarize. Anything it cannot confidently classify as syntax is spoken. + */ + +/** ``, ``, `` — not bare `<` in `a < b`. */ +const HTML_LIKE_TAG = /<\/?[a-zA-Z][^<>]*>/g; + +/** Fenced code blocks: the fence markers and language tag, not the code. */ +const CODE_FENCE = /^```.*$/gm; + +/** Inline code, bold, italic, and strikethrough markers around their content. */ +const INLINE_MARKERS = /[`*_~]/g; + +/** Leading `#`, `>`, `-`, `*`, `+` and list numbering at the start of a line. */ +const LINE_LEAD_MARKERS = /^\s{0,3}(?:#{1,6}\s+|>\s?|[-*+]\s+|\d+[.)]\s+)/gm; + +/** `[label](https://…)` — the label is speakable, the URL is not. */ +const MARKDOWN_LINK = /\[([^\]]*)\]\((?:[^)]*)\)/g; + +/** Anything with a letter or a digit has something to pronounce. */ +const HAS_SPEAKABLE_CONTENT = /[\p{L}\p{N}]/u; + +export function sanitizeTextForReadAloud(text: string): string { + return text + .replace(CODE_FENCE, " ") + .replace(HTML_LIKE_TAG, " ") + .replace(MARKDOWN_LINK, "$1") + .replace(LINE_LEAD_MARKERS, "") + .replace(INLINE_MARKERS, "") + .replace(/\s+/g, " ") + .trim(); +} + +/** + * Whether a fragment is worth sending to the provider. Punctuation-only + * fragments synthesize to empty audio, which the client cannot decode. + */ +export function hasSpeakableContent(text: string): boolean { + return HAS_SPEAKABLE_CONTENT.test(text); +} diff --git a/packages/server/src/server/speech/tts-text-splitter.ts b/packages/server/src/server/speech/tts-text-splitter.ts new file mode 100644 index 0000000000..2d82eeefdc --- /dev/null +++ b/packages/server/src/server/speech/tts-text-splitter.ts @@ -0,0 +1,104 @@ +export interface TtsSegment { + index: number; + text: string; +} + +export const MAX_TTS_SEGMENT_CHARS = 260; + +function splitOversizedFragment(fragment: string, maxChars: number): string[] { + const trimmed = fragment.trim(); + if (!trimmed) { + return []; + } + + if (trimmed.length <= maxChars) { + return [trimmed]; + } + + const clauseChunks = trimmed.split(/(?<=[,;:])\s+/); + if (clauseChunks.length > 1) { + const parts: string[] = []; + let current = ""; + + const pushCurrent = () => { + const value = current.trim(); + if (value) { + parts.push(value); + } + current = ""; + }; + + for (const clause of clauseChunks) { + const clauseText = clause.trim(); + if (!clauseText) { + continue; + } + + if (clauseText.length > maxChars) { + pushCurrent(); + parts.push(...splitOversizedFragment(clauseText, maxChars)); + continue; + } + + if (!current) { + current = clauseText; + continue; + } + + const candidate = `${current} ${clauseText}`; + if (candidate.length <= maxChars) { + current = candidate; + continue; + } + + pushCurrent(); + current = clauseText; + } + + pushCurrent(); + if (parts.length > 1 || parts[0] !== trimmed) { + return parts; + } + } + + const parts: string[] = []; + let remaining = trimmed; + while (remaining.length > maxChars) { + let idx = remaining.lastIndexOf(" ", maxChars); + if (idx < Math.floor(maxChars * 0.5)) { + idx = maxChars; + } + parts.push(remaining.slice(0, idx).trim()); + remaining = remaining.slice(idx).trim(); + } + if (remaining.length > 0) { + parts.push(remaining); + } + return parts; +} + +/** + * Split text into sentence-ish segments small enough to synthesize with low + * latency. Segments are the unit of streaming for both voice mode and read + * aloud: the first segment can play while later ones are still synthesizing. + */ +export function splitTextForTts(text: string): TtsSegment[] { + const normalized = text.trim().replace(/\s+/g, " "); + if (!normalized) { + throw new Error("Cannot synthesize empty text"); + } + + const sentences = normalized.split(/(?<=[.!?])\s+/); + const parts: TtsSegment[] = []; + let segmentIndex = 0; + + for (const sentence of sentences) { + const fragments = splitOversizedFragment(sentence, MAX_TTS_SEGMENT_CHARS); + for (const fragment of fragments) { + parts.push({ index: segmentIndex, text: fragment }); + segmentIndex += 1; + } + } + + return parts; +} diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index c807554bc7..51d9415042 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -1562,6 +1562,10 @@ export class VoiceAssistantWebSocketServer { agentDetach: true, // COMPAT(agentThinkingUpdate): added in v0.2.4, remove gate after 2027-01-28. agentThinkingUpdate: true, + // COMPAT(readAloud): added in v0.2.5, drop the gate when floor >= v0.2.5. + // Advertises the `speech.tts.read_aloud.*` RPC. Whether TTS is actually + // usable is reported separately by `capabilities.voice`. + readAloud: true, // COMPAT(daemonDiagnostics): added in v0.1.100, remove gate after 2026-12-25 once daemon floor >= v0.1.100. daemonDiagnostics: true, // COMPAT(daemonSelfUpdate): added in v0.1.93, remove gate after 2026-12-13. diff --git a/public-docs/voice.md b/public-docs/voice.md index 79022c15ad..63686df875 100644 --- a/public-docs/voice.md +++ b/public-docs/voice.md @@ -111,6 +111,20 @@ Paseo uses these paths under the configured OpenAI base URL: - voice mode STT: `/v1/audio/transcriptions` - voice mode TTS: `/v1/audio/speech` +## Read Aloud + +Every completed agent turn gets a speaker button in its footer, next to the copy button. Pressing it speaks the turn's closing message — the part the agent wrote after its last tool call, not the narration in between. The icon becomes a stop square while it plays, and pressing it again stops. Starting a read on another turn supersedes the first: one voice at a time. + +Read aloud uses the same text-to-speech provider as voice mode (`features.voiceMode.tts`), so local Kokoro and OpenAI both work with no extra configuration. Audio is synthesized sentence by sentence and streamed to the client, so a long message starts speaking on the first sentence. Messages above 4000 characters are rejected rather than queued. Markdown and Paseo's own wrapper tags are stripped before synthesis, so fences and `` are not read out. + +Three limits: + +- **Web and desktop only.** Native has no audio playback engine for this yet, so the button is hidden on iOS and Android rather than shown doing nothing. +- **Turns that end on a tool call have no button.** There is nothing to say after the last tool. +- **A turn is spoken by the host it came from.** Read aloud never sends text to another paired host, so a route with no host — settings, history — gets no button. + +Hosts older than v0.2.5 do not have the read-aloud RPC; against those the button never appears. + ## Environment Variables - `PASEO_VOICE_LLM_PROVIDER`, voice agent provider override