diff --git a/app/.env.example b/app/.env.example index 44ae2fe725..c6b874e736 100644 --- a/app/.env.example +++ b/app/.env.example @@ -91,6 +91,16 @@ VITE_DEV_FORCE_ONBOARDING=false # Set to false to hide the toggle for a build (kill-switch). # VITE_VOICE_MODE=false +# [optional] Human tab voice entry point (#5399). Enabled by default: the chat +# card shows the realtime "Start voice chat" control in the slot the classic +# push-to-talk mic used to occupy. Set to false to fall back to tap-and-speak. +# VITE_HUMAN_VOICE_REALTIME=false + +# [optional] Show BOTH Human-tab voice controls (realtime + tap-and-speak), +# stacked. Off by default — for comparing the two paths, not for shipping. +# Takes precedence over VITE_HUMAN_VOICE_REALTIME. +# VITE_HUMAN_VOICE_SHOW_BOTH=true + # [optional] Client-side timeout for skill callTool/triggerSync (seconds; default 120, max 3600). # Should match OPENHUMAN_TOOL_TIMEOUT_SECS on the core when set. # VITE_TOOL_TIMEOUT_SECS= diff --git a/app/src/components/settings/panels/VoicePanel.tsx b/app/src/components/settings/panels/VoicePanel.tsx index 4c77c01a60..09650e8edd 100644 --- a/app/src/components/settings/panels/VoicePanel.tsx +++ b/app/src/components/settings/panels/VoicePanel.tsx @@ -17,9 +17,6 @@ import { type VoiceProviderView, type VoiceSettings, } from '../../../services/api/voiceSettingsApi'; -import { useAppDispatch, useAppSelector } from '../../../store/hooks'; -import { selectVoiceMode, setVoiceMode } from '../../../store/mascotSlice'; -import { VOICE_MODE_FLAG_ENABLED } from '../../../utils/config'; import { openhumanGetVoiceServerSettings, openhumanUpdateVoiceServerSettings, @@ -102,8 +99,6 @@ interface VoicePanelProps { const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => { const { t } = useT(); - const dispatch = useAppDispatch(); - const voiceMode = useAppSelector(selectVoiceMode); const { navigateBack, navigateToSettings } = useSettingsNavigation(); const [settings, setSettings] = useState(null); const [savedSettings, setSavedSettings] = useState(null); @@ -582,25 +577,8 @@ const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => { /> - {/* ─── Realtime voice mode (beta, flag-gated) ──────────────────── */} - {VOICE_MODE_FLAG_ENABLED && ( - - dispatch(setVoiceMode(next ? 'realtime' : 'classic'))} - aria-label={t('voice.mode.realtime')} - /> - } - /> - - )} + {/* Realtime voice is always on now — its controls live on the Human tab, + so the former flag-gated toggle here was removed. */} {/* ─── Section 1: Voice Provider Chips ─────────────────────────── */} {/* Provider chips are intentional bespoke UI — kept as-is. */} diff --git a/app/src/components/settings/panels/__tests__/VoicePanel.realtimeMode.test.tsx b/app/src/components/settings/panels/__tests__/VoicePanel.realtimeMode.test.tsx deleted file mode 100644 index 8d3febc3b9..0000000000 --- a/app/src/components/settings/panels/__tests__/VoicePanel.realtimeMode.test.tsx +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Unit test for VoicePanel's realtime voice-mode toggle (#5399). The section is - * gated behind the `VOICE_MODE_FLAG_ENABLED` build flag (global setup ships it - * OFF), so it is flipped ON here. Toggling the switch dispatches - * `setVoiceMode('realtime')` against the mascot slice. Mount-time voice APIs are - * stubbed so the panel renders without a backend. - */ -import { act, fireEvent, screen } from '@testing-library/react'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; - -import { renderWithProviders } from '../../../../test/test-utils'; -import VoicePanel from '../VoicePanel'; - -// Flip the realtime gate ON for this file; keep every other config export real. -vi.mock('../../../../utils/config', async () => { - const actual = await vi.importActual( - '../../../../utils/config' - ); - return { ...actual, VOICE_MODE_FLAG_ENABLED: true }; -}); - -vi.mock('../../../../utils/tauriCommands', () => ({ - openhumanGetVoiceServerSettings: vi.fn(async () => ({ result: {}, logs: [] })), - openhumanUpdateVoiceServerSettings: vi.fn(async () => ({ result: {}, logs: [] })), - openhumanVoiceSetProviders: vi.fn(async () => ({})), - openhumanVoiceStatus: vi.fn(async () => ({ stt_provider: 'cloud', tts_provider: 'cloud' })), - syncNotchVisibility: vi.fn(async () => undefined), -})); - -vi.mock('../../../../services/api/voiceInstallApi', () => ({ - installWhisper: vi.fn(), - installPiper: vi.fn(), - whisperInstallStatus: vi.fn(async () => ({ engine: 'whisper', state: 'missing' })), - piperInstallStatus: vi.fn(async () => ({ engine: 'piper', state: 'missing' })), -})); - -vi.mock('../../../../services/api/voiceSettingsApi', async () => { - const actual = await vi.importActual( - '../../../../services/api/voiceSettingsApi' - ); - return { - ...actual, - loadVoiceSettings: vi.fn(async () => ({ - voiceProviders: [], - sttProvider: { kind: 'cloud' }, - ttsProvider: { kind: 'cloud' }, - })), - saveVoiceSettings: vi.fn(async () => undefined), - setVoiceProviderKey: vi.fn(async () => undefined), - clearVoiceProviderKey: vi.fn(async () => undefined), - testVoiceProvider: vi.fn(async () => ({ ok: true, detail: 'OK' })), - }; -}); - -vi.mock('../../../../features/human/voice/ttsClient', async () => { - const actual = await vi.importActual( - '../../../../features/human/voice/ttsClient' - ); - return { ...actual, synthesizeSpeech: vi.fn() }; -}); - -describe('VoicePanel — realtime voice-mode toggle', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('dispatches setVoiceMode when the realtime switch is toggled on', async () => { - const { store } = renderWithProviders(, { initialEntries: ['/settings/voice'] }); - - const toggle = await screen.findByTestId('voice-mode-realtime-toggle'); - expect(store.getState().mascot.voiceMode).toBe('classic'); - - await act(async () => { - fireEvent.click(toggle); - }); - - expect(store.getState().mascot.voiceMode).toBe('realtime'); - }); -}); diff --git a/app/src/features/conversations/Conversations.tsx b/app/src/features/conversations/Conversations.tsx index c8a2859406..bdfe7f49f1 100644 --- a/app/src/features/conversations/Conversations.tsx +++ b/app/src/features/conversations/Conversations.tsx @@ -1,6 +1,6 @@ import { convertFileSrc } from '@tauri-apps/api/core'; import debugFactory from 'debug'; -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { type ReactNode, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useLocation, useNavigate, useParams } from 'react-router-dom'; import { type ChatSendError, chatSendError } from '../../chat/chatSendError'; @@ -150,6 +150,19 @@ interface ConversationsProps { * Used by the mascot tab so the only interaction is voice. */ composer?: 'text' | 'mic-cloud'; + /** + * Voice-chat control rendered in the `mic-cloud` composer slot, above the mic + * button. Passed in as a node rather than imported here so this component + * keeps no dependency on the realtime voice stack (and the ElevenLabs SDK + * stays out of every consumer's module graph). Ignored outside `mic-cloud`. + */ + voiceChatControl?: ReactNode; + /** + * Whether the `mic-cloud` slot renders the push-to-talk mic composer. Default + * `true` — set `false` alongside {@link ConversationsProps.voiceChatControl} + * to replace tap-and-speak with the realtime control rather than stack them. + */ + showMicComposer?: boolean; /** * Project the thread list into the root sidebar's dynamic region even in the * `sidebar` variant. Page variant always projects it; this lets an embedded @@ -248,6 +261,8 @@ export function deriveChatErrorBanner( const Conversations = ({ variant = 'page', composer: composerProp = 'text', + voiceChatControl = null, + showMicComposer = true, projectThreadList = false, }: ConversationsProps = {}) => { const [composerOverride, setComposerOverride] = useState<'mic-cloud' | 'text' | null>(null); @@ -2314,16 +2329,19 @@ const Conversations = ({ // — this branch renders no ChatComposer to hang it off.
{mascotDock} - handleSendMessage(text)} - onError={message => setSendError(chatSendError('voice_transcription', message))} - showDeviceSelector - onSwitchToText={() => setComposerOverride('text')} - /> + {voiceChatControl} + {showMicComposer && ( + handleSendMessage(text)} + onError={message => setSendError(chatSendError('voice_transcription', message))} + showDeviceSelector + onSwitchToText={() => setComposerOverride('text')} + /> + )}
) : inputMode === 'text' ? ( <> diff --git a/app/src/features/human/HumanPage.realtimeMode.test.tsx b/app/src/features/human/HumanPage.realtimeMode.test.tsx index 0bd42facf0..e411735c1d 100644 --- a/app/src/features/human/HumanPage.realtimeMode.test.tsx +++ b/app/src/features/human/HumanPage.realtimeMode.test.tsx @@ -1,9 +1,12 @@ /** - * Unit tests for HumanPage's realtime voice overlay gate (#5399). The overlay - * renders only when the build flag is on AND the persisted mascot voice mode is - * `realtime`; the classic push-to-talk path is always present. Config is mocked - * with the flag ON here (the global setup mock ships it OFF), and - * RealtimeVoiceControls is stubbed so the ElevenLabs SDK never loads. + * Unit test for the Human tab's voice entry point (#5399). The realtime + * "Start voice chat" control now lives in the chat card's composer slot — the + * one the classic push-to-talk mic used to own — and which of the two renders is + * decided by two build flags. This pins the wiring from those flags through to + * the props HumanPage hands Conversations; the controls themselves and the + * precedence rule are covered separately (RealtimeVoiceControls.test.tsx, + * voiceEntry.test.ts). RealtimeVoiceControls is stubbed so the ElevenLabs SDK + * never loads. */ import { configureStore } from '@reduxjs/toolkit'; import { render, screen } from '@testing-library/react'; @@ -11,25 +14,46 @@ import { Provider } from 'react-redux'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import chatRuntimeReducer from '../../store/chatRuntimeSlice'; -import mascotReducer, { setVoiceMode } from '../../store/mascotSlice'; +import mascotReducer from '../../store/mascotSlice'; import threadReducer from '../../store/threadSlice'; -import HumanPage from './HumanPage'; -// Flip the realtime gate ON for this file (global setup ships it OFF). Spread -// the real module so every other config export keeps its production value. -vi.mock('../../utils/config', async () => { - const actual = await vi.importActual('../../utils/config'); - return { ...actual, VOICE_MODE_FLAG_ENABLED: true }; +const flags = { realtimeEnabled: true, showBoth: false }; + +// The global test setup mocks the whole config module, so override just the two +// flags this file drives — read through getters so a test can flip them between +// renders without re-importing the module. +vi.mock('../../utils/config', async importOriginal => { + const actual = await importOriginal>(); + return { + ...actual, + get HUMAN_VOICE_REALTIME_ENABLED() { + return flags.realtimeEnabled; + }, + get HUMAN_VOICE_SHOW_BOTH() { + return flags.showBoth; + }, + }; }); -// Stub the overlay so the ElevenLabs `ConversationProvider`/SDK never mounts — -// this test only pins the render gate, not the controls (covered separately). vi.mock('./RealtimeVoiceControls', () => ({ default: () =>
, })); +// Render the slot props so the test observes what the card would actually show, +// rather than asserting on prop identity. vi.mock('../conversations/Conversations', () => ({ - default: () =>
, + default: ({ + voiceChatControl, + showMicComposer, + }: { + voiceChatControl?: React.ReactNode; + showMicComposer?: boolean; + }) => ( +
+ {voiceChatControl} + {showMicComposer &&
} +
+ ), })); vi.mock('./Mascot', async importOriginal => { @@ -46,11 +70,11 @@ vi.mock('./Mascot/manifest/useMascotManifest', () => ({ useMascotManifest: () => ({ manifest: null, entry: null, loading: false, error: null }), })); -function renderWithVoiceMode(mode: 'classic' | 'realtime') { +async function renderPage() { + const { default: HumanPage } = await import('./HumanPage'); const store = configureStore({ reducer: { mascot: mascotReducer, thread: threadReducer, chatRuntime: chatRuntimeReducer }, }); - store.dispatch(setVoiceMode(mode)); return render( @@ -58,18 +82,47 @@ function renderWithVoiceMode(mode: 'classic' | 'realtime') { ); } -describe('HumanPage — realtime voice overlay gate', () => { +describe('HumanPage — voice entry point', () => { beforeEach(() => { localStorage.clear(); + flags.realtimeEnabled = true; + flags.showBoth = false; }); - it('renders the realtime controls when voice mode is realtime and the flag is on', () => { - renderWithVoiceMode('realtime'); + it('shows the realtime control in place of the mic composer by default', async () => { + await renderPage(); expect(screen.getByTestId('realtime-voice-controls-stub')).toBeInTheDocument(); + expect(screen.queryByTestId('mic-composer-stub')).not.toBeInTheDocument(); }); - it('hides the realtime controls when voice mode is classic', () => { - renderWithVoiceMode('classic'); + it('falls back to tap-and-speak when the realtime flag is off', async () => { + flags.realtimeEnabled = false; + await renderPage(); + expect(screen.getByTestId('mic-composer-stub')).toBeInTheDocument(); expect(screen.queryByTestId('realtime-voice-controls-stub')).not.toBeInTheDocument(); }); + + // Comparison mode keeps the two paths apart: the realtime control floats over + // the mascot stage (outside the card), tap-and-speak stays in the card. + it('shows both controls when the show-both flag is on, and not stacked', async () => { + flags.showBoth = true; + await renderPage(); + expect(screen.getByTestId('realtime-voice-controls-stub')).toBeInTheDocument(); + expect(screen.getByTestId('mic-composer-stub')).toBeInTheDocument(); + // The card's slot stays empty — the realtime control is rendered outside it. + expect( + screen.getByTestId('conversations-stub').querySelector('[data-testid$="voice-controls-stub"]') + ).toBeNull(); + }); + + // Whichever mode is on, exactly one realtime control exists: the single-control + // modes put it in the card, comparison mode floats it — never both at once. + it.each([ + ['realtime', { realtimeEnabled: true, showBoth: false }], + ['both', { realtimeEnabled: true, showBoth: true }], + ])('renders the realtime control exactly once in %s mode', async (_label, next) => { + Object.assign(flags, next); + await renderPage(); + expect(screen.getAllByTestId('realtime-voice-controls-stub')).toHaveLength(1); + }); }); diff --git a/app/src/features/human/HumanPage.tsx b/app/src/features/human/HumanPage.tsx index 753cc570a0..08ea7642aa 100644 --- a/app/src/features/human/HumanPage.tsx +++ b/app/src/features/human/HumanPage.tsx @@ -8,10 +8,9 @@ import { selectCustomSecondaryColor, selectMascotColor, selectSpeakReplies, - selectVoiceMode, setSpeakReplies, } from '../../store/mascotSlice'; -import { VOICE_MODE_FLAG_ENABLED } from '../../utils/config'; +import { HUMAN_VOICE_REALTIME_ENABLED, HUMAN_VOICE_SHOW_BOTH } from '../../utils/config'; import Conversations from '../conversations/Conversations'; import { CustomGifMascot, @@ -23,6 +22,7 @@ import { import { useMascotManifest } from './Mascot/manifest/useMascotManifest'; import RealtimeVoiceControls from './RealtimeVoiceControls'; import { useHumanMascot } from './useHumanMascot'; +import { resolveHumanVoiceEntry } from './voiceEntry'; const HumanPage = () => { const { t } = useT(); @@ -35,8 +35,6 @@ const HumanPage = () => { const speakReplies = useAppSelector(selectSpeakReplies); const { face, visemeCode } = useHumanMascot({ speakReplies }); - const voiceMode = useAppSelector(selectVoiceMode); - const realtimeEnabled = VOICE_MODE_FLAG_ENABLED && voiceMode === 'realtime'; const mascotColor = useAppSelector(selectMascotColor); const customPrimary = useAppSelector(selectCustomPrimaryColor); const customSecondary = useAppSelector(selectCustomSecondaryColor); @@ -53,6 +51,17 @@ const HumanPage = () => { [mascotColor, customSecondary, palette] ); + // Which voice control the tab offers. Build-flag driven (#5399). In the + // single-control modes the realtime button takes the slot the push-to-talk mic + // used to own, so the tab has one voice affordance rather than two competing + // ones. `both` keeps them apart instead of stacking them — the realtime button + // floats over the mascot stage where it used to live, the card keeps + // tap-and-speak — so the two paths stay visually distinct while being compared. + const voiceEntry = resolveHumanVoiceEntry({ + realtimeEnabled: HUMAN_VOICE_REALTIME_ENABLED, + showBoth: HUMAN_VOICE_SHOW_BOTH, + }); + // The mascot drives a ~60fps lipsync re-render while the agent is speaking // (useHumanMascot forces a frame each rAF tick). Conversations is a heavy // subtree, so co-rendering it here would reconcile the whole chat tree every @@ -60,9 +69,19 @@ const HumanPage = () => { // locked during TTS playback (#5357). Its props are constant, so hold a stable // element: React short-circuits reconciliation of an unchanged child, keeping // the per-frame mascot re-render off the chat tree and the UI responsive. + // `voiceEntry` is build-time constant, so it cannot invalidate this memo at + // runtime — it is in the deps only to keep the dependency honest. const chatPanel = useMemo( - () => , - [] + () => ( + : null} + showMicComposer={voiceEntry !== 'realtime'} + projectThreadList + /> + ), + [voiceEntry] ); return ( @@ -101,10 +120,10 @@ const HumanPage = () => {
- {/* Realtime voice-chat controls (#5399) — additive overlay shown only when - the flag + realtime mode are on; the classic push-to-talk path below - is untouched. */} - {realtimeEnabled && ( + {/* Comparison mode only: the realtime control keeps its own place over the + mascot stage, so it reads as a separate path from the card's + tap-and-speak rather than a second button stacked on it. */} + {voiceEntry === 'both' && (
diff --git a/app/src/features/human/voice/readbackPrefix.contract.test.ts b/app/src/features/human/voice/readbackPrefix.contract.test.ts new file mode 100644 index 0000000000..80d1c5d873 --- /dev/null +++ b/app/src/features/human/voice/readbackPrefix.contract.test.ts @@ -0,0 +1,53 @@ +/** + * The read-back prefix is a behavioural contract that spans two languages: the + * renderer wraps a deferred answer in it, and the Rust harness recognises it to + * answer the turn from the prompt instead of rebuilding an orchestrator — and to + * stop speak-back re-arming into an unbounded loop. + * + * Both sides carry the string verbatim with a "MUST match" comment, but a comment + * is not a check: if one side is edited, `readback_payload` silently stops + * matching and the loop guard fails open, with every unit test on each side still + * passing (they assert against their own copy). This reads the Rust source and + * pins the two together, so a divergence fails here instead of in a live call. + */ +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +import { READBACK_PREFIX } from './useRealtimeVoiceSession'; + +const HARNESS_RELATIVE = 'src/openhuman/voice/realtime_harness.rs'; + +/** + * Walk up from the working directory to the repo root. `import.meta.url` is not a + * `file:` URL under vitest's transform, and the working directory differs between + * running from `app/` and from the repo root — so anchor on the file itself. + */ +function findHarness(): string { + let dir = process.cwd(); + for (;;) { + const candidate = resolve(dir, HARNESS_RELATIVE); + if (existsSync(candidate)) return candidate; + const parent = dirname(dir); + if (parent === dir) + throw new Error(`could not locate ${HARNESS_RELATIVE} above ${process.cwd()}`); + dir = parent; + } +} + +const HARNESS_PATH = findHarness(); + +describe('read-back prefix contract (TS ↔ Rust)', () => { + const harness = readFileSync(HARNESS_PATH, 'utf8'); + + it('finds the Rust constant where the contract says it lives', () => { + // Guards the test itself: a moved or renamed constant would otherwise make + // the assertion below vacuous rather than failing. + expect(harness).toContain('const VOICE_READBACK_PREFIX: &str ='); + }); + + it('matches the Rust VOICE_READBACK_PREFIX verbatim', () => { + const match = harness.match(/const VOICE_READBACK_PREFIX: &str =\s*"([^"]*)"/); + expect(match?.[1]).toBe(READBACK_PREFIX); + }); +}); diff --git a/app/src/features/human/voice/useRealtimeVoiceSession.test.ts b/app/src/features/human/voice/useRealtimeVoiceSession.test.ts index d6226e44e5..a5232300bb 100644 --- a/app/src/features/human/voice/useRealtimeVoiceSession.test.ts +++ b/app/src/features/human/voice/useRealtimeVoiceSession.test.ts @@ -12,11 +12,31 @@ interface CapturedProps { let captured: CapturedProps | null = null; const startSession = vi.fn(); const endSession = vi.fn(); +const sendUserMessage = vi.fn(); vi.mock('@elevenlabs/react', () => ({ useConversation: (props: CapturedProps) => { captured = props; - return { startSession, endSession, isSpeaking: false, mode: 'listening' as const }; + return { + startSession, + endSession, + sendUserMessage, + isSpeaking: false, + mode: 'listening' as const, + }; + }, +})); + +// Capture the `voice_speak` subscription so a test can drive the speak-back path. +const socketHandlers: Record void> = {}; +vi.mock('../../../services/socketService', () => ({ + socketService: { + on: vi.fn((event: string, handler: (payload: unknown) => void) => { + socketHandlers[event] = handler; + }), + off: vi.fn((event: string) => { + delete socketHandlers[event]; + }), }, })); @@ -29,6 +49,7 @@ describe('useRealtimeVoiceSession', () => { beforeEach(() => { vi.clearAllMocks(); captured = null; + Object.keys(socketHandlers).forEach(k => delete socketHandlers[k]); }); it('fetches a signed URL and opens a WebSocket session with the voice override', async () => { @@ -137,4 +158,68 @@ describe('useRealtimeVoiceSession', () => { unmount(); expect(endSession).not.toHaveBeenCalled(); }); + + it('reads a deferred result aloud when voice_speak arrives during a live call', async () => { + mockFetch.mockResolvedValueOnce({ signedUrl: 'wss://x', agentId: 'a1', userToken: 'tok-1' }); + const { result } = renderHook(() => useRealtimeVoiceSession()); + await act(async () => { + await result.current.start(); + }); + act(() => captured?.onConnect()); // liveRef becomes true + act(() => socketHandlers['voice_speak']?.({ full_response: 'Your inbox summary.' })); + expect(sendUserMessage).toHaveBeenCalledTimes(1); + // Wrapped in the verbatim read-back prefix so the agent reads it aloud. + expect(sendUserMessage.mock.calls[0][0]).toContain('Your inbox summary.'); + expect(sendUserMessage.mock.calls[0][0]).toContain('Please read the following'); + }); + + // Each read-back is a real agent turn, so a repeat queues behind the first and + // pushes the call towards the provider's per-turn ceiling. + it('reads a redelivered answer aloud only once per call', async () => { + mockFetch.mockResolvedValue({ signedUrl: 'wss://x', agentId: 'a1', userToken: 'tok-1' }); + const { result } = renderHook(() => useRealtimeVoiceSession()); + await act(async () => { + await result.current.start(); + }); + act(() => captured?.onConnect()); + act(() => socketHandlers['voice_speak']?.({ full_response: 'Your inbox summary.' })); + act(() => socketHandlers['voice_speak']?.({ full_response: 'Your inbox summary.' })); + act(() => socketHandlers['voice_speak']?.({ full_response: 'A different answer.' })); + expect(sendUserMessage).toHaveBeenCalledTimes(2); + + // A later call is a fresh conversation: the same answer may legitimately be + // asked for and spoken again. + act(() => captured?.onDisconnect()); + await act(async () => { + await result.current.start(); + }); + act(() => captured?.onConnect()); + act(() => socketHandlers['voice_speak']?.({ full_response: 'Your inbox summary.' })); + expect(sendUserMessage).toHaveBeenCalledTimes(3); + }); + + it('ignores voice_speak when no call is live', () => { + renderHook(() => useRealtimeVoiceSession()); // never connected → liveRef stays false + act(() => socketHandlers['voice_speak']?.({ full_response: 'ignored' })); + expect(sendUserMessage).not.toHaveBeenCalled(); + }); + + it('ignores an empty or missing voice_speak payload', async () => { + mockFetch.mockResolvedValueOnce({ signedUrl: 'wss://x', agentId: 'a1', userToken: 'tok-1' }); + const { result } = renderHook(() => useRealtimeVoiceSession()); + await act(async () => { + await result.current.start(); + }); + act(() => captured?.onConnect()); + act(() => socketHandlers['voice_speak']?.({ full_response: ' ' })); + act(() => socketHandlers['voice_speak']?.(undefined)); + expect(sendUserMessage).not.toHaveBeenCalled(); + }); + + it('unsubscribes from voice_speak on unmount', () => { + const { unmount } = renderHook(() => useRealtimeVoiceSession()); + expect(socketHandlers['voice_speak']).toBeDefined(); + unmount(); + expect(socketHandlers['voice_speak']).toBeUndefined(); + }); }); diff --git a/app/src/features/human/voice/useRealtimeVoiceSession.ts b/app/src/features/human/voice/useRealtimeVoiceSession.ts index 32b489191c..f6d355a721 100644 --- a/app/src/features/human/voice/useRealtimeVoiceSession.ts +++ b/app/src/features/human/voice/useRealtimeVoiceSession.ts @@ -3,10 +3,22 @@ import createDebug from 'debug'; import { useCallback, useEffect, useRef, useState } from 'react'; import { fetchVoiceAgentSignedUrl } from '../../../services/api/voiceAgentApi'; +import { socketService } from '../../../services/socketService'; import { MASCOT_VOICE_ID } from '../../../utils/config'; const log = createDebug('app:human:realtime-voice'); +/** + * Instruction prefix for "speak-back". A slow voice turn (e.g. an email summary) + * is acknowledged aloud, finishes in the background, and its result is delivered + * to chat AND pushed here as a `voice_speak` event. We send it back into the live + * ElevenLabs session as a user message wrapped with this prefix so the agent reads + * it verbatim. MUST match `VOICE_READBACK_PREFIX` in `voice/realtime_harness.rs`, + * which uses it to avoid re-arming speak-back on the read-back turn (loop guard). + */ +export const READBACK_PREFIX = + 'Please read the following to me, word for word, and say nothing else:'; + /** * Lifecycle of a realtime ElevenLabs Agents voice session (#5399). * `idle → connecting → active → idle`, or `→ error`. @@ -38,6 +50,10 @@ export function useRealtimeVoiceSession(opts?: { voiceId?: string }): RealtimeVo // Tracks whether a session is live so the unmount teardown only ends a real // session, and so the cleanup closure isn't tied to a stale `state`. const liveRef = useRef(false); + // Answers already read aloud in THIS call, so a redelivered result is not + // spoken twice. Scoped per call: asking the same question again in a later + // session should of course be answered again. + const spokenRef = useRef>(new Set()); const conversation = useConversation({ onConnect: () => { @@ -72,6 +88,7 @@ export function useRealtimeVoiceSession(opts?: { voiceId?: string }): RealtimeVo startingRef.current = true; setError(null); setState('connecting'); + spokenRef.current.clear(); log('start: requesting signed url'); try { const { signedUrl, userToken } = await fetchVoiceAgentSignedUrl(); @@ -125,6 +142,34 @@ export function useRealtimeVoiceSession(opts?: { voiceId?: string }): RealtimeVo [] ); + // Speak-back: a slow voice turn (email/calendar summary) is acknowledged aloud, + // finishes in the background, and the core emits its result as a `voice_speak` + // event. While the call is still open, read it aloud by sending it back into the + // live ElevenLabs session wrapped in the verbatim prefix (a fast read-back turn). + // The result also lands in chat regardless (delivered core-side) — this is the + // spoken copy. Refs keep the subscription set up once while always seeing the + // live conversation and liveness. + useEffect(() => { + const handler = (payload: unknown) => { + if (!liveRef.current) return; // call already ended — the chat copy stands alone + const text = (payload as { full_response?: string } | undefined)?.full_response?.trim(); + if (!text) return; + // Each read-back is a real turn the agent has to speak, so a repeat is not + // merely redundant: it queues behind the first and pushes the session + // towards the provider's per-turn ceiling. Redelivery of the same answer + // (a retried turn, a reconnect) must therefore be spoken once. + if (spokenRef.current.has(text)) { + log('speak-back: already read this answer aloud — skipping'); + return; + } + spokenRef.current.add(text); + log('speak-back: reading deferred result aloud (%d chars)', text.length); + conversationRef.current.sendUserMessage(`${READBACK_PREFIX}\n\n${text}`); + }; + socketService.on('voice_speak', handler); + return () => socketService.off('voice_speak', handler); + }, []); + return { state, isSpeaking: conversation.isSpeaking, diff --git a/app/src/features/human/voiceEntry.test.ts b/app/src/features/human/voiceEntry.test.ts new file mode 100644 index 0000000000..a07095177f --- /dev/null +++ b/app/src/features/human/voiceEntry.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest'; + +import { resolveHumanVoiceEntry } from './voiceEntry'; + +describe('resolveHumanVoiceEntry', () => { + it('defaults to the realtime control', () => { + expect(resolveHumanVoiceEntry({ realtimeEnabled: true, showBoth: false })).toBe('realtime'); + }); + + it('falls back to push-to-talk when realtime is switched off', () => { + expect(resolveHumanVoiceEntry({ realtimeEnabled: false, showBoth: false })).toBe( + 'push-to-talk' + ); + }); + + // show-both wins either way: a build that asks to compare the two paths must + // not have one of them hidden by the other flag's rollback state. + it('shows both whenever show-both is set, regardless of the realtime flag', () => { + expect(resolveHumanVoiceEntry({ realtimeEnabled: true, showBoth: true })).toBe('both'); + expect(resolveHumanVoiceEntry({ realtimeEnabled: false, showBoth: true })).toBe('both'); + }); +}); diff --git a/app/src/features/human/voiceEntry.ts b/app/src/features/human/voiceEntry.ts new file mode 100644 index 0000000000..7e2ddbea4f --- /dev/null +++ b/app/src/features/human/voiceEntry.ts @@ -0,0 +1,22 @@ +/** + * Which voice entry point the Human tab renders in its chat card (#5399). + * + * - `realtime` — the "Start voice chat" control (ElevenLabs Agents session). + * - `push-to-talk` — the classic tap-and-speak mic composer. + * - `both` — both, stacked, for comparing the two paths. + */ +export type HumanVoiceEntry = 'realtime' | 'push-to-talk' | 'both'; + +/** + * Resolve the entry point from the two build flags. Pure so the precedence rule + * is testable without a build: `showBoth` wins over `realtimeEnabled`, because a + * build that deliberately asks to see both should not have one of them hidden by + * the other flag's rollback state. + */ +export function resolveHumanVoiceEntry(flags: { + realtimeEnabled: boolean; + showBoth: boolean; +}): HumanVoiceEntry { + if (flags.showBoth) return 'both'; + return flags.realtimeEnabled ? 'realtime' : 'push-to-talk'; +} diff --git a/app/src/test/setup.ts b/app/src/test/setup.ts index 980e0ffab4..2fc7b7397d 100644 --- a/app/src/test/setup.ts +++ b/app/src/test/setup.ts @@ -226,6 +226,10 @@ vi.mock('../utils/config', () => ({ MASCOT_MANIFEST_URL: 'https://raw.githubusercontent.com/tinyhumansai/mascots/main/dist/mascots.json', VOICE_MODE_FLAG_ENABLED: false, + // Production defaults, so a test that does not care about the voice entry + // point sees what a shipped build sees. + HUMAN_VOICE_REALTIME_ENABLED: true, + HUMAN_VOICE_SHOW_BOTH: false, })); vi.mock('../services/backendUrl', () => ({ diff --git a/app/src/utils/config.ts b/app/src/utils/config.ts index 50575b94c1..1da0a7f16b 100644 --- a/app/src/utils/config.ts +++ b/app/src/utils/config.ts @@ -290,6 +290,32 @@ export const MASCOT_VOICE_MODEL_ID = export const VOICE_MODE_FLAG_ENABLED = (import.meta.env.VITE_VOICE_MODE as string | undefined)?.trim() !== 'false'; +/** + * Which voice entry point the Human tab offers (#5399). + * + * On by default in every build: the tab shows the realtime "Start voice chat" + * control where the push-to-talk mic used to sit. Set + * `VITE_HUMAN_VOICE_REALTIME=false` to fall back to the classic tap-and-speak + * composer — the kill switch for the realtime path on this surface. + * + * Distinct from {@link VOICE_MODE_FLAG_ENABLED}, which gates the *chat* tab's + * mascot stage against the persisted `mascot.voiceMode`. Keep them separate: + * one surface's rollback must not silently change the other's. + */ +export const HUMAN_VOICE_REALTIME_ENABLED = + (import.meta.env.VITE_HUMAN_VOICE_REALTIME as string | undefined)?.trim() !== 'false'; + +/** + * Show BOTH voice entry points on the Human tab — the realtime control and the + * classic push-to-talk composer, stacked. Off by default: the two are alternative + * ways to say the same thing, so shipping both at once is a comparison aid (A/B a + * regression, demo the difference), not the intended product surface. Set + * `VITE_HUMAN_VOICE_SHOW_BOTH=true` to enable. Takes precedence over + * {@link HUMAN_VOICE_REALTIME_ENABLED}. + */ +export const HUMAN_VOICE_SHOW_BOTH = + (import.meta.env.VITE_HUMAN_VOICE_SHOW_BOTH as string | undefined)?.trim() === 'true'; + /** * URL of the published mascot manifest (`dist/mascots.json` from the * `tinyhumansai/mascots` repo). This is the authoritative source for the diff --git a/src/openhuman/agent/harness/session/runtime.rs b/src/openhuman/agent/harness/session/runtime.rs index d499c7fd6f..419825517d 100644 --- a/src/openhuman/agent/harness/session/runtime.rs +++ b/src/openhuman/agent/harness/session/runtime.rs @@ -178,6 +178,16 @@ impl Agent { &self.model_name } + /// Override the base model this session runs its top-level turns on. Set + /// once before running: per-turn classification is disabled (the main agent + /// is pinned to its configured model for KV-cache stability — see the model + /// pin in `turn/core.rs`), so this sticks for the session and is not flipped + /// mid-conversation. The realtime voice harness uses it to pin a fast, + /// non-thinking model within the provider's response-time ceiling. + pub fn set_model_name(&mut self, model_name: impl Into) { + self.model_name = model_name.into(); + } + /// The agent's currently-configured temperature. pub fn temperature(&self) -> f64 { self.temperature diff --git a/src/openhuman/agent/harness/session/turn/core.rs b/src/openhuman/agent/harness/session/turn/core.rs index fb3cd7a96c..ebfe921832 100644 --- a/src/openhuman/agent/harness/session/turn/core.rs +++ b/src/openhuman/agent/harness/session/turn/core.rs @@ -923,6 +923,16 @@ impl Agent { let mut agent_context_prepared_sources: Vec = Vec::new(); + // Triggered memory-agent recall runs on EVERY channel, voice included: + // dropping it on voice would strip the user's remembered context + // (preferences, people, prior facts) from spoken answers — a real quality + // loss the transcript alone can't replace. Recall adds a few seconds of + // embedding + retrieval before the first model token, but on realtime + // voice that latency is already covered end-to-end: the backend relay + // streams an audible keepalive filler from t=0 so the cloud session never + // sees a silent stall, and the desktop's ~8s ack-defer closes the spoken + // turn and finishes in the background if the work runs long. So the recall + // path is byte-for-byte identical across voice and chat. let (enriched, memory_agent_context_injected) = self .inject_triggered_memory_agent_context(user_message, enriched, &parent_context) .await; diff --git a/src/openhuman/voice/realtime_harness.rs b/src/openhuman/voice/realtime_harness.rs index 3eea90e1b4..074acc96e5 100644 --- a/src/openhuman/voice/realtime_harness.rs +++ b/src/openhuman/voice/realtime_harness.rs @@ -13,6 +13,7 @@ //! audit-trail path rather than running with trusted-CLI semantics. use std::collections::HashSet; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, LazyLock, Mutex}; use std::time::Duration; @@ -21,11 +22,51 @@ use serde_json::{json, Value}; use tokio::sync::Semaphore; use crate::openhuman::agent::harness::session::Agent; +use crate::openhuman::agent::progress::AgentProgress; use crate::openhuman::agent::turn_origin::{with_origin, AgentTurnOrigin}; use crate::openhuman::platform::socket::manager::global_socket_manager; const TURN_TIMEOUT_SECS: u64 = 90; +/// How long a voice turn may run before we stop making the caller wait and hand +/// the result off to chat. The cloud voice session cancels a turn with no spoken +/// token in ~11-12s, and a slow tool action (email/calendar summary can be +/// 20-30s of Composio round-trips) will never fit that window. Once this elapses +/// we close the voice turn cleanly — the caller has heard the relay's spoken +/// filler and is told the answer is still coming (see VOICE_HANDOFF_LINE) — and +/// let the orchestrator finish in the background, +/// delivering its answer into the user's in-app chat and, while the call is +/// still up, reading it aloud. Sits under the provider's cancel deadline so +/// `done` always beats the cut. +const VOICE_ACK_DEADLINE_SECS: u64 = 8; + +/// Spoken when the ack deadline closes a turn that never produced text of its +/// own, so the caller is told the answer is still coming instead of being left on +/// a trail of filler. +/// +/// The preface the voice directive asks the model for cannot be relied on to +/// arrive inside the window: building the per-turn orchestrator (config load, +/// tool registry, integration catalogue) and running memory recall takes seconds +/// before the first model token is even requested, and the model's first round is +/// often a tool call carrying no text at all. Measured against staging, the first +/// streamable token landed ~10.6s into a turn whose whole budget is 8s — so every +/// slow turn ended on the relay's ellipsis padding and nothing else, which sounds +/// like the assistant losing the thread rather than working on an answer. +/// +/// Both delivery paths honour the promise: the answer is posted to chat and, +/// while the call is still up, read aloud. +const VOICE_HANDOFF_LINE: &str = "I'll have that for you in a moment. "; + +/// Sent for a turn we have nothing to say to. +/// +/// A turn that ends with no spoken content at all is not a valid answer to the +/// cloud session: it ends the whole call with `custom_llm_error: LLM Cascade +/// Error: Brain returned no response` (confirmed live — three recognition +/// artefacts answered with empty turns killed a working call). So even "nothing +/// to say" has to say something, and the least intrusive something is an ellipsis +/// pause, which the provider voices as a beat of silence rather than words. +const VOICE_SILENT_REPLY: &str = "… "; + /// Voice-scoped transcript namespace. Building a fresh orchestrator per turn /// would otherwise resume the *chat* orchestrator's latest transcript by name, /// bleeding an unrelated conversation into (or out of) the voice session. A @@ -33,6 +74,48 @@ const TURN_TIMEOUT_SECS: u64 = 90; /// relayed `messages` we seed below, not from this resume path. const VOICE_AGENT_NAME: &str = "voice"; +/// Model pinned for realtime voice turns. The cloud voice session cancels a turn +/// that has produced no spoken token in ~11-12s ("Generating the LLM response +/// took too long"), and the orchestrator's default reasoning model spends that +/// whole budget *thinking* before its first word. `chat-v1` (DeepSeek-V4-Flash, +/// thinking off) is a short-turn, tool-capable SKU: the master still routes +/// delegation through the prompt (per-turn classification is disabled — see the +/// model pin in `agent/harness/session/turn/core.rs`), so tool turns keep working +/// while spoken replies start in ~1s instead of ~6s. Reasoning models are the +/// wrong tool for a latency-capped realtime channel. +const VOICE_MODEL: &str = "chat-v1"; + +/// Instruction prefix used for "speak-back": when a deferred result is ready, the +/// renderer's live voice session sends it back as a user message wrapped with this +/// prefix so the agent reads it aloud verbatim. The core recognises the prefix to +/// avoid re-arming speak-back on the read-back turn itself (which would loop). MUST +/// match the string the renderer prepends (`useRealtimeVoiceSession.ts`). +const VOICE_READBACK_PREFIX: &str = + "Please read the following to me, word for word, and say nothing else:"; + +/// Chat thread + client id the voice turn scopes as its approval / routing +/// surface, mirroring `deliver_voice_result_to_chat`. Setting these around the +/// turn (via `APPROVAL_CHAT_CONTEXT` + `with_thread_id`) is what makes the voice +/// orchestrator behave like the chat path for tools that need a *routable* +/// approval surface: +/// +/// - `composio_connect` fails closed with a `[policy-denied] … needs an +/// interactive chat turn` message whenever `APPROVAL_CHAT_CONTEXT` is absent +/// (see `integrations/composio/tools.rs`). On voice that message got +/// paraphrased back to the user as "your Gmail connection is throwing an auth +/// error, reconnect it" — the exact voice-only Gmail-summary failure — even +/// though the same request works in tap-and-speak (a `WebChat` turn, which +/// installs this context). With the context set, the tool reaches its +/// already-connected short-circuit and returns success instead. +/// - external_effect tool approvals raised on the `ExternalChannel` turn now +/// have a thread card to route to (the same `proactive:voice` thread where +/// deferred voice answers land) rather than silently TTL-denying. +/// - `with_thread_id` gives async delegation (`spawn_async_subagent`) the +/// `parent_thread_id` it requires and aligns inference logs / KV-cache with +/// the voice thread. +const VOICE_CHAT_THREAD_ID: &str = "proactive:voice"; +const VOICE_CHAT_CLIENT_ID: &str = "system"; + /// Cap on concurrent local-agent turns driven by the relay. Each turn loads /// config, builds a full orchestrator, and runs for up to `TURN_TIMEOUT_SECS`, /// so an unbounded burst (or retry storm) would spawn unbounded heavy agent @@ -97,9 +180,22 @@ fn messages_to_history_pairs(messages: &[Value]) -> Vec<(String, String)> { /// Spoken-output directive appended to the orchestrator profile so replies read /// naturally through TTS instead of as markdown. +/// +/// The tool-preface clause is latency-critical, not cosmetic: the cloud realtime +/// session enforces a per-response time ceiling, and a turn that delegates (email, +/// calendar, files) produces no top-level assistant text until the sub-agent +/// returns 10-20s later — past the ceiling. Emitting one short spoken sentence +/// first makes the model stream an immediate `TextDelta`, so audio reaches the +/// caller right away and the turn stays alive while the tool runs. const VOICE_DIRECTIVE: &str = "You are speaking aloud in a live voice conversation. \ Reply in natural, concise spoken sentences. Do not use markdown, code blocks, \ -bullet lists, headings, or emoji."; +bullet lists, headings, or emoji. Before you use a tool or delegate (for example \ +to check email, a calendar, files, or the web), first say one short spoken \ +sentence telling the user what you are doing — and, since those actions sometimes \ +take a while, that you'll follow up with the details in their chat IF it does — \ +then proceed. Example: \"Sure, let me pull up your inbox — if it takes a moment \ +I'll drop the summary in your chat.\" Keep that preface to one sentence so it \ +starts speaking immediately."; /// Extract the user prompt from an OpenAI-style `messages` array: the content of /// the last `user` message. Content may be a plain string or an array of @@ -125,9 +221,13 @@ fn content_to_text(content: Option<&Value>) -> String { } } -/// Handle one relayed voice turn end to end: run the orchestrator and emit the -/// reply back up the socket. Never panics — every failure path emits -/// `voice:harness:error` so the backend relay ends the turn cleanly. +/// Handle one relayed voice turn end to end. The reply streams token-by-token +/// back up the socket as it is produced. A turn that finishes inside the voice +/// window is spoken in full; a slow tool action (email/calendar summary — tens of +/// seconds of Composio round-trips) is acknowledged aloud, then finishes in the +/// background and delivers its answer into the user's in-app chat. Never panics — +/// every failure path emits `voice:harness:error` (or a clean `done`) so the +/// backend relay ends the turn cleanly. pub async fn handle_voice_harness_turn(correlation_id: String, messages: Vec) { let prompt = extract_prompt(&messages); if prompt.trim().is_empty() { @@ -135,23 +235,190 @@ pub async fn handle_voice_harness_turn(correlation_id: String, messages: Vec { - let spoken = reply.trim(); - if !spoken.is_empty() { + // Stream the reply token-by-token: the orchestrator emits `TextDelta` as it + // generates, and `forward_reply_deltas` relays each as a `voice:harness:delta` + // so the reply leaves the desktop as it is produced. + let streamed = Arc::new(AtomicBool::new(false)); + let (progress_tx, progress_rx) = tokio::sync::mpsc::channel::(256); + let forwarder = tokio::spawn(forward_reply_deltas( + progress_rx, + correlation_id.clone(), + streamed.clone(), + )); + + // Run the turn on a detached task so it can outlive the spoken ack. A slow + // delegation keeps working after we close the voice turn and delivers its + // result into the user's chat. The task owns the agent, the concurrency + // permit, and the in-flight guard so that bookkeeping tracks the real turn. + let (result_tx, result_rx) = tokio::sync::oneshot::channel::>(); + // Captured before the prompt moves into the task, so the foreground can still + // tell whether the user is waiting on an answer if that task dies. + let answerable = is_answerable_prompt(&prompt); + let turn_cid = correlation_id.clone(); + let turn_messages = messages; + let turn_prompt = prompt; + tokio::spawn(async move { + let _in_flight = in_flight; + // Bound concurrent heavy agent turns; excess turns queue HERE, inside the + // detached task, never in front of the ack deadline below. Acquiring in the + // foreground meant a turn queued behind slower ones started no clock and + // emitted no `done` at all, and the provider ended the whole session over + // it. Held for the REAL turn lifetime (including any background tail), so + // it still caps a retry storm. + let _permit = match VOICE_TURN_LIMITER.clone().acquire_owned().await { + Ok(permit) => permit, + Err(_) => { + warn!("[voice-harness] turn limiter unavailable correlation={turn_cid}"); + if is_answerable_prompt(&turn_prompt) { + deliver_voice_failure_to_chat(&turn_cid); + } + return; + } + }; + let outcome = run_voice_turn(&turn_cid, &turn_messages, &turn_prompt, progress_tx).await; + // Hand the result to the foreground. If it already deferred (dropped the + // receiver at the ack deadline), the send fails and we deliver the reply + // into the user's chat instead. + if let Err(unsent) = result_tx.send(outcome) { + match unsent { + Ok(reply) => { + // A read-back turn only re-reads an answer already delivered to + // chat; delivering it again would duplicate the chat message, and + // re-arming speak-back would loop. So deliver + arm speak-back ONLY + // for a genuine deferred answer, and skip the whole delivery for a + // read-back echo turn. + if should_arm_speak_back(&turn_prompt) { + deliver_voice_result_to_chat(&turn_cid, reply, true); + } else { + info!("[voice-harness] deferred read-back turn carries no new content; skipping chat delivery correlation={turn_cid}"); + } + } + Err(err) => { + // The spoken turn already closed with `done` and the preface may + // have promised a chat follow-up, so a silent failure would leave + // the user waiting for a message that never arrives. Post a brief + // failure notice to the same thread — but only for a turn whose + // answer the user is actually waiting on. A read-back or a + // recognition artefact has nothing to deliver, and a notice for + // one reads as the assistant failing a request nobody made. + warn!("[voice-harness] deferred turn failed correlation={turn_cid}: {err}"); + if is_answerable_prompt(&turn_prompt) { + deliver_voice_failure_to_chat(&turn_cid); + } + } + } + } + }); + + // Race the turn against the spoken-ack deadline. + match tokio::time::timeout(Duration::from_secs(VOICE_ACK_DEADLINE_SECS), result_rx).await { + Ok(Ok(outcome)) => { + // Finished inside the voice window — deltas already streamed. Join the + // forwarder so every delta is out before `done`, and learn whether + // anything streamed (fallback for a non-streaming reply). + let streamed_any = forwarder.await.unwrap_or(false); + match outcome { + Ok(reply) => { + if !streamed_any { + // A turn that produced no text still has to say something, or + // the provider ends the call (see VOICE_SILENT_REPLY). + let spoken = reply.trim(); + let spoken = if spoken.is_empty() { + VOICE_SILENT_REPLY + } else { + spoken + }; + emit_event( + "voice:harness:delta", + json!({ "correlationId": correlation_id, "text": spoken }), + ) + .await; + } + emit_event( + "voice:harness:done", + json!({ "correlationId": correlation_id }), + ) + .await; + } + Err(err) => { + warn!("[voice-harness] turn failed correlation={correlation_id}: {err}"); + emit_error(&correlation_id, &err).await; + } + } + } + Ok(Err(_recv)) => { + // Sender dropped without a value (task aborted or panicked). A silent + // turn with no spoken text and no chat delivery is hard to trace, so + // log with the correlation id before ending cleanly. + warn!( + "[voice-harness] turn task ended without a result (panicked or aborted) correlation={correlation_id}" + ); + // Unlike the ack-deadline path, nothing else will deliver here: the task + // died before reaching its own chat-delivery branch. The turn may already + // have promised a follow-up in chat, so post the notice from this side — + // for a turn the user is actually waiting on (see is_answerable_prompt). + if answerable { + deliver_voice_failure_to_chat(&correlation_id); + } + // Closing on nothing at all would end the whole call (see + // VOICE_SILENT_REPLY), so a lost turn still ends with a spoken beat. + if !streamed.load(Ordering::SeqCst) { emit_event( "voice:harness:delta", - json!({ "correlationId": correlation_id, "text": spoken }), + json!({ "correlationId": correlation_id, "text": VOICE_SILENT_REPLY }), ) .await; } @@ -161,18 +428,146 @@ pub async fn handle_voice_harness_turn(correlation_id: String, messages: Vec { - warn!("[voice-harness] turn failed correlation={correlation_id}: {err}"); - emit_error(&correlation_id, &err).await; + Err(_deadline) => { + // Still running (a slow tool action). Close the voice turn cleanly; the + // detached task keeps going and delivers its answer into the user's + // chat. `timeout` consumed `result_rx`, so the task's send fails and + // takes the chat-delivery path. The forwarder is left running to keep + // draining progress — any late deltas reach a settled relay turn and + // are dropped harmlessly. + info!("[voice-harness] ack deadline reached, handing off to chat correlation={correlation_id}"); + // If the turn never said anything of its own, the caller has heard only + // the relay's filler. Ending there sounds like the assistant lost the + // thread, so say the answer is still coming — it is, on both delivery + // paths (chat, and read aloud while the call is up). + if !streamed.load(Ordering::SeqCst) { + emit_event( + "voice:harness:delta", + json!({ "correlationId": correlation_id, "text": VOICE_HANDOFF_LINE }), + ) + .await; + } + emit_event( + "voice:harness:done", + json!({ "correlationId": correlation_id }), + ) + .await; } } } -async fn run_agent_turn( +/// Forward the orchestrator's streamed assistant text to the relay socket, one +/// `voice:harness:delta` per top-level `AgentProgress::TextDelta`, until the +/// turn's progress channel closes (the agent drops its sender when the turn +/// ends). Returns whether any non-empty delta was streamed, so the caller can +/// fall back to emitting the whole reply for a turn that produced text off the +/// streaming path. Only top-level assistant text is voiced — sub-agent narration, +/// thinking, tool-call args, and lifecycle events are deliberately not spoken. +async fn forward_reply_deltas( + mut progress_rx: tokio::sync::mpsc::Receiver, + correlation_id: String, + streamed: Arc, +) -> bool { + let mut streamed_any = false; + while let Some(progress) = progress_rx.recv().await { + let Some(text) = spoken_delta(&progress) else { + continue; + }; + // Skip only truly empty deltas — whitespace carries word boundaries and + // must be forwarded so the concatenated speech isn't run together. + if text.is_empty() { + continue; + } + streamed_any = true; + // Published for the ack + handoff decisions, which need to know whether the + // orchestrator is talking *while* the turn is still open. + streamed.store(true, Ordering::SeqCst); + emit_event( + "voice:harness:delta", + json!({ "correlationId": correlation_id, "text": text }), + ) + .await; + } + streamed_any +} + +/// The spoken text carried by a progress event, or `None` for events that must +/// not be voiced. Only the top-level assistant `TextDelta` is spoken; sub-agent +/// deltas, thinking, tool-call args, and lifecycle events are internal. Pure + +/// unit-tested. +fn spoken_delta(progress: &AgentProgress) -> Option<&str> { + match progress { + AgentProgress::TextDelta { delta, .. } => Some(delta), + _ => None, + } +} + +/// The answer a read-back turn is asking to have spoken, or `None` for an +/// ordinary turn. Leading whitespace is tolerated because the renderer joins the +/// prefix and payload with a blank line. Pure + unit-tested. +fn readback_payload(prompt: &str) -> Option<&str> { + let trimmed = prompt.trim_start(); + trimmed.strip_prefix(VOICE_READBACK_PREFIX).map(str::trim) +} + +/// Whether a prompt carries nothing to answer. Speech recognition emits `"..."` +/// (and similar punctuation-only artefacts) for a pause, and the provider relays +/// those as real turns. Anything with a letter or a digit in it — in any script — +/// is a genuine prompt. Pure + unit-tested. +fn is_content_free(prompt: &str) -> bool { + !prompt.chars().any(char::is_alphanumeric) +} + +/// Whether the user is waiting on this turn's answer, and so should be told when +/// it fails. False for a read-back (its answer is already in chat) and for a +/// recognition artefact (nothing was asked). Pure + unit-tested. +fn is_answerable_prompt(prompt: &str) -> bool { + !is_content_free(prompt) && should_arm_speak_back(prompt) +} + +/// Whether a completed voice turn should arm speak-back — i.e. push its deferred +/// answer back into the live session to be read aloud. A read-back turn is itself +/// a verbatim-read request (its prompt is wrapped with [`VOICE_READBACK_PREFIX`] +/// by the renderer), so re-arming speak-back on it would deliver the spoken copy +/// to a turn that then asks to read it again — an unbounded loop. Suppress those. +/// Pure + unit-tested; leading whitespace is tolerated because the renderer joins +/// the prefix and payload with a blank line. +fn should_arm_speak_back(prompt: &str) -> bool { + !prompt.trim_start().starts_with(VOICE_READBACK_PREFIX) +} + +/// Build the fresh voice orchestrator, attach the streaming sink, run one turn +/// under the hard per-turn ceiling, then detach the sink so the forwarder's +/// channel closes. Runs entirely on the background task, so the ack deadline in +/// the caller covers both the build and the model round-trips. +async fn run_voice_turn( correlation_id: &str, messages: &[Value], prompt: &str, + progress_tx: tokio::sync::mpsc::Sender, ) -> Result { + let mut agent = build_voice_agent(correlation_id, messages, prompt).await?; + + // Attach the streaming sink before the turn: its presence switches the harness + // onto the true per-token streaming path, and each `AgentProgress::TextDelta` + // is forwarded to the relay socket by `forward_reply_deltas`. + agent.set_on_progress(Some(progress_tx)); + + let outcome = run_single_with_timeout(&mut agent, correlation_id, prompt).await; + + // Detach the sink so the forwarder's channel closes the moment the turn ends, + // deterministically rather than waiting on `agent`'s drop. + agent.set_on_progress(None); + outcome +} + +/// Construct the per-turn voice orchestrator: load config, pin the fast voice +/// model, isolate the transcript namespace, and seed the relayed history. +async fn build_voice_agent( + correlation_id: &str, + messages: &[Value], + prompt: &str, +) -> Result { let config = crate::openhuman::config::ops::load_config_with_timeout().await?; let mut agent = Agent::from_config_for_agent_with_profile( &config, @@ -186,6 +581,9 @@ async fn run_agent_turn( // Isolate the voice transcript namespace from the chat orchestrator so a // fresh-per-turn agent can't resume an unrelated conversation by name. agent.set_agent_definition_name(VOICE_AGENT_NAME); + // Pin a fast, non-thinking model so the first spoken token lands inside the + // realtime session's response-time ceiling (see VOICE_MODEL). + agent.set_model_name(VOICE_MODEL); // Seed the authoritative prior turns the relay carries (OpenAI `messages`), // so follow-ups like "what about tomorrow?" keep their context. No-ops when @@ -200,7 +598,31 @@ async fn run_agent_turn( prompt.chars().count(), messages.len() ); + Ok(agent) +} +/// Run the orchestrator turn under the hard per-turn ceiling. The streaming sink +/// must already be attached; deltas flow out while this runs. +async fn run_single_with_timeout( + agent: &mut Agent, + correlation_id: &str, + prompt: &str, +) -> Result { + // Scope the turn with the SAME chat context the web-chat path installs + // (`APPROVAL_CHAT_CONTEXT` + `with_thread_id`), so approval-surfaced tools + // behave identically on voice. Without it `composio_connect` fails closed + // for lack of a routable surface, which the model paraphrases to the user as + // a confabulated "reconnect your Gmail" mid email-summary (#5399). See + // VOICE_CHAT_THREAD_ID for the full rationale. Nesting mirrors web chat: + // origin (outer) → approval context → thread id → the agent run. + let approval_ctx = crate::openhuman::security::approval::ApprovalChatContext { + thread_id: VOICE_CHAT_THREAD_ID.to_string(), + client_id: VOICE_CHAT_CLIENT_ID.to_string(), + }; + let scoped_run = crate::openhuman::agent::tinyagents::thread_context::with_thread_id( + VOICE_CHAT_THREAD_ID, + agent.run_single(prompt), + ); let fut = with_origin( AgentTurnOrigin::ExternalChannel { channel: "voice".to_string(), @@ -208,7 +630,7 @@ async fn run_agent_turn( reply_target: correlation_id.to_string(), message_id: format!("voice-{correlation_id}"), }, - agent.run_single(prompt), + crate::openhuman::security::approval::APPROVAL_CHAT_CONTEXT.scope(approval_ctx, scoped_run), ); match tokio::time::timeout(Duration::from_secs(TURN_TIMEOUT_SECS), fut).await { @@ -220,6 +642,77 @@ async fn run_agent_turn( } } +/// Deliver a deferred voice turn's answer into the user's in-app chat. Publishes +/// a `proactive_message` on the web-channel event bus — the same seam cron and the +/// subconscious use — which the frontend renders as an assistant message in a +/// visible thread. Web-only: it does not fan out to external channels (#5399). +fn deliver_voice_result_to_chat(correlation_id: &str, reply: String, allow_speak_back: bool) { + let spoken = reply.trim(); + if spoken.is_empty() { + warn!("[voice-harness] deferred turn produced no text correlation={correlation_id}"); + // The spoken ack already promised a chat follow-up, so an empty deferred + // reply must still surface a message rather than leave the user waiting. + deliver_voice_failure_to_chat(correlation_id); + return; + } + info!( + "[voice-harness] delivering deferred result to chat correlation={correlation_id} chars={} speak_back={allow_speak_back}", + spoken.chars().count() + ); + crate::openhuman::web_chat::publish_web_channel_event(crate::core::socketio::WebChannelEvent { + event: "proactive_message".to_string(), + client_id: VOICE_CHAT_CLIENT_ID.to_string(), + thread_id: VOICE_CHAT_THREAD_ID.to_string(), + full_response: Some(spoken.to_string()), + success: Some(true), + ..Default::default() + }); + + // Speak-back: push the finished answer to the renderer's LIVE voice session so + // the agent can read it aloud. The frontend voice hook listens for `voice_speak` + // and, only while the call is still open, sends it back into the ElevenLabs + // session (a fast read-back turn). Skipped for read-back turns themselves to + // avoid a loop; harmless if the call already ended (nobody is subscribed). + if allow_speak_back { + crate::openhuman::web_chat::publish_web_channel_event( + crate::core::socketio::WebChannelEvent { + event: "voice_speak".to_string(), + client_id: VOICE_CHAT_CLIENT_ID.to_string(), + full_response: Some(spoken.to_string()), + success: Some(true), + ..Default::default() + }, + ); + } +} + +/// Deliver a short "couldn't complete" notice to the voice chat thread for a +/// deferred turn that produced no answer the user can see — either it errored, +/// or it completed past the ack deadline with empty text (on this path +/// `run_single`'s returned text is the sole answer channel: the orchestrator +/// folds any tool/subagent output into its final reply, so an empty reply means +/// nothing was produced for the user, not that the answer went elsewhere). +/// Because the spoken preface may have told the user their answer would land in +/// chat, staying silent would leave them waiting on a message that never comes — +/// this makes the promised message always appear. Delivered as a normal +/// assistant message (not spoken) on the same `proactive:voice` surface as a +/// successful deferred answer. +fn deliver_voice_failure_to_chat(correlation_id: &str) { + info!( + "[voice-harness] delivering deferred failure notice to chat correlation={correlation_id}" + ); + crate::openhuman::web_chat::publish_web_channel_event(crate::core::socketio::WebChannelEvent { + event: "proactive_message".to_string(), + client_id: VOICE_CHAT_CLIENT_ID.to_string(), + thread_id: VOICE_CHAT_THREAD_ID.to_string(), + full_response: Some( + "Sorry — I couldn't finish that request just now. Please try again.".to_string(), + ), + success: Some(false), + ..Default::default() + }); +} + async fn emit_event(event: &str, payload: Value) { match global_socket_manager() { Some(mgr) => { @@ -302,4 +795,128 @@ mod tests { let pairs = messages_to_history_pairs(&messages); assert_eq!(pairs, vec![("user".to_string(), "hello".to_string())]); } + + #[test] + fn spoken_delta_forwards_only_top_level_assistant_text() { + assert_eq!( + spoken_delta(&AgentProgress::TextDelta { + delta: "hey there".to_string(), + iteration: 1, + }), + Some("hey there") + ); + // Whitespace-only deltas carry word boundaries and are still spoken text — + // the empty-skip lives in the forwarder, not here. + assert_eq!( + spoken_delta(&AgentProgress::TextDelta { + delta: " ".to_string(), + iteration: 2, + }), + Some(" ") + ); + } + + #[test] + fn spoken_delta_suppresses_internal_events() { + // Reasoning must never be voiced. + assert_eq!( + spoken_delta(&AgentProgress::ThinkingDelta { + delta: "let me think".to_string(), + iteration: 1, + }), + None + ); + // A delegated sub-agent's narration is internal, not the spoken answer. + assert_eq!( + spoken_delta(&AgentProgress::SubagentTextDelta { + agent_id: "a".to_string(), + task_id: "t".to_string(), + delta: "fetching inbox".to_string(), + iteration: 1, + }), + None + ); + // Lifecycle events carry no spoken text. + assert_eq!( + spoken_delta(&AgentProgress::TurnCompleted { iterations: 1 }), + None + ); + } + + #[test] + fn speak_back_armed_for_a_genuine_answer_turn() { + assert!(should_arm_speak_back("summarize my unread emails")); + assert!(should_arm_speak_back("what's on my calendar tomorrow?")); + } + + #[test] + fn speak_back_suppressed_for_a_read_back_turn() { + // The bare prefix, and the real renderer shape (prefix + blank line + + // payload, possibly with leading whitespace) must both be recognised so + // the spoken copy never re-arms into an unbounded loop. + assert!(!should_arm_speak_back(VOICE_READBACK_PREFIX)); + assert!(!should_arm_speak_back(&format!( + "{VOICE_READBACK_PREFIX}\n\nHere is your inbox summary." + ))); + assert!(!should_arm_speak_back(&format!( + " \n{VOICE_READBACK_PREFIX} trailing payload" + ))); + } + + #[test] + fn read_back_payload_is_the_text_to_speak() { + assert_eq!( + readback_payload(&format!( + "{VOICE_READBACK_PREFIX}\n\nHere is your inbox summary." + )), + Some("Here is your inbox summary.") + ); + // The renderer may prepend whitespace; the payload must survive it intact. + assert_eq!( + readback_payload(&format!( + " \n{VOICE_READBACK_PREFIX} two things need attention" + )), + Some("two things need attention") + ); + // A prefix with nothing behind it is still a read-back — it just has + // nothing to say, and must not be relayed as a question. + assert_eq!(readback_payload(VOICE_READBACK_PREFIX), Some("")); + } + + #[test] + fn ordinary_prompts_are_not_read_backs() { + assert_eq!(readback_payload("summarize my emails"), None); + assert_eq!(readback_payload("please read my emails to me"), None); + } + + #[test] + fn recognition_artefacts_are_content_free() { + // What the provider actually relayed for a pause during a filler-heavy + // turn, plus the shapes next to it. + assert!(is_content_free("...")); + assert!(is_content_free("…")); + assert!(is_content_free(" ? ")); + assert!(is_content_free("-")); + } + + #[test] + fn real_questions_are_not_content_free() { + assert!(!is_content_free("summarize my emails")); + // A single digit is a real answer to "how many?" — and scripts other than + // Latin must never be mistaken for punctuation. + assert!(!is_content_free("3")); + assert!(!is_content_free("मेरे ईमेल पढ़ो")); + assert!(!is_content_free("总结我的邮件")); + } + + #[test] + fn failure_notice_is_limited_to_turns_the_user_is_waiting_on() { + assert!(is_answerable_prompt("summarize my emails")); + // A read-back's answer is already in chat; a notice would report a failure + // for a request the user never made. + assert!(!is_answerable_prompt(&format!( + "{VOICE_READBACK_PREFIX}\n\nHere is your inbox summary." + ))); + assert!(!is_answerable_prompt("...")); + } }