From 2748458ad17fe46e9498f9c611722c539aa345a9 Mon Sep 17 00:00:00 2001 From: Shanu Date: Mon, 10 Aug 2026 22:23:21 +0530 Subject: [PATCH 01/18] feat(voice): realtime ElevenLabs voice agent alongside the classic path Add the ElevenLabs Voice Agents realtime path as an always-on "Start Voice Chat" control on the Human tab, driven by the local orchestrator over the voice:harness relay. The classic push-to-talk path is kept unchanged so both run side by side in staging. - Pin a fast, non-thinking voice model per turn (set_model_name) so the first spoken token lands inside the provider's response-time ceiling. - Ack-and-defer slow turns (email/calendar): speak a short ack, finish in the background, deliver the answer to chat and read it back aloud. - Scope each voice turn with the same approval-chat context + thread as chat, so composio_connect reaches its already-connected path instead of surfacing a false "reconnect your Gmail" auth error. - Keep triggered memory recall on voice so spoken answers use the user's remembered context; the added latency is covered by the relay's audible keepalive and the background-defer. - Stream reply tokens live; guard speak-back from re-arming on a read-back turn (should_arm_speak_back, unit-tested). - Make the realtime control always visible on the Human tab and remove the now-unused voice-mode settings toggle. Addresses #5399. Old-path removal (AC7) is intentionally deferred: old and new run side by side in staging for comparison. --- .../components/settings/panels/VoicePanel.tsx | 26 +- app/src/features/human/HumanPage.tsx | 6 +- .../human/voice/useRealtimeVoiceSession.ts | 30 ++ .../agent/harness/session/runtime.rs | 10 + .../agent/harness/session/turn/core.rs | 10 + src/openhuman/voice/realtime_harness.rs | 410 +++++++++++++++++- 6 files changed, 440 insertions(+), 52 deletions(-) diff --git a/app/src/components/settings/panels/VoicePanel.tsx b/app/src/components/settings/panels/VoicePanel.tsx index 4a77989bd1..963e418570 100644 --- a/app/src/components/settings/panels/VoicePanel.tsx +++ b/app/src/components/settings/panels/VoicePanel.tsx @@ -17,9 +17,6 @@ import { type VoiceProviderView, type VoiceSettings, } from '../../../services/api/voiceSettingsApi'; -import { useAppDispatch, useAppSelector } from '../../../store/hooks'; -import { selectVoiceMode, setVoiceMode } from '../../../store/mascotSlice'; -import { VOICE_MODE_FLAG_ENABLED } from '../../../utils/config'; import { openhumanGetVoiceServerSettings, openhumanUpdateVoiceServerSettings, @@ -102,8 +99,6 @@ interface VoicePanelProps { const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => { const { t } = useT(); - const dispatch = useAppDispatch(); - const voiceMode = useAppSelector(selectVoiceMode); const { navigateBack, navigateToSettings } = useSettingsNavigation(); const [settings, setSettings] = useState(null); const [savedSettings, setSavedSettings] = useState(null); @@ -582,25 +577,8 @@ const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => { /> - {/* ─── Realtime voice mode (beta, flag-gated) ──────────────────── */} - {VOICE_MODE_FLAG_ENABLED && ( - - dispatch(setVoiceMode(next ? 'realtime' : 'classic'))} - aria-label={t('voice.mode.realtime')} - /> - } - /> - - )} + {/* Realtime voice is always on now — its controls live on the Human tab, + so the former flag-gated toggle here was removed. */} {/* ─── Section 1: Voice Provider Chips ─────────────────────────── */} {/* Provider chips are intentional bespoke UI — kept as-is. */} diff --git a/app/src/features/human/HumanPage.tsx b/app/src/features/human/HumanPage.tsx index 753cc570a0..0e2d4099eb 100644 --- a/app/src/features/human/HumanPage.tsx +++ b/app/src/features/human/HumanPage.tsx @@ -8,10 +8,8 @@ import { selectCustomSecondaryColor, selectMascotColor, selectSpeakReplies, - selectVoiceMode, setSpeakReplies, } from '../../store/mascotSlice'; -import { VOICE_MODE_FLAG_ENABLED } from '../../utils/config'; import Conversations from '../conversations/Conversations'; import { CustomGifMascot, @@ -35,8 +33,8 @@ const HumanPage = () => { const speakReplies = useAppSelector(selectSpeakReplies); const { face, visemeCode } = useHumanMascot({ speakReplies }); - const voiceMode = useAppSelector(selectVoiceMode); - const realtimeEnabled = VOICE_MODE_FLAG_ENABLED && voiceMode === 'realtime'; + // Realtime voice controls are always shown — no settings/flag gate. + const realtimeEnabled = true; const mascotColor = useAppSelector(selectMascotColor); const customPrimary = useAppSelector(selectCustomPrimaryColor); const customSecondary = useAppSelector(selectCustomSecondaryColor); diff --git a/app/src/features/human/voice/useRealtimeVoiceSession.ts b/app/src/features/human/voice/useRealtimeVoiceSession.ts index 32b489191c..d6a62acf79 100644 --- a/app/src/features/human/voice/useRealtimeVoiceSession.ts +++ b/app/src/features/human/voice/useRealtimeVoiceSession.ts @@ -3,10 +3,21 @@ import createDebug from 'debug'; import { useCallback, useEffect, useRef, useState } from 'react'; import { fetchVoiceAgentSignedUrl } from '../../../services/api/voiceAgentApi'; +import { socketService } from '../../../services/socketService'; import { MASCOT_VOICE_ID } from '../../../utils/config'; const log = createDebug('app:human:realtime-voice'); +/** + * Instruction prefix for "speak-back". A slow voice turn (e.g. an email summary) + * is acknowledged aloud, finishes in the background, and its result is delivered + * to chat AND pushed here as a `voice_speak` event. We send it back into the live + * ElevenLabs session as a user message wrapped with this prefix so the agent reads + * it verbatim. MUST match `VOICE_READBACK_PREFIX` in `voice/realtime_harness.rs`, + * which uses it to avoid re-arming speak-back on the read-back turn (loop guard). + */ +const READBACK_PREFIX = 'Please read the following to me, word for word, and say nothing else:'; + /** * Lifecycle of a realtime ElevenLabs Agents voice session (#5399). * `idle → connecting → active → idle`, or `→ error`. @@ -125,6 +136,25 @@ export function useRealtimeVoiceSession(opts?: { voiceId?: string }): RealtimeVo [] ); + // Speak-back: a slow voice turn (email/calendar summary) is acknowledged aloud, + // finishes in the background, and the core emits its result as a `voice_speak` + // event. While the call is still open, read it aloud by sending it back into the + // live ElevenLabs session wrapped in the verbatim prefix (a fast read-back turn). + // The result also lands in chat regardless (delivered core-side) — this is the + // spoken copy. Refs keep the subscription set up once while always seeing the + // live conversation and liveness. + useEffect(() => { + const handler = (payload: unknown) => { + if (!liveRef.current) return; // call already ended — the chat copy stands alone + const text = (payload as { full_response?: string } | undefined)?.full_response?.trim(); + if (!text) return; + log('speak-back: reading deferred result aloud (%d chars)', text.length); + conversationRef.current.sendUserMessage(`${READBACK_PREFIX}\n\n${text}`); + }; + socketService.on('voice_speak', handler); + return () => socketService.off('voice_speak', handler); + }, []); + return { state, isSpeaking: conversation.isSpeaking, diff --git a/src/openhuman/agent/harness/session/runtime.rs b/src/openhuman/agent/harness/session/runtime.rs index d499c7fd6f..419825517d 100644 --- a/src/openhuman/agent/harness/session/runtime.rs +++ b/src/openhuman/agent/harness/session/runtime.rs @@ -178,6 +178,16 @@ impl Agent { &self.model_name } + /// Override the base model this session runs its top-level turns on. Set + /// once before running: per-turn classification is disabled (the main agent + /// is pinned to its configured model for KV-cache stability — see the model + /// pin in `turn/core.rs`), so this sticks for the session and is not flipped + /// mid-conversation. The realtime voice harness uses it to pin a fast, + /// non-thinking model within the provider's response-time ceiling. + pub fn set_model_name(&mut self, model_name: impl Into) { + self.model_name = model_name.into(); + } + /// The agent's currently-configured temperature. pub fn temperature(&self) -> f64 { self.temperature diff --git a/src/openhuman/agent/harness/session/turn/core.rs b/src/openhuman/agent/harness/session/turn/core.rs index fb3cd7a96c..ebfe921832 100644 --- a/src/openhuman/agent/harness/session/turn/core.rs +++ b/src/openhuman/agent/harness/session/turn/core.rs @@ -923,6 +923,16 @@ impl Agent { let mut agent_context_prepared_sources: Vec = Vec::new(); + // Triggered memory-agent recall runs on EVERY channel, voice included: + // dropping it on voice would strip the user's remembered context + // (preferences, people, prior facts) from spoken answers — a real quality + // loss the transcript alone can't replace. Recall adds a few seconds of + // embedding + retrieval before the first model token, but on realtime + // voice that latency is already covered end-to-end: the backend relay + // streams an audible keepalive filler from t=0 so the cloud session never + // sees a silent stall, and the desktop's ~8s ack-defer closes the spoken + // turn and finishes in the background if the work runs long. So the recall + // path is byte-for-byte identical across voice and chat. let (enriched, memory_agent_context_injected) = self .inject_triggered_memory_agent_context(user_message, enriched, &parent_context) .await; diff --git a/src/openhuman/voice/realtime_harness.rs b/src/openhuman/voice/realtime_harness.rs index 3eea90e1b4..6b13e92904 100644 --- a/src/openhuman/voice/realtime_harness.rs +++ b/src/openhuman/voice/realtime_harness.rs @@ -21,11 +21,22 @@ use serde_json::{json, Value}; use tokio::sync::Semaphore; use crate::openhuman::agent::harness::session::Agent; +use crate::openhuman::agent::progress::AgentProgress; use crate::openhuman::agent::turn_origin::{with_origin, AgentTurnOrigin}; use crate::openhuman::platform::socket::manager::global_socket_manager; const TURN_TIMEOUT_SECS: u64 = 90; +/// How long a voice turn may run before we stop making the caller wait and hand +/// the result off to chat. The cloud voice session cancels a turn with no spoken +/// token in ~11-12s, and a slow tool action (email/calendar summary can be +/// 20-30s of Composio round-trips) will never fit that window. Once this elapses +/// we close the voice turn cleanly — the spoken acknowledgement has already +/// streamed — and let the orchestrator finish in the background, delivering its +/// answer into the user's in-app chat. Sits comfortably under the provider's +/// cancel deadline so `done` always beats the cut. +const VOICE_ACK_DEADLINE_SECS: u64 = 8; + /// Voice-scoped transcript namespace. Building a fresh orchestrator per turn /// would otherwise resume the *chat* orchestrator's latest transcript by name, /// bleeding an unrelated conversation into (or out of) the voice session. A @@ -33,6 +44,48 @@ const TURN_TIMEOUT_SECS: u64 = 90; /// relayed `messages` we seed below, not from this resume path. const VOICE_AGENT_NAME: &str = "voice"; +/// Model pinned for realtime voice turns. The cloud voice session cancels a turn +/// that has produced no spoken token in ~11-12s ("Generating the LLM response +/// took too long"), and the orchestrator's default reasoning model spends that +/// whole budget *thinking* before its first word. `chat-v1` (DeepSeek-V4-Flash, +/// thinking off) is a short-turn, tool-capable SKU: the master still routes +/// delegation through the prompt (per-turn classification is disabled — see the +/// model pin in `agent/harness/session/turn/core.rs`), so tool turns keep working +/// while spoken replies start in ~1s instead of ~6s. Reasoning models are the +/// wrong tool for a latency-capped realtime channel. +const VOICE_MODEL: &str = "chat-v1"; + +/// Instruction prefix used for "speak-back": when a deferred result is ready, the +/// renderer's live voice session sends it back as a user message wrapped with this +/// prefix so the agent reads it aloud verbatim. The core recognises the prefix to +/// avoid re-arming speak-back on the read-back turn itself (which would loop). MUST +/// match the string the renderer prepends (`useRealtimeVoiceSession.ts`). +const VOICE_READBACK_PREFIX: &str = + "Please read the following to me, word for word, and say nothing else:"; + +/// Chat thread + client id the voice turn scopes as its approval / routing +/// surface, mirroring `deliver_voice_result_to_chat`. Setting these around the +/// turn (via `APPROVAL_CHAT_CONTEXT` + `with_thread_id`) is what makes the voice +/// orchestrator behave like the chat path for tools that need a *routable* +/// approval surface: +/// +/// - `composio_connect` fails closed with a `[policy-denied] … needs an +/// interactive chat turn` message whenever `APPROVAL_CHAT_CONTEXT` is absent +/// (see `integrations/composio/tools.rs`). On voice that message got +/// paraphrased back to the user as "your Gmail connection is throwing an auth +/// error, reconnect it" — the exact voice-only Gmail-summary failure — even +/// though the same request works in tap-and-speak (a `WebChat` turn, which +/// installs this context). With the context set, the tool reaches its +/// already-connected short-circuit and returns success instead. +/// - external_effect tool approvals raised on the `ExternalChannel` turn now +/// have a thread card to route to (the same `proactive:voice` thread where +/// deferred voice answers land) rather than silently TTL-denying. +/// - `with_thread_id` gives async delegation (`spawn_async_subagent`) the +/// `parent_thread_id` it requires and aligns inference logs / KV-cache with +/// the voice thread. +const VOICE_CHAT_THREAD_ID: &str = "proactive:voice"; +const VOICE_CHAT_CLIENT_ID: &str = "system"; + /// Cap on concurrent local-agent turns driven by the relay. Each turn loads /// config, builds a full orchestrator, and runs for up to `TURN_TIMEOUT_SECS`, /// so an unbounded burst (or retry storm) would spawn unbounded heavy agent @@ -97,9 +150,21 @@ fn messages_to_history_pairs(messages: &[Value]) -> Vec<(String, String)> { /// Spoken-output directive appended to the orchestrator profile so replies read /// naturally through TTS instead of as markdown. +/// +/// The tool-preface clause is latency-critical, not cosmetic: the cloud realtime +/// session enforces a per-response time ceiling, and a turn that delegates (email, +/// calendar, files) produces no top-level assistant text until the sub-agent +/// returns 10-20s later — past the ceiling. Emitting one short spoken sentence +/// first makes the model stream an immediate `TextDelta`, so audio reaches the +/// caller right away and the turn stays alive while the tool runs. const VOICE_DIRECTIVE: &str = "You are speaking aloud in a live voice conversation. \ Reply in natural, concise spoken sentences. Do not use markdown, code blocks, \ -bullet lists, headings, or emoji."; +bullet lists, headings, or emoji. Before you use a tool or delegate (for example \ +to check email, a calendar, files, or the web), first say one short spoken \ +sentence telling the user what you are doing — and, because those actions can take \ +a while, that you will put the details in their chat — then proceed. Example: \ +\"Sure, let me pull up your inbox — I'll drop the summary in your chat.\" Keep that \ +preface to one sentence so it starts speaking immediately."; /// Extract the user prompt from an OpenAI-style `messages` array: the content of /// the last `user` message. Content may be a plain string or an array of @@ -125,9 +190,13 @@ fn content_to_text(content: Option<&Value>) -> String { } } -/// Handle one relayed voice turn end to end: run the orchestrator and emit the -/// reply back up the socket. Never panics — every failure path emits -/// `voice:harness:error` so the backend relay ends the turn cleanly. +/// Handle one relayed voice turn end to end. The reply streams token-by-token +/// back up the socket as it is produced. A turn that finishes inside the voice +/// window is spoken in full; a slow tool action (email/calendar summary — tens of +/// seconds of Composio round-trips) is acknowledged aloud, then finishes in the +/// background and delivers its answer into the user's in-app chat. Never panics — +/// every failure path emits `voice:harness:error` (or a clean `done`) so the +/// backend relay ends the turn cleanly. pub async fn handle_voice_harness_turn(correlation_id: String, messages: Vec) { let prompt = extract_prompt(&messages); if prompt.trim().is_empty() { @@ -135,44 +204,202 @@ pub async fn handle_voice_harness_turn(correlation_id: String, messages: Vec { - let spoken = reply.trim(); - if !spoken.is_empty() { - emit_event( - "voice:harness:delta", - json!({ "correlationId": correlation_id, "text": spoken }), - ) - .await; + // Bound concurrent heavy agent turns; excess turns queue here. Owned so it is + // held for the REAL turn lifetime (including any background tail), not just the + // shortened spoken ack — which also caps the retry storm when the provider + // re-sends a turn it thinks stalled. + let permit = match VOICE_TURN_LIMITER.clone().acquire_owned().await { + Ok(permit) => permit, + Err(_) => { + emit_error(&correlation_id, "voice turn limiter unavailable").await; + return; + } + }; + + // Stream the reply token-by-token: the orchestrator emits `TextDelta` as it + // generates, and `forward_reply_deltas` relays each as a `voice:harness:delta` + // so the spoken acknowledgement leaves the desktop immediately. + let (progress_tx, progress_rx) = tokio::sync::mpsc::channel::(256); + let forwarder = tokio::spawn(forward_reply_deltas(progress_rx, correlation_id.clone())); + + // Run the turn on a detached task so it can outlive the spoken ack. A slow + // delegation keeps working after we close the voice turn and delivers its + // result into the user's chat. The task owns the agent, the concurrency + // permit, and the in-flight guard so that bookkeeping tracks the real turn. + let (result_tx, result_rx) = tokio::sync::oneshot::channel::>(); + let turn_cid = correlation_id.clone(); + let turn_messages = messages; + let turn_prompt = prompt; + tokio::spawn(async move { + let _in_flight = in_flight; + let _permit = permit; + let outcome = run_voice_turn(&turn_cid, &turn_messages, &turn_prompt, progress_tx).await; + // Hand the result to the foreground. If it already deferred (dropped the + // receiver at the ack deadline), the send fails and we deliver the reply + // into the user's chat instead. + if let Err(unsent) = result_tx.send(outcome) { + match unsent { + Ok(reply) => { + // Arm speak-back only for a genuine deferred answer — never for a + // read-back echo turn, or the spoken copy would loop forever. + let allow_speak_back = should_arm_speak_back(&turn_prompt); + deliver_voice_result_to_chat(&turn_cid, reply, allow_speak_back); + } + Err(err) => { + warn!("[voice-harness] deferred turn failed correlation={turn_cid}: {err}") + } } + } + }); + + // Race the turn against the spoken-ack deadline. + match tokio::time::timeout(Duration::from_secs(VOICE_ACK_DEADLINE_SECS), result_rx).await { + Ok(Ok(outcome)) => { + // Finished inside the voice window — deltas already streamed. Join the + // forwarder so every delta is out before `done`, and learn whether + // anything streamed (fallback for a non-streaming reply). + let streamed_any = forwarder.await.unwrap_or(false); + match outcome { + Ok(reply) => { + if !streamed_any { + let spoken = reply.trim(); + if !spoken.is_empty() { + emit_event( + "voice:harness:delta", + json!({ "correlationId": correlation_id, "text": spoken }), + ) + .await; + } + } + emit_event( + "voice:harness:done", + json!({ "correlationId": correlation_id }), + ) + .await; + } + Err(err) => { + warn!("[voice-harness] turn failed correlation={correlation_id}: {err}"); + emit_error(&correlation_id, &err).await; + } + } + } + Ok(Err(_recv)) => { + // Sender dropped without a value (task aborted). End cleanly. + emit_event( + "voice:harness:done", + json!({ "correlationId": correlation_id }), + ) + .await; + } + Err(_deadline) => { + // Still running (a slow tool action). The spoken ack has streamed, so + // close the voice turn cleanly; the detached task keeps going and + // delivers its answer into the user's chat. `timeout` consumed + // `result_rx`, so the task's send fails and takes the chat-delivery + // path. The forwarder is left running to keep draining progress — any + // late deltas reach a settled relay turn and are dropped harmlessly. + info!("[voice-harness] ack deadline reached, handing off to chat correlation={correlation_id}"); emit_event( "voice:harness:done", json!({ "correlationId": correlation_id }), ) .await; } - Err(err) => { - warn!("[voice-harness] turn failed correlation={correlation_id}: {err}"); - emit_error(&correlation_id, &err).await; + } +} + +/// Forward the orchestrator's streamed assistant text to the relay socket, one +/// `voice:harness:delta` per top-level `AgentProgress::TextDelta`, until the +/// turn's progress channel closes (the agent drops its sender when the turn +/// ends). Returns whether any non-empty delta was streamed, so the caller can +/// fall back to emitting the whole reply for a turn that produced text off the +/// streaming path. Only top-level assistant text is voiced — sub-agent narration, +/// thinking, tool-call args, and lifecycle events are deliberately not spoken. +async fn forward_reply_deltas( + mut progress_rx: tokio::sync::mpsc::Receiver, + correlation_id: String, +) -> bool { + let mut streamed_any = false; + while let Some(progress) = progress_rx.recv().await { + let Some(text) = spoken_delta(&progress) else { + continue; + }; + // Skip only truly empty deltas — whitespace carries word boundaries and + // must be forwarded so the concatenated speech isn't run together. + if text.is_empty() { + continue; } + streamed_any = true; + emit_event( + "voice:harness:delta", + json!({ "correlationId": correlation_id, "text": text }), + ) + .await; + } + streamed_any +} + +/// The spoken text carried by a progress event, or `None` for events that must +/// not be voiced. Only the top-level assistant `TextDelta` is spoken; sub-agent +/// deltas, thinking, tool-call args, and lifecycle events are internal. Pure + +/// unit-tested. +fn spoken_delta(progress: &AgentProgress) -> Option<&str> { + match progress { + AgentProgress::TextDelta { delta, .. } => Some(delta), + _ => None, } } -async fn run_agent_turn( +/// Whether a completed voice turn should arm speak-back — i.e. push its deferred +/// answer back into the live session to be read aloud. A read-back turn is itself +/// a verbatim-read request (its prompt is wrapped with [`VOICE_READBACK_PREFIX`] +/// by the renderer), so re-arming speak-back on it would deliver the spoken copy +/// to a turn that then asks to read it again — an unbounded loop. Suppress those. +/// Pure + unit-tested; leading whitespace is tolerated because the renderer joins +/// the prefix and payload with a blank line. +fn should_arm_speak_back(prompt: &str) -> bool { + !prompt.trim_start().starts_with(VOICE_READBACK_PREFIX) +} + +/// Build the fresh voice orchestrator, attach the streaming sink, run one turn +/// under the hard per-turn ceiling, then detach the sink so the forwarder's +/// channel closes. Runs entirely on the background task, so the ack deadline in +/// the caller covers both the build and the model round-trips. +async fn run_voice_turn( correlation_id: &str, messages: &[Value], prompt: &str, + progress_tx: tokio::sync::mpsc::Sender, ) -> Result { + let mut agent = build_voice_agent(correlation_id, messages, prompt).await?; + + // Attach the streaming sink before the turn: its presence switches the harness + // onto the true per-token streaming path, and each `AgentProgress::TextDelta` + // is forwarded to the relay socket by `forward_reply_deltas`. + agent.set_on_progress(Some(progress_tx)); + + let outcome = run_single_with_timeout(&mut agent, correlation_id, prompt).await; + + // Detach the sink so the forwarder's channel closes the moment the turn ends, + // deterministically rather than waiting on `agent`'s drop. + agent.set_on_progress(None); + outcome +} + +/// Construct the per-turn voice orchestrator: load config, pin the fast voice +/// model, isolate the transcript namespace, and seed the relayed history. +async fn build_voice_agent( + correlation_id: &str, + messages: &[Value], + prompt: &str, +) -> Result { let config = crate::openhuman::config::ops::load_config_with_timeout().await?; let mut agent = Agent::from_config_for_agent_with_profile( &config, @@ -186,6 +413,9 @@ async fn run_agent_turn( // Isolate the voice transcript namespace from the chat orchestrator so a // fresh-per-turn agent can't resume an unrelated conversation by name. agent.set_agent_definition_name(VOICE_AGENT_NAME); + // Pin a fast, non-thinking model so the first spoken token lands inside the + // realtime session's response-time ceiling (see VOICE_MODEL). + agent.set_model_name(VOICE_MODEL); // Seed the authoritative prior turns the relay carries (OpenAI `messages`), // so follow-ups like "what about tomorrow?" keep their context. No-ops when @@ -200,7 +430,31 @@ async fn run_agent_turn( prompt.chars().count(), messages.len() ); + Ok(agent) +} +/// Run the orchestrator turn under the hard per-turn ceiling. The streaming sink +/// must already be attached; deltas flow out while this runs. +async fn run_single_with_timeout( + agent: &mut Agent, + correlation_id: &str, + prompt: &str, +) -> Result { + // Scope the turn with the SAME chat context the web-chat path installs + // (`APPROVAL_CHAT_CONTEXT` + `with_thread_id`), so approval-surfaced tools + // behave identically on voice. Without it `composio_connect` fails closed + // for lack of a routable surface, which the model paraphrases to the user as + // a confabulated "reconnect your Gmail" mid email-summary (#5399). See + // VOICE_CHAT_THREAD_ID for the full rationale. Nesting mirrors web chat: + // origin (outer) → approval context → thread id → the agent run. + let approval_ctx = crate::openhuman::security::approval::ApprovalChatContext { + thread_id: VOICE_CHAT_THREAD_ID.to_string(), + client_id: VOICE_CHAT_CLIENT_ID.to_string(), + }; + let scoped_run = crate::openhuman::agent::tinyagents::thread_context::with_thread_id( + VOICE_CHAT_THREAD_ID, + agent.run_single(prompt), + ); let fut = with_origin( AgentTurnOrigin::ExternalChannel { channel: "voice".to_string(), @@ -208,7 +462,7 @@ async fn run_agent_turn( reply_target: correlation_id.to_string(), message_id: format!("voice-{correlation_id}"), }, - agent.run_single(prompt), + crate::openhuman::security::approval::APPROVAL_CHAT_CONTEXT.scope(approval_ctx, scoped_run), ); match tokio::time::timeout(Duration::from_secs(TURN_TIMEOUT_SECS), fut).await { @@ -220,6 +474,47 @@ async fn run_agent_turn( } } +/// Deliver a deferred voice turn's answer into the user's in-app chat. Publishes +/// a `proactive_message` on the web-channel event bus — the same seam cron and the +/// subconscious use — which the frontend renders as an assistant message in a +/// visible thread. Web-only: it does not fan out to external channels (#5399). +fn deliver_voice_result_to_chat(correlation_id: &str, reply: String, allow_speak_back: bool) { + let spoken = reply.trim(); + if spoken.is_empty() { + warn!("[voice-harness] deferred turn produced no text correlation={correlation_id}"); + return; + } + info!( + "[voice-harness] delivering deferred result to chat correlation={correlation_id} chars={} speak_back={allow_speak_back}", + spoken.chars().count() + ); + crate::openhuman::web_chat::publish_web_channel_event(crate::core::socketio::WebChannelEvent { + event: "proactive_message".to_string(), + client_id: VOICE_CHAT_CLIENT_ID.to_string(), + thread_id: VOICE_CHAT_THREAD_ID.to_string(), + full_response: Some(spoken.to_string()), + success: Some(true), + ..Default::default() + }); + + // Speak-back: push the finished answer to the renderer's LIVE voice session so + // the agent can read it aloud. The frontend voice hook listens for `voice_speak` + // and, only while the call is still open, sends it back into the ElevenLabs + // session (a fast read-back turn). Skipped for read-back turns themselves to + // avoid a loop; harmless if the call already ended (nobody is subscribed). + if allow_speak_back { + crate::openhuman::web_chat::publish_web_channel_event( + crate::core::socketio::WebChannelEvent { + event: "voice_speak".to_string(), + client_id: VOICE_CHAT_CLIENT_ID.to_string(), + full_response: Some(spoken.to_string()), + success: Some(true), + ..Default::default() + }, + ); + } +} + async fn emit_event(event: &str, payload: Value) { match global_socket_manager() { Some(mgr) => { @@ -302,4 +597,71 @@ mod tests { let pairs = messages_to_history_pairs(&messages); assert_eq!(pairs, vec![("user".to_string(), "hello".to_string())]); } + + #[test] + fn spoken_delta_forwards_only_top_level_assistant_text() { + assert_eq!( + spoken_delta(&AgentProgress::TextDelta { + delta: "hey there".to_string(), + iteration: 1, + }), + Some("hey there") + ); + // Whitespace-only deltas carry word boundaries and are still spoken text — + // the empty-skip lives in the forwarder, not here. + assert_eq!( + spoken_delta(&AgentProgress::TextDelta { + delta: " ".to_string(), + iteration: 2, + }), + Some(" ") + ); + } + + #[test] + fn spoken_delta_suppresses_internal_events() { + // Reasoning must never be voiced. + assert_eq!( + spoken_delta(&AgentProgress::ThinkingDelta { + delta: "let me think".to_string(), + iteration: 1, + }), + None + ); + // A delegated sub-agent's narration is internal, not the spoken answer. + assert_eq!( + spoken_delta(&AgentProgress::SubagentTextDelta { + agent_id: "a".to_string(), + task_id: "t".to_string(), + delta: "fetching inbox".to_string(), + iteration: 1, + }), + None + ); + // Lifecycle events carry no spoken text. + assert_eq!( + spoken_delta(&AgentProgress::TurnCompleted { iterations: 1 }), + None + ); + } + + #[test] + fn speak_back_armed_for_a_genuine_answer_turn() { + assert!(should_arm_speak_back("summarize my unread emails")); + assert!(should_arm_speak_back("what's on my calendar tomorrow?")); + } + + #[test] + fn speak_back_suppressed_for_a_read_back_turn() { + // The bare prefix, and the real renderer shape (prefix + blank line + + // payload, possibly with leading whitespace) must both be recognised so + // the spoken copy never re-arms into an unbounded loop. + assert!(!should_arm_speak_back(VOICE_READBACK_PREFIX)); + assert!(!should_arm_speak_back(&format!( + "{VOICE_READBACK_PREFIX}\n\nHere is your inbox summary." + ))); + assert!(!should_arm_speak_back(&format!( + " \n{VOICE_READBACK_PREFIX} trailing payload" + ))); + } } From 4469d525c7ba94b28cf39564f7a3ea33438018e3 Mon Sep 17 00:00:00 2001 From: Shanu Date: Mon, 10 Aug 2026 22:54:00 +0530 Subject: [PATCH 02/18] fix(voice): address review on the deferred-turn path - Make the VOICE_DIRECTIVE chat promise conditional ("if it takes a while, I'll follow up in your chat"), so a fast in-window turn no longer promises a chat entry that never appears. - Skip chat delivery entirely for a deferred read-back turn (not just the spoken copy), so the echoed answer is never re-posted to proactive:voice and a read-back turn can't duplicate the chat message. - Post a failure notice to the voice chat thread when a deferred turn errors after the spoken turn already closed, so the promised chat message always appears instead of only a warn log. --- src/openhuman/voice/realtime_harness.rs | 52 ++++++++++++++++++++----- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/src/openhuman/voice/realtime_harness.rs b/src/openhuman/voice/realtime_harness.rs index 6b13e92904..529c0c4332 100644 --- a/src/openhuman/voice/realtime_harness.rs +++ b/src/openhuman/voice/realtime_harness.rs @@ -161,10 +161,11 @@ const VOICE_DIRECTIVE: &str = "You are speaking aloud in a live voice conversati Reply in natural, concise spoken sentences. Do not use markdown, code blocks, \ bullet lists, headings, or emoji. Before you use a tool or delegate (for example \ to check email, a calendar, files, or the web), first say one short spoken \ -sentence telling the user what you are doing — and, because those actions can take \ -a while, that you will put the details in their chat — then proceed. Example: \ -\"Sure, let me pull up your inbox — I'll drop the summary in your chat.\" Keep that \ -preface to one sentence so it starts speaking immediately."; +sentence telling the user what you are doing — and, since those actions sometimes \ +take a while, that you'll follow up with the details in their chat IF it does — \ +then proceed. Example: \"Sure, let me pull up your inbox — if it takes a moment \ +I'll drop the summary in your chat.\" Keep that preface to one sentence so it \ +starts speaking immediately."; /// Extract the user prompt from an OpenAI-style `messages` array: the content of /// the last `user` message. Content may be a plain string or an array of @@ -247,13 +248,24 @@ pub async fn handle_voice_harness_turn(correlation_id: String, messages: Vec { - // Arm speak-back only for a genuine deferred answer — never for a - // read-back echo turn, or the spoken copy would loop forever. - let allow_speak_back = should_arm_speak_back(&turn_prompt); - deliver_voice_result_to_chat(&turn_cid, reply, allow_speak_back); + // A read-back turn only re-reads an answer already delivered to + // chat; delivering it again would duplicate the chat message, and + // re-arming speak-back would loop. So deliver + arm speak-back ONLY + // for a genuine deferred answer, and skip the whole delivery for a + // read-back echo turn. + if should_arm_speak_back(&turn_prompt) { + deliver_voice_result_to_chat(&turn_cid, reply, true); + } else { + info!("[voice-harness] deferred read-back turn carries no new content; skipping chat delivery correlation={turn_cid}"); + } } Err(err) => { - warn!("[voice-harness] deferred turn failed correlation={turn_cid}: {err}") + // The spoken turn already closed with `done` and the preface may + // have promised a chat follow-up, so a silent failure would leave + // the user waiting for a message that never arrives. Post a brief + // failure notice to the same thread. + warn!("[voice-harness] deferred turn failed correlation={turn_cid}: {err}"); + deliver_voice_failure_to_chat(&turn_cid); } } } @@ -515,6 +527,28 @@ fn deliver_voice_result_to_chat(correlation_id: &str, reply: String, allow_speak } } +/// Deliver a short failure notice to the voice chat thread when a deferred turn +/// errors after the spoken turn already closed. Because the spoken preface may +/// have told the user their answer would land in chat, a silent failure would +/// leave them waiting on a message that never comes — this makes the promised +/// message always appear. Delivered as a normal assistant message (not spoken) +/// on the same `proactive:voice` surface as a successful deferred answer. +fn deliver_voice_failure_to_chat(correlation_id: &str) { + info!( + "[voice-harness] delivering deferred failure notice to chat correlation={correlation_id}" + ); + crate::openhuman::web_chat::publish_web_channel_event(crate::core::socketio::WebChannelEvent { + event: "proactive_message".to_string(), + client_id: VOICE_CHAT_CLIENT_ID.to_string(), + thread_id: VOICE_CHAT_THREAD_ID.to_string(), + full_response: Some( + "Sorry — I couldn't finish that request just now. Please try again.".to_string(), + ), + success: Some(true), + ..Default::default() + }); +} + async fn emit_event(event: &str, payload: Value) { match global_socket_manager() { Some(mgr) => { From fdb33802b195ca846a808ef9f912fab4f6675b09 Mon Sep 17 00:00:00 2001 From: Shanu Date: Mon, 10 Aug 2026 23:26:06 +0530 Subject: [PATCH 03/18] chore(build): fix pre-existing fmt + gates-off breakage on the base This PR's rust changes make CI run the whole-repo fmt check and the gates-off smoke build, which surface two latent issues on the base that are unrelated to the voice change itself: - web3/wallet/chains/btc.rs: apply rustfmt (toolchain 1.96.1) to a stray unformatted line. - memory/diff/mod.rs: gate `pub use tools::MemoryDiffTool` behind `memory-git` (its `tools` module is already `#[cfg(feature = "memory-git")]`), so the slim build no longer references a compiled-out module. - memory/diff/stub.rs: import Checkpoint/CrossSourceDiff/Snapshot from `tinycortex::memory::diff::types` (where mod.rs re-exports them) instead of a non-existent `super::types`, so the gates-off build resolves them. --- src/openhuman/memory/diff/mod.rs | 1 + src/openhuman/memory/diff/stub.rs | 2 +- src/openhuman/web3/wallet/chains/btc.rs | 4 +--- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/openhuman/memory/diff/mod.rs b/src/openhuman/memory/diff/mod.rs index 30e8357766..fecf0954af 100644 --- a/src/openhuman/memory/diff/mod.rs +++ b/src/openhuman/memory/diff/mod.rs @@ -76,4 +76,5 @@ pub use tinycortex::memory::diff::types::{ ChangeKind, Checkpoint, CrossSourceDiff, DiffResult, DiffSummary, ItemChange, Snapshot, SnapshotTrigger, }; +#[cfg(feature = "memory-git")] pub use tools::MemoryDiffTool; diff --git a/src/openhuman/memory/diff/stub.rs b/src/openhuman/memory/diff/stub.rs index ed0e1428e2..0d76947233 100644 --- a/src/openhuman/memory/diff/stub.rs +++ b/src/openhuman/memory/diff/stub.rs @@ -26,7 +26,7 @@ use crate::openhuman::config::Config; use crate::openhuman::memory::sources::types::MemorySourceEntry; -use super::types::{Checkpoint, CrossSourceDiff, Snapshot}; +use tinycortex::memory::diff::types::{Checkpoint, CrossSourceDiff, Snapshot}; /// The message every disabled entry point returns. /// diff --git a/src/openhuman/web3/wallet/chains/btc.rs b/src/openhuman/web3/wallet/chains/btc.rs index b44524ccb4..71024c6f05 100644 --- a/src/openhuman/web3/wallet/chains/btc.rs +++ b/src/openhuman/web3/wallet/chains/btc.rs @@ -464,9 +464,7 @@ mod tests { #[test] fn validate_btc_address_rejects_testnet() { - let err = - validate_btc_address("tb1qw508d6qejxtdg4y5r3zarvary0c5xw7kxpjzsx") - .unwrap_err(); + let err = validate_btc_address("tb1qw508d6qejxtdg4y5r3zarvary0c5xw7kxpjzsx").unwrap_err(); // `tinywallet` reports a wrong-network address as a distinct condition // from a malformed one, so the message names the required network. assert!(err.contains("not on mainnet"), "got: {err}"); From 41c6f4f5e78988e9504f739cb6aee6e85d6ba8af Mon Sep 17 00:00:00 2001 From: Shanu Date: Mon, 10 Aug 2026 23:30:28 +0530 Subject: [PATCH 04/18] fix(voice): surface a chat message on empty deferred reply + correct failure flag - Route an empty deferred reply through deliver_voice_failure_to_chat so the chat follow-up the spoken preface promised always appears, instead of a silent early return. - Set success=false on the deferred-failure notice so a client that branches on the flag reads it as a failed turn rather than a success. --- src/openhuman/voice/realtime_harness.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/openhuman/voice/realtime_harness.rs b/src/openhuman/voice/realtime_harness.rs index 529c0c4332..e41e4a5468 100644 --- a/src/openhuman/voice/realtime_harness.rs +++ b/src/openhuman/voice/realtime_harness.rs @@ -494,6 +494,9 @@ fn deliver_voice_result_to_chat(correlation_id: &str, reply: String, allow_speak let spoken = reply.trim(); if spoken.is_empty() { warn!("[voice-harness] deferred turn produced no text correlation={correlation_id}"); + // The spoken ack already promised a chat follow-up, so an empty deferred + // reply must still surface a message rather than leave the user waiting. + deliver_voice_failure_to_chat(correlation_id); return; } info!( @@ -544,7 +547,7 @@ fn deliver_voice_failure_to_chat(correlation_id: &str) { full_response: Some( "Sorry — I couldn't finish that request just now. Please try again.".to_string(), ), - success: Some(true), + success: Some(false), ..Default::default() }); } From 2988802c5cdf2c41ec7d6e987a19dc1b0baaff5e Mon Sep 17 00:00:00 2001 From: Shanu Date: Mon, 10 Aug 2026 23:56:05 +0530 Subject: [PATCH 05/18] test(voice): cover the speak-back path in useRealtimeVoiceSession Add tests for the voice_speak subscription: reads a deferred result aloud while a call is live, ignores it when no call is live or the payload is empty, and unsubscribes on unmount. Covers the changed lines so the diff meets the coverage gate. --- .../voice/useRealtimeVoiceSession.test.ts | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/app/src/features/human/voice/useRealtimeVoiceSession.test.ts b/app/src/features/human/voice/useRealtimeVoiceSession.test.ts index d6226e44e5..6f73e36814 100644 --- a/app/src/features/human/voice/useRealtimeVoiceSession.test.ts +++ b/app/src/features/human/voice/useRealtimeVoiceSession.test.ts @@ -12,11 +12,31 @@ interface CapturedProps { let captured: CapturedProps | null = null; const startSession = vi.fn(); const endSession = vi.fn(); +const sendUserMessage = vi.fn(); vi.mock('@elevenlabs/react', () => ({ useConversation: (props: CapturedProps) => { captured = props; - return { startSession, endSession, isSpeaking: false, mode: 'listening' as const }; + return { + startSession, + endSession, + sendUserMessage, + isSpeaking: false, + mode: 'listening' as const, + }; + }, +})); + +// Capture the `voice_speak` subscription so a test can drive the speak-back path. +const socketHandlers: Record void> = {}; +vi.mock('../../../services/socketService', () => ({ + socketService: { + on: vi.fn((event: string, handler: (payload: unknown) => void) => { + socketHandlers[event] = handler; + }), + off: vi.fn((event: string) => { + delete socketHandlers[event]; + }), }, })); @@ -29,6 +49,7 @@ describe('useRealtimeVoiceSession', () => { beforeEach(() => { vi.clearAllMocks(); captured = null; + Object.keys(socketHandlers).forEach(k => delete socketHandlers[k]); }); it('fetches a signed URL and opens a WebSocket session with the voice override', async () => { @@ -137,4 +158,43 @@ describe('useRealtimeVoiceSession', () => { unmount(); expect(endSession).not.toHaveBeenCalled(); }); + + it('reads a deferred result aloud when voice_speak arrives during a live call', async () => { + mockFetch.mockResolvedValueOnce({ signedUrl: 'wss://x', agentId: 'a1', userToken: 'tok-1' }); + const { result } = renderHook(() => useRealtimeVoiceSession()); + await act(async () => { + await result.current.start(); + }); + act(() => captured?.onConnect()); // liveRef becomes true + act(() => socketHandlers['voice_speak']?.({ full_response: 'Your inbox summary.' })); + expect(sendUserMessage).toHaveBeenCalledTimes(1); + // Wrapped in the verbatim read-back prefix so the agent reads it aloud. + expect(sendUserMessage.mock.calls[0][0]).toContain('Your inbox summary.'); + expect(sendUserMessage.mock.calls[0][0]).toContain('Please read the following'); + }); + + it('ignores voice_speak when no call is live', () => { + renderHook(() => useRealtimeVoiceSession()); // never connected → liveRef stays false + act(() => socketHandlers['voice_speak']?.({ full_response: 'ignored' })); + expect(sendUserMessage).not.toHaveBeenCalled(); + }); + + it('ignores an empty or missing voice_speak payload', async () => { + mockFetch.mockResolvedValueOnce({ signedUrl: 'wss://x', agentId: 'a1', userToken: 'tok-1' }); + const { result } = renderHook(() => useRealtimeVoiceSession()); + await act(async () => { + await result.current.start(); + }); + act(() => captured?.onConnect()); + act(() => socketHandlers['voice_speak']?.({ full_response: ' ' })); + act(() => socketHandlers['voice_speak']?.(undefined)); + expect(sendUserMessage).not.toHaveBeenCalled(); + }); + + it('unsubscribes from voice_speak on unmount', () => { + const { unmount } = renderHook(() => useRealtimeVoiceSession()); + expect(socketHandlers['voice_speak']).toBeDefined(); + unmount(); + expect(socketHandlers['voice_speak']).toBeUndefined(); + }); }); From 2fe7e2eafdfab06151ebe17e99a9cc99077b4835 Mon Sep 17 00:00:00 2001 From: Shanu Date: Tue, 11 Aug 2026 00:01:32 +0530 Subject: [PATCH 06/18] fix(build): gate the memory_diff capability registration on memory-git Under gates-off the memory diff controllers compile out (the stub returns none), but core/all.rs registered the memory_diff capability unconditionally -- leaving a stale namespace with no controllers behind it, which the gates-off capability-map guard tests reject. Gate the push_cap on memory-git so the namespace is absent when the feature is, matching the already-passing memory_diff_controllers_absent test. --- src/core/all.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/all.rs b/src/core/all.rs index 5e6029b0a6..3e04386bf4 100644 --- a/src/core/all.rs +++ b/src/core/all.rs @@ -866,7 +866,11 @@ fn build_registered_controllers() -> Vec { Some(Capability::Sources), crate::openhuman::memory::sources::all_memory_sources_registered_controllers(), ); - // Memory diff — snapshot-based change tracking for memory sources + // Memory diff — snapshot-based change tracking for memory sources. + // Gated on `memory-git`: with the feature off the diff controllers compile + // out (the stub returns none), so registering the capability unconditionally + // would leave a stale `memory_diff` namespace with no controllers behind it. + #[cfg(feature = "memory-git")] push_cap( &mut controllers, DomainGroup::Memory, From 67c63453ab51556d222e43e20590c52a093a50af Mon Sep 17 00:00:00 2001 From: Shanu Date: Tue, 11 Aug 2026 00:42:41 +0530 Subject: [PATCH 07/18] fix(voice): update realtime-mode tests; revert gates-off scope-creep - HumanPage.realtimeMode.test.tsx: the realtime controls are now shown unconditionally (the former build-flag + voice-mode gate was removed), so assert they render regardless of the persisted mode instead of hiding on the classic default. - Remove VoicePanel.realtimeMode.test.tsx: it exercised the voice-mode toggle this PR removed from the panel. - Revert src/core/all.rs and src/openhuman/memory/diff/{mod,stub}.rs to upstream. The slim (--no-default-features) capability surface needs a coordinated fix across core/all.rs, tools/ops.rs and the memory-diff stub; that belongs in a dedicated cleanup, not this voice change. --- .../VoicePanel.realtimeMode.test.tsx | 79 ------------------- .../human/HumanPage.realtimeMode.test.tsx | 29 +++---- src/core/all.rs | 6 +- src/openhuman/memory/diff/mod.rs | 1 - src/openhuman/memory/diff/stub.rs | 2 +- 5 files changed, 14 insertions(+), 103 deletions(-) delete mode 100644 app/src/components/settings/panels/__tests__/VoicePanel.realtimeMode.test.tsx diff --git a/app/src/components/settings/panels/__tests__/VoicePanel.realtimeMode.test.tsx b/app/src/components/settings/panels/__tests__/VoicePanel.realtimeMode.test.tsx deleted file mode 100644 index 8d3febc3b9..0000000000 --- a/app/src/components/settings/panels/__tests__/VoicePanel.realtimeMode.test.tsx +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Unit test for VoicePanel's realtime voice-mode toggle (#5399). The section is - * gated behind the `VOICE_MODE_FLAG_ENABLED` build flag (global setup ships it - * OFF), so it is flipped ON here. Toggling the switch dispatches - * `setVoiceMode('realtime')` against the mascot slice. Mount-time voice APIs are - * stubbed so the panel renders without a backend. - */ -import { act, fireEvent, screen } from '@testing-library/react'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; - -import { renderWithProviders } from '../../../../test/test-utils'; -import VoicePanel from '../VoicePanel'; - -// Flip the realtime gate ON for this file; keep every other config export real. -vi.mock('../../../../utils/config', async () => { - const actual = await vi.importActual( - '../../../../utils/config' - ); - return { ...actual, VOICE_MODE_FLAG_ENABLED: true }; -}); - -vi.mock('../../../../utils/tauriCommands', () => ({ - openhumanGetVoiceServerSettings: vi.fn(async () => ({ result: {}, logs: [] })), - openhumanUpdateVoiceServerSettings: vi.fn(async () => ({ result: {}, logs: [] })), - openhumanVoiceSetProviders: vi.fn(async () => ({})), - openhumanVoiceStatus: vi.fn(async () => ({ stt_provider: 'cloud', tts_provider: 'cloud' })), - syncNotchVisibility: vi.fn(async () => undefined), -})); - -vi.mock('../../../../services/api/voiceInstallApi', () => ({ - installWhisper: vi.fn(), - installPiper: vi.fn(), - whisperInstallStatus: vi.fn(async () => ({ engine: 'whisper', state: 'missing' })), - piperInstallStatus: vi.fn(async () => ({ engine: 'piper', state: 'missing' })), -})); - -vi.mock('../../../../services/api/voiceSettingsApi', async () => { - const actual = await vi.importActual( - '../../../../services/api/voiceSettingsApi' - ); - return { - ...actual, - loadVoiceSettings: vi.fn(async () => ({ - voiceProviders: [], - sttProvider: { kind: 'cloud' }, - ttsProvider: { kind: 'cloud' }, - })), - saveVoiceSettings: vi.fn(async () => undefined), - setVoiceProviderKey: vi.fn(async () => undefined), - clearVoiceProviderKey: vi.fn(async () => undefined), - testVoiceProvider: vi.fn(async () => ({ ok: true, detail: 'OK' })), - }; -}); - -vi.mock('../../../../features/human/voice/ttsClient', async () => { - const actual = await vi.importActual( - '../../../../features/human/voice/ttsClient' - ); - return { ...actual, synthesizeSpeech: vi.fn() }; -}); - -describe('VoicePanel — realtime voice-mode toggle', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('dispatches setVoiceMode when the realtime switch is toggled on', async () => { - const { store } = renderWithProviders(, { initialEntries: ['/settings/voice'] }); - - const toggle = await screen.findByTestId('voice-mode-realtime-toggle'); - expect(store.getState().mascot.voiceMode).toBe('classic'); - - await act(async () => { - fireEvent.click(toggle); - }); - - expect(store.getState().mascot.voiceMode).toBe('realtime'); - }); -}); diff --git a/app/src/features/human/HumanPage.realtimeMode.test.tsx b/app/src/features/human/HumanPage.realtimeMode.test.tsx index 0bd42facf0..23819363d0 100644 --- a/app/src/features/human/HumanPage.realtimeMode.test.tsx +++ b/app/src/features/human/HumanPage.realtimeMode.test.tsx @@ -1,8 +1,8 @@ /** - * Unit tests for HumanPage's realtime voice overlay gate (#5399). The overlay - * renders only when the build flag is on AND the persisted mascot voice mode is - * `realtime`; the classic push-to-talk path is always present. Config is mocked - * with the flag ON here (the global setup mock ships it OFF), and + * Unit test for HumanPage's realtime voice overlay (#5399). The realtime + * controls are now shown unconditionally on the Human tab — the former + * build-flag + persisted-voice-mode gate was removed so the "Start Voice Chat" + * control is always available — alongside the classic push-to-talk path. * RealtimeVoiceControls is stubbed so the ElevenLabs SDK never loads. */ import { configureStore } from '@reduxjs/toolkit'; @@ -15,13 +15,6 @@ import mascotReducer, { setVoiceMode } from '../../store/mascotSlice'; import threadReducer from '../../store/threadSlice'; import HumanPage from './HumanPage'; -// Flip the realtime gate ON for this file (global setup ships it OFF). Spread -// the real module so every other config export keeps its production value. -vi.mock('../../utils/config', async () => { - const actual = await vi.importActual('../../utils/config'); - return { ...actual, VOICE_MODE_FLAG_ENABLED: true }; -}); - // Stub the overlay so the ElevenLabs `ConversationProvider`/SDK never mounts — // this test only pins the render gate, not the controls (covered separately). vi.mock('./RealtimeVoiceControls', () => ({ @@ -58,18 +51,20 @@ function renderWithVoiceMode(mode: 'classic' | 'realtime') { ); } -describe('HumanPage — realtime voice overlay gate', () => { +describe('HumanPage — realtime voice overlay', () => { beforeEach(() => { localStorage.clear(); }); - it('renders the realtime controls when voice mode is realtime and the flag is on', () => { - renderWithVoiceMode('realtime'); + it('renders the realtime controls regardless of the persisted voice mode', () => { + // The gate was removed, so the controls appear even when the persisted mode + // is the classic default — the two paths now coexist on the Human tab. + renderWithVoiceMode('classic'); expect(screen.getByTestId('realtime-voice-controls-stub')).toBeInTheDocument(); }); - it('hides the realtime controls when voice mode is classic', () => { - renderWithVoiceMode('classic'); - expect(screen.queryByTestId('realtime-voice-controls-stub')).not.toBeInTheDocument(); + it('still renders the realtime controls when the persisted mode is realtime', () => { + renderWithVoiceMode('realtime'); + expect(screen.getByTestId('realtime-voice-controls-stub')).toBeInTheDocument(); }); }); diff --git a/src/core/all.rs b/src/core/all.rs index 3e04386bf4..5e6029b0a6 100644 --- a/src/core/all.rs +++ b/src/core/all.rs @@ -866,11 +866,7 @@ fn build_registered_controllers() -> Vec { Some(Capability::Sources), crate::openhuman::memory::sources::all_memory_sources_registered_controllers(), ); - // Memory diff — snapshot-based change tracking for memory sources. - // Gated on `memory-git`: with the feature off the diff controllers compile - // out (the stub returns none), so registering the capability unconditionally - // would leave a stale `memory_diff` namespace with no controllers behind it. - #[cfg(feature = "memory-git")] + // Memory diff — snapshot-based change tracking for memory sources push_cap( &mut controllers, DomainGroup::Memory, diff --git a/src/openhuman/memory/diff/mod.rs b/src/openhuman/memory/diff/mod.rs index fecf0954af..30e8357766 100644 --- a/src/openhuman/memory/diff/mod.rs +++ b/src/openhuman/memory/diff/mod.rs @@ -76,5 +76,4 @@ pub use tinycortex::memory::diff::types::{ ChangeKind, Checkpoint, CrossSourceDiff, DiffResult, DiffSummary, ItemChange, Snapshot, SnapshotTrigger, }; -#[cfg(feature = "memory-git")] pub use tools::MemoryDiffTool; diff --git a/src/openhuman/memory/diff/stub.rs b/src/openhuman/memory/diff/stub.rs index 0d76947233..ed0e1428e2 100644 --- a/src/openhuman/memory/diff/stub.rs +++ b/src/openhuman/memory/diff/stub.rs @@ -26,7 +26,7 @@ use crate::openhuman::config::Config; use crate::openhuman::memory::sources::types::MemorySourceEntry; -use tinycortex::memory::diff::types::{Checkpoint, CrossSourceDiff, Snapshot}; +use super::types::{Checkpoint, CrossSourceDiff, Snapshot}; /// The message every disabled entry point returns. /// From 47a6fd517320d7c036166ea576a81151b30514f2 Mon Sep 17 00:00:00 2001 From: Shanu Date: Tue, 11 Aug 2026 00:58:30 +0530 Subject: [PATCH 08/18] fix(build): gate the memory_diff capability consistently for the slim build With `memory-git` off the diff tool + controllers compile out, but several sites still advertised the `diff` capability, so the gates-off and rss-bench builds did not even compile / were inconsistent (an earlier one-sided gate made it worse: the table claimed diff gated something the registry did not). Gate all four sites in lockstep, all on `memory-git` (default ON, so the shipped build is byte-identical): - memory/diff/mod.rs: gate the `MemoryDiffTool` re-export (its `tools` module is already `#[cfg(feature = "memory-git")]`) and fix the stub's type import to `tinycortex::memory::diff::types` so the slim build compiles. - core/all.rs: gate the `memory_diff` capability + controller registration. - tools/ops.rs: gate the `"memory_diff" => Capability::Diff` table arm so the capability table and the live registry agree (arm falls through to the existing `_ => None`). --- src/core/all.rs | 7 ++++++- src/openhuman/memory/diff/mod.rs | 1 + src/openhuman/memory/diff/stub.rs | 2 +- src/openhuman/tools/ops.rs | 4 ++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/core/all.rs b/src/core/all.rs index 5e6029b0a6..1b78091152 100644 --- a/src/core/all.rs +++ b/src/core/all.rs @@ -866,7 +866,12 @@ fn build_registered_controllers() -> Vec { Some(Capability::Sources), crate::openhuman::memory::sources::all_memory_sources_registered_controllers(), ); - // Memory diff — snapshot-based change tracking for memory sources + // Memory diff — snapshot-based change tracking for memory sources. + // Gated on `memory-git`: with the feature off the diff controllers compile + // out (the stub returns none), so registering the capability + namespace + // would leave a stale `memory_diff` entry with no controllers behind it. + // The capability table in `tools/ops::capability_for` is gated in lockstep. + #[cfg(feature = "memory-git")] push_cap( &mut controllers, DomainGroup::Memory, diff --git a/src/openhuman/memory/diff/mod.rs b/src/openhuman/memory/diff/mod.rs index 30e8357766..fecf0954af 100644 --- a/src/openhuman/memory/diff/mod.rs +++ b/src/openhuman/memory/diff/mod.rs @@ -76,4 +76,5 @@ pub use tinycortex::memory::diff::types::{ ChangeKind, Checkpoint, CrossSourceDiff, DiffResult, DiffSummary, ItemChange, Snapshot, SnapshotTrigger, }; +#[cfg(feature = "memory-git")] pub use tools::MemoryDiffTool; diff --git a/src/openhuman/memory/diff/stub.rs b/src/openhuman/memory/diff/stub.rs index ed0e1428e2..0d76947233 100644 --- a/src/openhuman/memory/diff/stub.rs +++ b/src/openhuman/memory/diff/stub.rs @@ -26,7 +26,7 @@ use crate::openhuman::config::Config; use crate::openhuman::memory::sources::types::MemorySourceEntry; -use super::types::{Checkpoint, CrossSourceDiff, Snapshot}; +use tinycortex::memory::diff::types::{Checkpoint, CrossSourceDiff, Snapshot}; /// The message every disabled entry point returns. /// diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index 8ca4d0c003..cb9f5a6b60 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -1602,6 +1602,10 @@ fn tool_capability(name: &str) -> Option Capability::Entities, + // Gated on `memory-git` in lockstep with the capability registration in + // `core/all.rs`: with the feature off the diff tool + controllers compile + // out, so the table must not claim the `diff` capability gates anything. + #[cfg(feature = "memory-git")] "memory_diff" => Capability::Diff, "memory_doctor" => Capability::Maintenance, "tool_stats" => Capability::ToolMemory, From 2e2769230e1c15a4a8b1dccce8235cbff7485f9e Mon Sep 17 00:00:00 2001 From: Shanu Date: Tue, 11 Aug 2026 01:26:11 +0530 Subject: [PATCH 09/18] fix(build): keep the memory-diff slim compile fix, drop the capability gating Retain the minimal fix that lets the slim / rss-bench build compile (gate the MemoryDiffTool re-export whose module is already feature-gated, and point the stub's types at tinycortex::memory::diff::types). Revert the core/all.rs + tools/ops.rs capability gating: making the gates-off capability surface consistent needs a coordinated change across the registry, the capability_for table, and the representative-tool table (the guard tests are interlocked), which belongs in a dedicated cleanup rather than this voice PR. --- src/core/all.rs | 7 +------ src/openhuman/tools/ops.rs | 4 ---- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/core/all.rs b/src/core/all.rs index 1b78091152..5e6029b0a6 100644 --- a/src/core/all.rs +++ b/src/core/all.rs @@ -866,12 +866,7 @@ fn build_registered_controllers() -> Vec { Some(Capability::Sources), crate::openhuman::memory::sources::all_memory_sources_registered_controllers(), ); - // Memory diff — snapshot-based change tracking for memory sources. - // Gated on `memory-git`: with the feature off the diff controllers compile - // out (the stub returns none), so registering the capability + namespace - // would leave a stale `memory_diff` entry with no controllers behind it. - // The capability table in `tools/ops::capability_for` is gated in lockstep. - #[cfg(feature = "memory-git")] + // Memory diff — snapshot-based change tracking for memory sources push_cap( &mut controllers, DomainGroup::Memory, diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index cb9f5a6b60..8ca4d0c003 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -1602,10 +1602,6 @@ fn tool_capability(name: &str) -> Option Capability::Entities, - // Gated on `memory-git` in lockstep with the capability registration in - // `core/all.rs`: with the feature off the diff tool + controllers compile - // out, so the table must not claim the `diff` capability gates anything. - #[cfg(feature = "memory-git")] "memory_diff" => Capability::Diff, "memory_doctor" => Capability::Maintenance, "tool_stats" => Capability::ToolMemory, From 1f2aad0b3b68bcc97a8c8dbb4049a5bafff104de Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 12 Aug 2026 12:09:37 +0530 Subject: [PATCH 10/18] refactor(voice): drop always-true realtime gate; log aborted voice turn Address the two CodeRabbit nitpicks on the realtime voice path: - HumanPage: remove the hard-coded realtimeEnabled=true constant and its dead conditional, render RealtimeVoiceControls directly, and fix the stale overlay comment that still referenced a flag/mode gate. - realtime_harness: log a warn! with the correlation id in the aborted-sender arm (task panicked or aborted) so a silent voice turn is traceable. --- app/src/features/human/HumanPage.tsx | 15 +++++---------- src/openhuman/voice/realtime_harness.rs | 7 ++++++- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/app/src/features/human/HumanPage.tsx b/app/src/features/human/HumanPage.tsx index 0e2d4099eb..35e318aa2b 100644 --- a/app/src/features/human/HumanPage.tsx +++ b/app/src/features/human/HumanPage.tsx @@ -33,8 +33,6 @@ const HumanPage = () => { const speakReplies = useAppSelector(selectSpeakReplies); const { face, visemeCode } = useHumanMascot({ speakReplies }); - // Realtime voice controls are always shown — no settings/flag gate. - const realtimeEnabled = true; const mascotColor = useAppSelector(selectMascotColor); const customPrimary = useAppSelector(selectCustomPrimaryColor); const customSecondary = useAppSelector(selectCustomSecondaryColor); @@ -99,14 +97,11 @@ const HumanPage = () => { - {/* Realtime voice-chat controls (#5399) — additive overlay shown only when - the flag + realtime mode are on; the classic push-to-talk path below - is untouched. */} - {realtimeEnabled && ( -
- -
- )} + {/* Realtime voice-chat controls (#5399) — always shown; the classic + push-to-talk path below is untouched. */} +
+ +