Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
2748458
feat(voice): realtime ElevenLabs voice agent alongside the classic path
YellowSnnowmann Aug 10, 2026
4469d52
fix(voice): address review on the deferred-turn path
YellowSnnowmann Aug 10, 2026
4c98670
Merge remote-tracking branch 'upstream/main' into feat/5489-merge
YellowSnnowmann Aug 10, 2026
fdb3380
chore(build): fix pre-existing fmt + gates-off breakage on the base
YellowSnnowmann Aug 10, 2026
41c6f4f
fix(voice): surface a chat message on empty deferred reply + correct …
YellowSnnowmann Aug 10, 2026
2988802
test(voice): cover the speak-back path in useRealtimeVoiceSession
YellowSnnowmann Aug 10, 2026
2fe7e2e
fix(build): gate the memory_diff capability registration on memory-git
YellowSnnowmann Aug 10, 2026
67c6345
fix(voice): update realtime-mode tests; revert gates-off scope-creep
YellowSnnowmann Aug 10, 2026
47a6fd5
fix(build): gate the memory_diff capability consistently for the slim…
YellowSnnowmann Aug 10, 2026
2e27692
fix(build): keep the memory-diff slim compile fix, drop the capabilit…
YellowSnnowmann Aug 10, 2026
1f2aad0
refactor(voice): drop always-true realtime gate; log aborted voice turn
YellowSnnowmann Aug 12, 2026
ea6c3d7
Merge upstream/main into feat/5399-voice-elevenlabs-realtime
YellowSnnowmann Aug 12, 2026
5ef91d6
build(tauri): resync app/src-tauri/Cargo.lock with the current manifests
YellowSnnowmann Aug 12, 2026
5405a1e
docs(voice): correct deliver_voice_failure_to_chat doc for the empty-…
YellowSnnowmann Aug 12, 2026
acd3bc0
fix(voice): never end a realtime turn on nothing, and stop the read-b…
YellowSnnowmann Aug 13, 2026
c152be9
feat(voice): put the Human tab's voice control behind a build flag
YellowSnnowmann Aug 13, 2026
c3d69da
fix(voice): float the realtime control in comparison mode
YellowSnnowmann Aug 13, 2026
db1d938
Merge remote-tracking branch 'upstream/main' into feat/5399-voice-ele…
YellowSnnowmann Aug 13, 2026
b3ed62c
fix(voice): deliver the promised chat notice when a voice turn dies
YellowSnnowmann Aug 13, 2026
9928d9e
chore: apply formatting
YellowSnnowmann Aug 13, 2026
630060f
fix(build): restore upstream's app/src-tauri Cargo.lock
YellowSnnowmann Aug 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 2 additions & 24 deletions app/src/components/settings/panels/VoicePanel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,6 @@ import {
type VoiceProviderView,
type VoiceSettings,
} from '../../../services/api/voiceSettingsApi';
import { useAppDispatch, useAppSelector } from '../../../store/hooks';
import { selectVoiceMode, setVoiceMode } from '../../../store/mascotSlice';
import { VOICE_MODE_FLAG_ENABLED } from '../../../utils/config';
import {
openhumanGetVoiceServerSettings,
openhumanUpdateVoiceServerSettings,
Expand Down Expand Up @@ -102,8 +99,6 @@ interface VoicePanelProps {

const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => {
const { t } = useT();
const dispatch = useAppDispatch();
const voiceMode = useAppSelector(selectVoiceMode);
const { navigateBack, navigateToSettings } = useSettingsNavigation();
const [settings, setSettings] = useState<VoiceServerSettings | null>(null);
const [savedSettings, setSavedSettings] = useState<VoiceServerSettings | null>(null);
Expand Down Expand Up @@ -582,25 +577,8 @@ const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => {
/>
</SettingsSection>

{/* ─── Realtime voice mode (beta, flag-gated) ──────────────────── */}
{VOICE_MODE_FLAG_ENABLED && (
<SettingsSection title={t('voice.mode.title')} description={t('voice.mode.desc')}>
<SettingsRow
htmlFor="voice-mode-realtime"
label={t('voice.mode.realtime')}
description={t('voice.mode.realtimeDesc')}
control={
<SettingsSwitch
id="voice-mode-realtime"
data-testid="voice-mode-realtime-toggle"
checked={voiceMode === 'realtime'}
onCheckedChange={next => dispatch(setVoiceMode(next ? 'realtime' : 'classic'))}
aria-label={t('voice.mode.realtime')}
/>
}
/>
</SettingsSection>
)}
{/* Realtime voice is always on now — its controls live on the Human tab,
so the former flag-gated toggle here was removed. */}

{/* ─── Section 1: Voice Provider Chips ─────────────────────────── */}
{/* Provider chips are intentional bespoke UI — kept as-is. */}
Expand Down
6 changes: 2 additions & 4 deletions app/src/features/human/HumanPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,8 @@ import {
selectCustomSecondaryColor,
selectMascotColor,
selectSpeakReplies,
selectVoiceMode,
setSpeakReplies,
} from '../../store/mascotSlice';
import { VOICE_MODE_FLAG_ENABLED } from '../../utils/config';
import Conversations from '../conversations/Conversations';
import {
CustomGifMascot,
Expand All @@ -35,8 +33,8 @@ const HumanPage = () => {
const speakReplies = useAppSelector(selectSpeakReplies);

const { face, visemeCode } = useHumanMascot({ speakReplies });
const voiceMode = useAppSelector(selectVoiceMode);
const realtimeEnabled = VOICE_MODE_FLAG_ENABLED && voiceMode === 'realtime';
// Realtime voice controls are always shown — no settings/flag gate.
const realtimeEnabled = true;
const mascotColor = useAppSelector(selectMascotColor);
const customPrimary = useAppSelector(selectCustomPrimaryColor);
const customSecondary = useAppSelector(selectCustomSecondaryColor);
Expand Down
30 changes: 30 additions & 0 deletions app/src/features/human/voice/useRealtimeVoiceSession.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,21 @@ import createDebug from 'debug';
import { useCallback, useEffect, useRef, useState } from 'react';

import { fetchVoiceAgentSignedUrl } from '../../../services/api/voiceAgentApi';
import { socketService } from '../../../services/socketService';
import { MASCOT_VOICE_ID } from '../../../utils/config';

const log = createDebug('app:human:realtime-voice');

/**
* Instruction prefix for "speak-back". A slow voice turn (e.g. an email summary)
* is acknowledged aloud, finishes in the background, and its result is delivered
* to chat AND pushed here as a `voice_speak` event. We send it back into the live
* ElevenLabs session as a user message wrapped with this prefix so the agent reads
* it verbatim. MUST match `VOICE_READBACK_PREFIX` in `voice/realtime_harness.rs`,
* which uses it to avoid re-arming speak-back on the read-back turn (loop guard).
*/
const READBACK_PREFIX = 'Please read the following to me, word for word, and say nothing else:';

/**
* Lifecycle of a realtime ElevenLabs Agents voice session (#5399).
* `idle → connecting → active → idle`, or `→ error`.
Expand Down Expand Up @@ -125,6 +136,25 @@ export function useRealtimeVoiceSession(opts?: { voiceId?: string }): RealtimeVo
[]
);

// Speak-back: a slow voice turn (email/calendar summary) is acknowledged aloud,
// finishes in the background, and the core emits its result as a `voice_speak`
// event. While the call is still open, read it aloud by sending it back into the
// live ElevenLabs session wrapped in the verbatim prefix (a fast read-back turn).
// The result also lands in chat regardless (delivered core-side) — this is the
// spoken copy. Refs keep the subscription set up once while always seeing the
// live conversation and liveness.
useEffect(() => {
const handler = (payload: unknown) => {
if (!liveRef.current) return; // call already ended — the chat copy stands alone
const text = (payload as { full_response?: string } | undefined)?.full_response?.trim();
if (!text) return;
log('speak-back: reading deferred result aloud (%d chars)', text.length);
conversationRef.current.sendUserMessage(`${READBACK_PREFIX}\n\n${text}`);
};
socketService.on('voice_speak', handler);
return () => socketService.off('voice_speak', handler);
}, []);

return {
state,
isSpeaking: conversation.isSpeaking,
Expand Down
10 changes: 10 additions & 0 deletions src/openhuman/agent/harness/session/runtime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,16 @@ impl Agent {
&self.model_name
}

/// Override the base model this session runs its top-level turns on. Set
/// once before running: per-turn classification is disabled (the main agent
/// is pinned to its configured model for KV-cache stability — see the model
/// pin in `turn/core.rs`), so this sticks for the session and is not flipped
/// mid-conversation. The realtime voice harness uses it to pin a fast,
/// non-thinking model within the provider's response-time ceiling.
pub fn set_model_name(&mut self, model_name: impl Into<String>) {
self.model_name = model_name.into();
}

/// The agent's currently-configured temperature.
pub fn temperature(&self) -> f64 {
self.temperature
Expand Down
10 changes: 10 additions & 0 deletions src/openhuman/agent/harness/session/turn/core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -923,6 +923,16 @@ impl Agent {

let mut agent_context_prepared_sources: Vec<harness::AgentContextPreparedSource> =
Vec::new();
// Triggered memory-agent recall runs on EVERY channel, voice included:
// dropping it on voice would strip the user's remembered context
// (preferences, people, prior facts) from spoken answers — a real quality
// loss the transcript alone can't replace. Recall adds a few seconds of
// embedding + retrieval before the first model token, but on realtime
// voice that latency is already covered end-to-end: the backend relay
// streams an audible keepalive filler from t=0 so the cloud session never
// sees a silent stall, and the desktop's ~8s ack-defer closes the spoken
// turn and finishes in the background if the work runs long. So the recall
// path is byte-for-byte identical across voice and chat.
let (enriched, memory_agent_context_injected) = self
.inject_triggered_memory_agent_context(user_message, enriched, &parent_context)
.await;
Expand Down
1 change: 1 addition & 0 deletions src/openhuman/memory/diff/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -76,4 +76,5 @@ pub use tinycortex::memory::diff::types::{
ChangeKind, Checkpoint, CrossSourceDiff, DiffResult, DiffSummary, ItemChange, Snapshot,
SnapshotTrigger,
};
#[cfg(feature = "memory-git")]
pub use tools::MemoryDiffTool;
2 changes: 1 addition & 1 deletion src/openhuman/memory/diff/stub.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
use crate::openhuman::config::Config;
use crate::openhuman::memory::sources::types::MemorySourceEntry;

use super::types::{Checkpoint, CrossSourceDiff, Snapshot};
use tinycortex::memory::diff::types::{Checkpoint, CrossSourceDiff, Snapshot};

/// The message every disabled entry point returns.
///
Expand Down
Loading
Loading