Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
2748458
feat(voice): realtime ElevenLabs voice agent alongside the classic path
YellowSnnowmann Aug 10, 2026
4469d52
fix(voice): address review on the deferred-turn path
YellowSnnowmann Aug 10, 2026
4c98670
Merge remote-tracking branch 'upstream/main' into feat/5489-merge
YellowSnnowmann Aug 10, 2026
fdb3380
chore(build): fix pre-existing fmt + gates-off breakage on the base
YellowSnnowmann Aug 10, 2026
41c6f4f
fix(voice): surface a chat message on empty deferred reply + correct …
YellowSnnowmann Aug 10, 2026
2988802
test(voice): cover the speak-back path in useRealtimeVoiceSession
YellowSnnowmann Aug 10, 2026
2fe7e2e
fix(build): gate the memory_diff capability registration on memory-git
YellowSnnowmann Aug 10, 2026
67c6345
fix(voice): update realtime-mode tests; revert gates-off scope-creep
YellowSnnowmann Aug 10, 2026
47a6fd5
fix(build): gate the memory_diff capability consistently for the slim…
YellowSnnowmann Aug 10, 2026
2e27692
fix(build): keep the memory-diff slim compile fix, drop the capabilit…
YellowSnnowmann Aug 10, 2026
1f2aad0
refactor(voice): drop always-true realtime gate; log aborted voice turn
YellowSnnowmann Aug 12, 2026
ea6c3d7
Merge upstream/main into feat/5399-voice-elevenlabs-realtime
YellowSnnowmann Aug 12, 2026
5ef91d6
build(tauri): resync app/src-tauri/Cargo.lock with the current manifests
YellowSnnowmann Aug 12, 2026
5405a1e
docs(voice): correct deliver_voice_failure_to_chat doc for the empty-…
YellowSnnowmann Aug 12, 2026
acd3bc0
fix(voice): never end a realtime turn on nothing, and stop the read-b…
YellowSnnowmann Aug 13, 2026
c152be9
feat(voice): put the Human tab's voice control behind a build flag
YellowSnnowmann Aug 13, 2026
c3d69da
fix(voice): float the realtime control in comparison mode
YellowSnnowmann Aug 13, 2026
db1d938
Merge remote-tracking branch 'upstream/main' into feat/5399-voice-ele…
YellowSnnowmann Aug 13, 2026
b3ed62c
fix(voice): deliver the promised chat notice when a voice turn dies
YellowSnnowmann Aug 13, 2026
9928d9e
chore: apply formatting
YellowSnnowmann Aug 13, 2026
630060f
fix(build): restore upstream's app/src-tauri Cargo.lock
YellowSnnowmann Aug 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions app/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,16 @@ VITE_DEV_FORCE_ONBOARDING=false
# Set to false to hide the toggle for a build (kill-switch).
# VITE_VOICE_MODE=false

# [optional] Human tab voice entry point (#5399). Enabled by default: the chat
# card shows the realtime "Start voice chat" control in the slot the classic
# push-to-talk mic used to occupy. Set to false to fall back to tap-and-speak.
# VITE_HUMAN_VOICE_REALTIME=false

# [optional] Show BOTH Human-tab voice controls (realtime + tap-and-speak),
# stacked. Off by default — for comparing the two paths, not for shipping.
# Takes precedence over VITE_HUMAN_VOICE_REALTIME.
# VITE_HUMAN_VOICE_SHOW_BOTH=true

# [optional] Client-side timeout for skill callTool/triggerSync (seconds; default 120, max 3600).
# Should match OPENHUMAN_TOOL_TIMEOUT_SECS on the core when set.
# VITE_TOOL_TIMEOUT_SECS=
Expand Down
26 changes: 2 additions & 24 deletions app/src/components/settings/panels/VoicePanel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,6 @@ import {
type VoiceProviderView,
type VoiceSettings,
} from '../../../services/api/voiceSettingsApi';
import { useAppDispatch, useAppSelector } from '../../../store/hooks';
import { selectVoiceMode, setVoiceMode } from '../../../store/mascotSlice';
import { VOICE_MODE_FLAG_ENABLED } from '../../../utils/config';
import {
openhumanGetVoiceServerSettings,
openhumanUpdateVoiceServerSettings,
Expand Down Expand Up @@ -102,8 +99,6 @@ interface VoicePanelProps {

const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => {
const { t } = useT();
const dispatch = useAppDispatch();
const voiceMode = useAppSelector(selectVoiceMode);
const { navigateBack, navigateToSettings } = useSettingsNavigation();
const [settings, setSettings] = useState<VoiceServerSettings | null>(null);
const [savedSettings, setSavedSettings] = useState<VoiceServerSettings | null>(null);
Expand Down Expand Up @@ -582,25 +577,8 @@ const VoicePanel = ({ embedded = false }: VoicePanelProps = {}) => {
/>
</SettingsSection>

{/* ─── Realtime voice mode (beta, flag-gated) ──────────────────── */}
{VOICE_MODE_FLAG_ENABLED && (
<SettingsSection title={t('voice.mode.title')} description={t('voice.mode.desc')}>
<SettingsRow
htmlFor="voice-mode-realtime"
label={t('voice.mode.realtime')}
description={t('voice.mode.realtimeDesc')}
control={
<SettingsSwitch
id="voice-mode-realtime"
data-testid="voice-mode-realtime-toggle"
checked={voiceMode === 'realtime'}
onCheckedChange={next => dispatch(setVoiceMode(next ? 'realtime' : 'classic'))}
aria-label={t('voice.mode.realtime')}
/>
}
/>
</SettingsSection>
)}
{/* Realtime voice is always on now — its controls live on the Human tab,
so the former flag-gated toggle here was removed. */}

{/* ─── Section 1: Voice Provider Chips ─────────────────────────── */}
{/* Provider chips are intentional bespoke UI — kept as-is. */}
Expand Down

This file was deleted.

40 changes: 29 additions & 11 deletions app/src/features/conversations/Conversations.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { convertFileSrc } from '@tauri-apps/api/core';
import debugFactory from 'debug';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { type ReactNode, useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useLocation, useNavigate, useParams } from 'react-router-dom';

import { type ChatSendError, chatSendError } from '../../chat/chatSendError';
Expand Down Expand Up @@ -150,6 +150,19 @@ interface ConversationsProps {
* Used by the mascot tab so the only interaction is voice.
*/
composer?: 'text' | 'mic-cloud';
/**
* Voice-chat control rendered in the `mic-cloud` composer slot, above the mic
* button. Passed in as a node rather than imported here so this component
* keeps no dependency on the realtime voice stack (and the ElevenLabs SDK
* stays out of every consumer's module graph). Ignored outside `mic-cloud`.
*/
voiceChatControl?: ReactNode;
/**
* Whether the `mic-cloud` slot renders the push-to-talk mic composer. Default
* `true` — set `false` alongside {@link ConversationsProps.voiceChatControl}
* to replace tap-and-speak with the realtime control rather than stack them.
*/
showMicComposer?: boolean;
/**
* Project the thread list into the root sidebar's dynamic region even in the
* `sidebar` variant. Page variant always projects it; this lets an embedded
Expand Down Expand Up @@ -248,6 +261,8 @@ export function deriveChatErrorBanner(
const Conversations = ({
variant = 'page',
composer: composerProp = 'text',
voiceChatControl = null,
showMicComposer = true,
projectThreadList = false,
}: ConversationsProps = {}) => {
const [composerOverride, setComposerOverride] = useState<'mic-cloud' | 'text' | null>(null);
Expand Down Expand Up @@ -2314,16 +2329,19 @@ const Conversations = ({
// — this branch renders no ChatComposer to hang it off.
<div className="relative flex flex-col items-center gap-3 py-1">
{mascotDock}
<MicComposer
// Without `!selectedThreadId`, a mic submit before a thread is
// ready hits `handleSendMessage`'s early return and the
// transcript is silently dropped — the user spoke into the void.
disabled={composerInteractionBlocked || isSending || !selectedThreadId}
onSubmit={text => handleSendMessage(text)}
onError={message => setSendError(chatSendError('voice_transcription', message))}
showDeviceSelector
onSwitchToText={() => setComposerOverride('text')}
/>
{voiceChatControl}
{showMicComposer && (
<MicComposer
// Without `!selectedThreadId`, a mic submit before a thread is
// ready hits `handleSendMessage`'s early return and the
// transcript is silently dropped — the user spoke into the void.
disabled={composerInteractionBlocked || isSending || !selectedThreadId}
onSubmit={text => handleSendMessage(text)}
onError={message => setSendError(chatSendError('voice_transcription', message))}
showDeviceSelector
onSwitchToText={() => setComposerOverride('text')}
/>
)}
</div>
) : inputMode === 'text' ? (
<>
Expand Down
97 changes: 75 additions & 22 deletions app/src/features/human/HumanPage.realtimeMode.test.tsx
Original file line number Diff line number Diff line change
@@ -1,35 +1,59 @@
/**
* Unit tests for HumanPage's realtime voice overlay gate (#5399). The overlay
* renders only when the build flag is on AND the persisted mascot voice mode is
* `realtime`; the classic push-to-talk path is always present. Config is mocked
* with the flag ON here (the global setup mock ships it OFF), and
* RealtimeVoiceControls is stubbed so the ElevenLabs SDK never loads.
* Unit test for the Human tab's voice entry point (#5399). The realtime
* "Start voice chat" control now lives in the chat card's composer slot — the
* one the classic push-to-talk mic used to own — and which of the two renders is
* decided by two build flags. This pins the wiring from those flags through to
* the props HumanPage hands Conversations; the controls themselves and the
* precedence rule are covered separately (RealtimeVoiceControls.test.tsx,
* voiceEntry.test.ts). RealtimeVoiceControls is stubbed so the ElevenLabs SDK
* never loads.
*/
import { configureStore } from '@reduxjs/toolkit';
import { render, screen } from '@testing-library/react';
import { Provider } from 'react-redux';
import { beforeEach, describe, expect, it, vi } from 'vitest';

import chatRuntimeReducer from '../../store/chatRuntimeSlice';
import mascotReducer, { setVoiceMode } from '../../store/mascotSlice';
import mascotReducer from '../../store/mascotSlice';
import threadReducer from '../../store/threadSlice';
import HumanPage from './HumanPage';

// Flip the realtime gate ON for this file (global setup ships it OFF). Spread
// the real module so every other config export keeps its production value.
vi.mock('../../utils/config', async () => {
const actual = await vi.importActual<typeof import('../../utils/config')>('../../utils/config');
return { ...actual, VOICE_MODE_FLAG_ENABLED: true };
const flags = { realtimeEnabled: true, showBoth: false };

// The global test setup mocks the whole config module, so override just the two
// flags this file drives — read through getters so a test can flip them between
// renders without re-importing the module.
vi.mock('../../utils/config', async importOriginal => {
const actual = await importOriginal<Record<string, unknown>>();
return {
...actual,
get HUMAN_VOICE_REALTIME_ENABLED() {
return flags.realtimeEnabled;
},
get HUMAN_VOICE_SHOW_BOTH() {
return flags.showBoth;
},
};
});

// Stub the overlay so the ElevenLabs `ConversationProvider`/SDK never mounts —
// this test only pins the render gate, not the controls (covered separately).
vi.mock('./RealtimeVoiceControls', () => ({
default: () => <div data-testid="realtime-voice-controls-stub" />,
}));

// Render the slot props so the test observes what the card would actually show,
// rather than asserting on prop identity.
vi.mock('../conversations/Conversations', () => ({
default: () => <div data-testid="conversations-stub" />,
default: ({
voiceChatControl,
showMicComposer,
}: {
voiceChatControl?: React.ReactNode;
showMicComposer?: boolean;
}) => (
<div data-testid="conversations-stub">
{voiceChatControl}
{showMicComposer && <div data-testid="mic-composer-stub" />}
</div>
),
}));

vi.mock('./Mascot', async importOriginal => {
Expand All @@ -46,30 +70,59 @@ vi.mock('./Mascot/manifest/useMascotManifest', () => ({
useMascotManifest: () => ({ manifest: null, entry: null, loading: false, error: null }),
}));

function renderWithVoiceMode(mode: 'classic' | 'realtime') {
async function renderPage() {
const { default: HumanPage } = await import('./HumanPage');
const store = configureStore({
reducer: { mascot: mascotReducer, thread: threadReducer, chatRuntime: chatRuntimeReducer },
});
store.dispatch(setVoiceMode(mode));
return render(
<Provider store={store}>
<HumanPage />
</Provider>
);
}

describe('HumanPage — realtime voice overlay gate', () => {
describe('HumanPage — voice entry point', () => {
beforeEach(() => {
localStorage.clear();
flags.realtimeEnabled = true;
flags.showBoth = false;
});

it('renders the realtime controls when voice mode is realtime and the flag is on', () => {
renderWithVoiceMode('realtime');
it('shows the realtime control in place of the mic composer by default', async () => {
await renderPage();
expect(screen.getByTestId('realtime-voice-controls-stub')).toBeInTheDocument();
expect(screen.queryByTestId('mic-composer-stub')).not.toBeInTheDocument();
});

it('hides the realtime controls when voice mode is classic', () => {
renderWithVoiceMode('classic');
it('falls back to tap-and-speak when the realtime flag is off', async () => {
flags.realtimeEnabled = false;
await renderPage();
expect(screen.getByTestId('mic-composer-stub')).toBeInTheDocument();
expect(screen.queryByTestId('realtime-voice-controls-stub')).not.toBeInTheDocument();
});

// Comparison mode keeps the two paths apart: the realtime control floats over
// the mascot stage (outside the card), tap-and-speak stays in the card.
it('shows both controls when the show-both flag is on, and not stacked', async () => {
flags.showBoth = true;
await renderPage();
expect(screen.getByTestId('realtime-voice-controls-stub')).toBeInTheDocument();
expect(screen.getByTestId('mic-composer-stub')).toBeInTheDocument();
// The card's slot stays empty — the realtime control is rendered outside it.
expect(
screen.getByTestId('conversations-stub').querySelector('[data-testid$="voice-controls-stub"]')
).toBeNull();
});

// Whichever mode is on, exactly one realtime control exists: the single-control
// modes put it in the card, comparison mode floats it — never both at once.
it.each([
['realtime', { realtimeEnabled: true, showBoth: false }],
['both', { realtimeEnabled: true, showBoth: true }],
])('renders the realtime control exactly once in %s mode', async (_label, next) => {
Object.assign(flags, next);
await renderPage();
expect(screen.getAllByTestId('realtime-voice-controls-stub')).toHaveLength(1);
});
});
Loading
Loading