From 4baa52951d3d0f4a430c2d87b603e4282d05a22f Mon Sep 17 00:00:00 2001 From: Michael Yagudaev Date: Thu, 23 Apr 2026 21:19:45 -0700 Subject: [PATCH] Add real-time brain mode (NAN-656) --- desktop/src/main/ipc-handlers.ts | 21 +- .../src/renderer/src/lib/realtime-settings.ts | 38 ++ desktop/src/renderer/src/lib/use-realtime.ts | 22 +- desktop/src/renderer/src/pages/ChatPage.tsx | 152 +++---- .../src/renderer/src/pages/SettingsPage.tsx | 374 +++++++++++------- 5 files changed, 385 insertions(+), 222 deletions(-) create mode 100644 desktop/src/renderer/src/lib/realtime-settings.ts diff --git a/desktop/src/main/ipc-handlers.ts b/desktop/src/main/ipc-handlers.ts index 9dcdfc47..6f3f1e38 100644 --- a/desktop/src/main/ipc-handlers.ts +++ b/desktop/src/main/ipc-handlers.ts @@ -35,7 +35,7 @@ export function registerIpcHandlers() { (SELECT content FROM messages WHERE conversation_id = c.id ORDER BY created_at ASC LIMIT 1) as preview, (SELECT COUNT(*) FROM messages WHERE conversation_id = c.id) as message_count FROM conversations c - ORDER BY c.updated_at DESC`, + ORDER BY c.updated_at DESC` ) .all() }) @@ -57,7 +57,7 @@ export function registerIpcHandlers() { db.prepare('UPDATE conversations SET title = ?, updated_at = ? WHERE id = ?').run( title, Date.now(), - id, + id ) }) @@ -76,14 +76,14 @@ export function registerIpcHandlers() { conversationId: number, role: string, content: string, - latency?: { sttLatencyMs?: number, llmLatencyMs?: number, ttsLatencyMs?: number }, - providers?: { sttProvider?: string, llmProvider?: string, ttsProvider?: string }, + latency?: { sttLatencyMs?: number; llmLatencyMs?: number; ttsLatencyMs?: number }, + providers?: { sttProvider?: string; llmProvider?: string; ttsProvider?: string } ) => { const db = getDb() const now = Date.now() const result = db .prepare( - 'INSERT INTO messages (conversation_id, role, content, created_at, stt_latency_ms, llm_latency_ms, tts_latency_ms, stt_provider, llm_provider, tts_provider) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + 'INSERT INTO messages (conversation_id, role, content, created_at, stt_latency_ms, llm_latency_ms, tts_latency_ms, stt_provider, llm_provider, tts_provider) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)' ) .run( conversationId, @@ -95,7 +95,7 @@ export function registerIpcHandlers() { latency?.ttsLatencyMs ?? null, providers?.sttProvider ?? null, providers?.llmProvider ?? null, - providers?.ttsProvider ?? null, + providers?.ttsProvider ?? null ) db.prepare('UPDATE conversations SET updated_at = ? WHERE id = ?').run(now, conversationId) return { @@ -111,7 +111,7 @@ export function registerIpcHandlers() { llm_provider: providers?.llmProvider ?? null, tts_provider: providers?.ttsProvider ?? null, } - }, + } ) ipcMain.handle('db:getMessages', (_e, conversationId: number) => { @@ -133,13 +133,13 @@ export function registerIpcHandlers() { ipcMain.handle('db:setSetting', (_e, key: string, value: string) => { const db = getDb() db.prepare( - 'INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?', + 'INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?' ).run(key, value, value) }) ipcMain.handle('db:getAllSettings', () => { const db = getDb() - const rows = db.prepare('SELECT * FROM settings').all() as { key: string, value: string }[] + const rows = db.prepare('SELECT * FROM settings').all() as { key: string; value: string }[] return Object.fromEntries(rows.map((r) => [r.key, r.value])) }) @@ -172,8 +172,7 @@ function toHealthUrl(url: string): string | null { if (parsed.protocol === 'ws:') parsed.protocol = 'http:' else if (parsed.protocol === 'wss:') parsed.protocol = 'https:' else return null - // Strip /ws path and append /health - parsed.pathname = parsed.pathname.replace(/\/ws\/?$/, '') + parsed.pathname = parsed.pathname.replace(/\/(?:ws|voiceclaw\/realtime)\/?$/, '') parsed.pathname = parsed.pathname.replace(/\/$/, '') + '/health' return parsed.toString() } catch { diff --git a/desktop/src/renderer/src/lib/realtime-settings.ts b/desktop/src/renderer/src/lib/realtime-settings.ts new file mode 100644 index 00000000..c547882e --- /dev/null +++ b/desktop/src/renderer/src/lib/realtime-settings.ts @@ -0,0 +1,38 @@ +export type RealtimeConnectionMode = 'relay' | 'realtime-brain' + +export const RELAY_DEFAULT_SERVER_URL = 'ws://localhost:8080/ws' +export const REALTIME_BRAIN_DEFAULT_SERVER_URL = 'ws://localhost:19789/voiceclaw/realtime' +export const LEGACY_REALTIME_SERVER_URL_SETTING = 'realtime_server_url' +export const REALTIME_CONNECTION_MODE_SETTING = 'realtime_connection_mode' + +const SERVER_URL_SETTING_BY_MODE: Record = { + relay: 'realtime_server_url_relay', + 'realtime-brain': 'realtime_server_url_realtime_brain', +} + +export function isRealtimeConnectionMode(value: string | null): value is RealtimeConnectionMode { + return value === 'relay' || value === 'realtime-brain' +} + +export function defaultServerUrlForMode(mode: RealtimeConnectionMode): string { + return mode === 'realtime-brain' ? REALTIME_BRAIN_DEFAULT_SERVER_URL : RELAY_DEFAULT_SERVER_URL +} + +export function serverUrlSettingKeyForMode(mode: RealtimeConnectionMode): string { + return SERVER_URL_SETTING_BY_MODE[mode] +} + +export function resolveServerUrlForMode( + mode: RealtimeConnectionMode, + savedUrl: string | null +): string { + const trimmed = savedUrl?.trim() + if (!trimmed) return defaultServerUrlForMode(mode) + if (mode === 'realtime-brain' && trimmed === RELAY_DEFAULT_SERVER_URL) { + return REALTIME_BRAIN_DEFAULT_SERVER_URL + } + if (mode === 'relay' && trimmed === REALTIME_BRAIN_DEFAULT_SERVER_URL) { + return RELAY_DEFAULT_SERVER_URL + } + return trimmed +} diff --git a/desktop/src/renderer/src/lib/use-realtime.ts b/desktop/src/renderer/src/lib/use-realtime.ts index 0a01e16f..ac37e11c 100644 --- a/desktop/src/renderer/src/lib/use-realtime.ts +++ b/desktop/src/renderer/src/lib/use-realtime.ts @@ -3,13 +3,15 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { AudioEngine } from './audio-engine' +import type { RealtimeConnectionMode } from './realtime-settings' export interface RealtimeConfig { + connectionMode?: RealtimeConnectionMode serverUrl: string voice: string model?: string brainAgent: 'enabled' | 'none' - apiKey: string + apiKey?: string sessionKey?: string volume?: number inputDeviceId?: string @@ -20,7 +22,7 @@ export interface RealtimeConfig { deviceModel?: string } instructionsOverride?: string - conversationHistory?: { role: 'user' | 'assistant', text: string }[] + conversationHistory?: { role: 'user' | 'assistant'; text: string }[] tracingEnabled?: boolean } @@ -72,7 +74,7 @@ export function useRealtime(callbacks: RealtimeCallbacks): RealtimeControls { if (!configRef.current?.tracingEnabled) return if (wsRef.current?.readyState !== WebSocket.OPEN) return wsRef.current.send( - JSON.stringify({ type: 'client.timing', phase, ms, turnId: turnId ?? undefined }), + JSON.stringify({ type: 'client.timing', phase, ms, turnId: turnId ?? undefined }) ) }, []) @@ -160,7 +162,7 @@ export function useRealtime(callbacks: RealtimeCallbacks): RealtimeControls { break } }, - [sendTiming], + [sendTiming] ) const start = useCallback( @@ -208,12 +210,12 @@ export function useRealtime(callbacks: RealtimeCallbacks): RealtimeControls { voice: config.voice, model: config.model, brainAgent: config.brainAgent, - apiKey: config.apiKey, + apiKey: config.apiKey ?? '', sessionKey: config.sessionKey, deviceContext: config.deviceContext, instructionsOverride: config.instructionsOverride, conversationHistory: config.conversationHistory, - }), + }) ) // Start mic capture — audio data flows to WebSocket @@ -243,7 +245,9 @@ export function useRealtime(callbacks: RealtimeCallbacks): RealtimeControls { // Only surface the error on the initial connection attempt. // During reconnect, onclose handles retry logic. if (!hasConnectedRef.current && reconnectAttemptsRef.current === 0) { - callbacksRef.current.onError?.('Could not connect to relay server. Is it running?', 0) + const label = + config.connectionMode === 'realtime-brain' ? 'OpenClaw real-time brain' : 'relay server' + callbacksRef.current.onError?.(`Could not connect to ${label}. Is it running?`, 0) } } @@ -263,7 +267,7 @@ export function useRealtime(callbacks: RealtimeCallbacks): RealtimeControls { reconnectAttemptsRef.current += 1 setIsReconnecting(true) console.log( - `[useRealtime] Unexpected disconnect — reconnect attempt ${reconnectAttemptsRef.current}/${MAX_RECONNECT_ATTEMPTS} in ${delay}ms`, + `[useRealtime] Unexpected disconnect — reconnect attempt ${reconnectAttemptsRef.current}/${MAX_RECONNECT_ATTEMPTS} in ${delay}ms` ) reconnectTimerRef.current = setTimeout(() => { reconnectTimerRef.current = null @@ -279,7 +283,7 @@ export function useRealtime(callbacks: RealtimeCallbacks): RealtimeControls { } } }, - [handleMessage], + [handleMessage] ) const stop = useCallback(() => { diff --git a/desktop/src/renderer/src/pages/ChatPage.tsx b/desktop/src/renderer/src/pages/ChatPage.tsx index 19266d3b..15f1e25e 100644 --- a/desktop/src/renderer/src/pages/ChatPage.tsx +++ b/desktop/src/renderer/src/pages/ChatPage.tsx @@ -8,6 +8,13 @@ import { ScreenSharePicker } from '../components/ScreenSharePicker' import { ScreenCapture, type ScreenSource } from '../lib/screen-capture' import { useRealtime, type RealtimeCallbacks } from '../lib/use-realtime' import { useConversationContext } from '../lib/conversation-context' +import { + isRealtimeConnectionMode, + LEGACY_REALTIME_SERVER_URL_SETTING, + REALTIME_CONNECTION_MODE_SETTING, + resolveServerUrlForMode, + serverUrlSettingKeyForMode, +} from '../lib/realtime-settings' import { addMessage, createConversation, @@ -20,7 +27,16 @@ import { const DEFAULT_REALTIME_MODEL = 'gemini-3.1-flash-live-preview' const REALTIME_MODELS = ['gemini-3.1-flash-live-preview', 'grok-voice-think-fast-1.0'] as const -const GEMINI_VOICES = ['Puck', 'Charon', 'Kore', 'Fenrir', 'Aoede', 'Leda', 'Orus', 'Zephyr'] as const +const GEMINI_VOICES = [ + 'Puck', + 'Charon', + 'Kore', + 'Fenrir', + 'Aoede', + 'Leda', + 'Orus', + 'Zephyr', +] as const const XAI_VOICES = ['eve', 'ara', 'rex', 'sal', 'leo'] as const export function ChatPage() { @@ -168,7 +184,7 @@ export function ChatPage() { setStreamingText('') }, onError: (message) => { - console.error('[ChatPage] Relay error:', message) + console.error('[ChatPage] Realtime error:', message) setConnectionError(message) setIsConnecting(false) setIsCallActive(false) @@ -183,7 +199,12 @@ export function ChatPage() { const startCall = useCallback(async () => { setConnectionError('') setIsConnecting(true) - const serverUrl = (await getSetting('realtime_server_url')) || 'ws://localhost:8080/ws' + const savedMode = await getSetting(REALTIME_CONNECTION_MODE_SETTING) + const connectionMode = isRealtimeConnectionMode(savedMode) ? savedMode : 'relay' + const savedUrl = await getSetting(serverUrlSettingKeyForMode(connectionMode)) + const legacyUrl = + connectionMode === 'relay' ? await getSetting(LEGACY_REALTIME_SERVER_URL_SETTING) : null + const serverUrl = resolveServerUrlForMode(connectionMode, savedUrl || legacyUrl) const model = normalizeRealtimeModel(await getSetting('realtime_model')) const voice = normalizeRealtimeVoice(model, await getSetting('realtime_voice')) const apiKey = (await getSetting('realtime_api_key')) || '' @@ -194,6 +215,7 @@ export function ChatPage() { setActiveRealtimeModel(model) realtime.start({ + connectionMode, serverUrl, voice, model, @@ -235,23 +257,26 @@ export function ChatPage() { titleGeneratedRef.current = false }, [isCallActive, endCall]) - const startScreenShare = useCallback(async (source: ScreenSource) => { - if (activeRealtimeModel.startsWith('grok-voice-')) return - setShowScreenPicker(false) - const capture = new ScreenCapture() - capture.setSourceName(source.name) - screenCaptureRef.current = capture - try { - await capture.start(source.id, (base64Jpeg) => { - realtime.sendFrame(base64Jpeg) - }) - setIsScreenSharing(true) - setScreenSourceName(source.name) - } catch (err) { - console.error('[ChatPage] Screen capture failed:', err) - screenCaptureRef.current = null - } - }, [activeRealtimeModel, realtime]) + const startScreenShare = useCallback( + async (source: ScreenSource) => { + if (activeRealtimeModel.startsWith('grok-voice-')) return + setShowScreenPicker(false) + const capture = new ScreenCapture() + capture.setSourceName(source.name) + screenCaptureRef.current = capture + try { + await capture.start(source.id, (base64Jpeg) => { + realtime.sendFrame(base64Jpeg) + }) + setIsScreenSharing(true) + setScreenSourceName(source.name) + } catch (err) { + console.error('[ChatPage] Screen capture failed:', err) + screenCaptureRef.current = null + } + }, + [activeRealtimeModel, realtime] + ) const stopScreenShare = useCallback(() => { screenCaptureRef.current?.stop() @@ -267,7 +292,8 @@ export function ChatPage() { } }, [isCallActive, isScreenSharing, stopScreenShare]) - const screenShareDisabled = isConnecting || (!isScreenSharing && activeRealtimeModel.startsWith('grok-voice-')) + const screenShareDisabled = + isConnecting || (!isScreenSharing && activeRealtimeModel.startsWith('grok-voice-')) const screenShareTitle = isScreenSharing ? 'Stop screen sharing' : activeRealtimeModel.startsWith('grok-voice-') @@ -304,13 +330,11 @@ export function ChatPage() { }, [isCallActive, newConversation, toggleMute, endCall]) return ( -
+
{/* Header */} -
+
- {messages.length > 0 - ? `${messages.length} messages` - : 'Start a conversation'} + {messages.length > 0 ? `${messages.length} messages` : 'Start a conversation'}
@@ -381,18 +402,15 @@ export function ChatPage() { {/* Connection error */} {connectionError && ( -
+
{connectionError}
)} {/* Call controls */} -
+
{!isCallActive && !isConnecting ? ( - @@ -402,8 +420,7 @@ export function ChatPage() { variant="ghost" size="icon" onClick={toggleMute} - className={isMuted ? 'text-destructive' : 'text-foreground'} - > + className={isMuted ? 'text-destructive' : 'text-foreground'}> {isMuted ? : } @@ -411,25 +428,25 @@ export function ChatPage() { variant="ghost" size="icon" onClick={isScreenSharing ? stopScreenShare : () => setShowScreenPicker(true)} - className={isScreenSharing ? 'text-green-500' : screenShareDisabled ? 'text-muted-foreground opacity-50' : 'text-foreground'} - disabled={screenShareDisabled} - > + className={ + isScreenSharing + ? 'text-green-500' + : screenShareDisabled + ? 'text-muted-foreground opacity-50' + : 'text-foreground' + } + disabled={screenShareDisabled}> {isScreenSharing ? : } - {isConnecting && ( - Connecting... + Connecting... )} {realtime.isReconnecting && ( - Reconnecting... + Reconnecting... )} )} @@ -460,13 +477,16 @@ function generateTitle(text: string): string { return title.trim() + '...' } -function normalizeRealtimeModel(model: string | null): typeof REALTIME_MODELS[number] { +function normalizeRealtimeModel(model: string | null): (typeof REALTIME_MODELS)[number] { return (REALTIME_MODELS as readonly string[]).includes(model ?? '') - ? model as typeof REALTIME_MODELS[number] + ? (model as (typeof REALTIME_MODELS)[number]) : DEFAULT_REALTIME_MODEL } -function normalizeRealtimeVoice(model: typeof REALTIME_MODELS[number], voice: string | null): string { +function normalizeRealtimeVoice( + model: (typeof REALTIME_MODELS)[number], + voice: string | null +): string { if (model.startsWith('grok-voice-')) { return voice && (XAI_VOICES as readonly string[]).includes(voice) ? voice : 'eve' } diff --git a/desktop/src/renderer/src/pages/SettingsPage.tsx b/desktop/src/renderer/src/pages/SettingsPage.tsx index 16e9adf8..c4c9ecf9 100644 --- a/desktop/src/renderer/src/pages/SettingsPage.tsx +++ b/desktop/src/renderer/src/pages/SettingsPage.tsx @@ -1,5 +1,5 @@ import { useCallback, useEffect, useRef, useState } from 'react' -import { Wifi, WifiOff, Eye, EyeOff } from 'lucide-react' +import { Wifi, WifiOff, Eye, EyeOff, Network, BrainCircuit } from 'lucide-react' import { Button } from '../components/ui/Button' import { Card } from '../components/ui/Card' import { Input } from '../components/ui/Input' @@ -8,9 +8,27 @@ import { Toggle } from '../components/ui/Toggle' import { useTheme, type Theme } from '../lib/use-theme' import { enumerateAudioDevices, type AudioDevice } from '../lib/audio-engine' import { getSetting, setSetting } from '../lib/db' - -const GEMINI_VOICES = ['Puck', 'Charon', 'Kore', 'Fenrir', 'Aoede', 'Leda', 'Orus', 'Zephyr'] as const -const GEMINI_VOICE_LABELS: Record = { +import { + defaultServerUrlForMode, + isRealtimeConnectionMode, + LEGACY_REALTIME_SERVER_URL_SETTING, + REALTIME_CONNECTION_MODE_SETTING, + resolveServerUrlForMode, + serverUrlSettingKeyForMode, + type RealtimeConnectionMode, +} from '../lib/realtime-settings' + +const GEMINI_VOICES = [ + 'Puck', + 'Charon', + 'Kore', + 'Fenrir', + 'Aoede', + 'Leda', + 'Orus', + 'Zephyr', +] as const +const GEMINI_VOICE_LABELS: Record<(typeof GEMINI_VOICES)[number], string> = { Puck: 'Puck (M)', Charon: 'Charon (M)', Kore: 'Kore (F)', @@ -22,7 +40,7 @@ const GEMINI_VOICE_LABELS: Record = { } const XAI_VOICES = ['eve', 'ara', 'rex', 'sal', 'leo'] as const -const XAI_VOICE_LABELS: Record = { +const XAI_VOICE_LABELS: Record<(typeof XAI_VOICES)[number], string> = { eve: 'Eve (F)', ara: 'Ara (F)', rex: 'Rex (M)', @@ -37,7 +55,8 @@ export function SettingsPage() { const { theme, setTheme } = useTheme() // Connection - const [serverUrl, setServerUrl] = useState('ws://localhost:8080/ws') + const [connectionMode, setConnectionMode] = useState('relay') + const [serverUrl, setServerUrl] = useState(defaultServerUrlForMode('relay')) const [apiKey, setApiKey] = useState('') const [showApiKey, setShowApiKey] = useState(false) const [testStatus, setTestStatus] = useState<'idle' | 'testing' | 'ok' | 'error'>('idle') @@ -63,8 +82,11 @@ export function SettingsPage() { // Load all settings on mount useEffect(() => { ;(async () => { - const url = await getSetting('realtime_server_url') - if (url) setServerUrl(url) + const mode = await getSetting(REALTIME_CONNECTION_MODE_SETTING) + const nextMode = isRealtimeConnectionMode(mode) ? mode : 'relay' + setConnectionMode(nextMode) + const url = await loadServerUrlForMode(nextMode) + setServerUrl(url) const key = await getSetting('realtime_api_key') if (key) setApiKey(key) const m = await getSetting('realtime_model') @@ -99,52 +121,90 @@ export function SettingsPage() { setSetting(key, value) }, []) - const updateServerUrl = useCallback((v: string) => { - setServerUrl(v) - if (loadedRef.current) save('realtime_server_url', v) - }, [save]) - - const updateApiKey = useCallback((v: string) => { - setApiKey(v) - if (loadedRef.current) save('realtime_api_key', v) - }, [save]) - - const updateModel = useCallback((v: RealtimeModel) => { - setModel(v) - if (loadedRef.current) save('realtime_model', v) - // Reset voice when switching providers - const isGemini = v.startsWith('gemini-') - const currentIsGemini = (GEMINI_VOICES as readonly string[]).includes(voice) - const isXAI = v.startsWith('grok-voice-') - const currentIsXAI = (XAI_VOICES as readonly string[]).includes(voice) - if (isGemini && !currentIsGemini) { - setVoice('Zephyr') - save('realtime_voice', 'Zephyr') - } else if (isXAI && !currentIsXAI) { - setVoice('eve') - save('realtime_voice', 'eve') - } - }, [save, voice]) + const updateServerUrl = useCallback( + (v: string) => { + setServerUrl(v) + setTestStatus('idle') + if (loadedRef.current) { + save(serverUrlSettingKeyForMode(connectionMode), v) + if (connectionMode === 'relay') save(LEGACY_REALTIME_SERVER_URL_SETTING, v) + } + }, + [connectionMode, save] + ) + + const updateApiKey = useCallback( + (v: string) => { + setApiKey(v) + setTestStatus('idle') + if (loadedRef.current) save('realtime_api_key', v) + }, + [save] + ) + + const updateModel = useCallback( + (v: RealtimeModel) => { + setModel(v) + if (loadedRef.current) save('realtime_model', v) + // Reset voice when switching providers + const isGemini = v.startsWith('gemini-') + const currentIsGemini = (GEMINI_VOICES as readonly string[]).includes(voice) + const isXAI = v.startsWith('grok-voice-') + const currentIsXAI = (XAI_VOICES as readonly string[]).includes(voice) + if (isGemini && !currentIsGemini) { + setVoice('Zephyr') + save('realtime_voice', 'Zephyr') + } else if (isXAI && !currentIsXAI) { + setVoice('eve') + save('realtime_voice', 'eve') + } + }, + [save, voice] + ) - const updateVoice = useCallback((v: string) => { - setVoice(v) - if (loadedRef.current) save('realtime_voice', v) - }, [save]) + const updateConnectionMode = useCallback( + async (v: RealtimeConnectionMode) => { + setConnectionMode(v) + setTestStatus('idle') + if (loadedRef.current) save(REALTIME_CONNECTION_MODE_SETTING, v) - const updateVolume = useCallback((v: number) => { - setVolume(v) - if (loadedRef.current) save('realtime_volume', String(v)) - }, [save]) + const nextUrl = await loadServerUrlForMode(v) + setServerUrl(nextUrl) + }, + [save] + ) - const updateInputDevice = useCallback((v: string) => { - setInputDeviceId(v) - if (loadedRef.current) save('input_device_id', v) - }, [save]) + const updateVoice = useCallback( + (v: string) => { + setVoice(v) + if (loadedRef.current) save('realtime_voice', v) + }, + [save] + ) - const updateOutputDevice = useCallback((v: string) => { - setOutputDeviceId(v) - if (loadedRef.current) save('output_device_id', v) - }, [save]) + const updateVolume = useCallback( + (v: number) => { + setVolume(v) + if (loadedRef.current) save('realtime_volume', String(v)) + }, + [save] + ) + + const updateInputDevice = useCallback( + (v: string) => { + setInputDeviceId(v) + if (loadedRef.current) save('input_device_id', v) + }, + [save] + ) + + const updateOutputDevice = useCallback( + (v: string) => { + setOutputDeviceId(v) + if (loadedRef.current) save('output_device_id', v) + }, + [save] + ) const toggleDebugMode = useCallback((v: boolean) => { setDebugMode(v) @@ -181,54 +241,72 @@ export function SettingsPage() { const inputDevices = audioDevices.filter((d) => d.kind === 'audioinput') const outputDevices = audioDevices.filter((d) => d.kind === 'audiooutput') + const isRealtimeBrainMode = connectionMode === 'realtime-brain' + const serverUrlLabel = isRealtimeBrainMode ? 'OpenClaw Realtime URL' : 'Relay Server URL' + const serverUrlPlaceholder = defaultServerUrlForMode(connectionMode) + const apiKeyLabel = isRealtimeBrainMode ? 'OpenClaw Gateway Token' : 'API Key' + const apiKeyPlaceholder = isRealtimeBrainMode + ? 'Enter gateway token or password' + : 'Enter your API key' return ( -
-
- +
+
{/* Connection */} - +

Connection

+
+ {( + [ + { value: 'relay', label: 'Relay', icon: Network }, + { value: 'realtime-brain', label: 'Real-time brain', icon: BrainCircuit }, + ] as const + ).map(({ value, label, icon: Icon }) => ( + + ))} +
+
- + updateServerUrl(e.target.value)} - placeholder="ws://localhost:8080/ws" + placeholder={serverUrlPlaceholder} />
- +
-
+
updateApiKey(e.target.value)} - placeholder="Enter your API key" + placeholder={apiKeyPlaceholder} className="pr-10" />
- + {/* Model */} - +

Model

{(['gemini-3.1-flash-live-preview', 'grok-voice-think-fast-1.0'] as const).map((m) => { @@ -236,18 +314,16 @@ export function SettingsPage() { ) @@ -256,57 +332,54 @@ export function SettingsPage() { {/* Voice */} - +

Voice

{(model.startsWith('gemini-') ? GEMINI_VOICES : XAI_VOICES).map((v) => ( ))}
{/* Audio Devices */} - +

Audio Devices

- updateInputDevice(e.target.value)}> {inputDevices.map((d) => ( - + ))}
- updateOutputDevice(e.target.value)}> {outputDevices.map((d) => ( - + ))}
- + enumerateAudioDevices().then(setAudioDevices)} - > + onClick={() => enumerateAudioDevices().then(setAudioDevices)}> Refresh Devices {/* Appearance */} - +

Appearance

{(['dark', 'light', 'system'] as Theme[]).map((t) => ( ))} @@ -350,7 +419,7 @@ export function SettingsPage() { {/* Debug */} - +

Debug

@@ -364,7 +433,9 @@ export function SettingsPage() {

Show Latency

-

Display latency badges on chat messages

+

+ Display latency badges on chat messages +

@@ -372,23 +443,41 @@ export function SettingsPage() {

Send Traces

-

Post per-turn latency to Langfuse via relay

+

+ Post per-turn latency to Langfuse via the active realtime server +

{/* Setup Instructions */} - +

Setup

-
    -
  1. Start the relay server: cd relay-server && yarn dev
    2. -
  2. Enter the relay server URL shown on startup
    3. -
  3. Enter your API key for authentication
    4. -
  4. Click Test to verify the connection
    5. -
+ {isRealtimeBrainMode ? ( +
    +
  1. Start OpenClaw from the realtime brain worktree on port 19789
    2. +
  2. + Set the URL to{' '} + + ws://localhost:19789/voiceclaw/realtime + +
    3. +
  3. Enter the OpenClaw gateway token or password if gateway auth is enabled
    4. +
  4. Click Test to verify the connection
    5. +
+ ) : ( +
    +
  1. + Start the relay server:{' '} + cd relay-server && yarn dev +
    2. +
  2. Enter the relay server URL shown on startup
    3. +
  3. Enter your API key for authentication
    4. +
  4. Click Test to verify the connection
    5. +
+ )} -
) @@ -412,6 +501,12 @@ function normalizeRealtimeVoice(model: RealtimeModel, voice: string | null): str // --- Helper Components --- +async function loadServerUrlForMode(mode: RealtimeConnectionMode): Promise { + const modeUrl = await getSetting(serverUrlSettingKeyForMode(mode)) + const legacyUrl = mode === 'relay' ? await getSetting(LEGACY_REALTIME_SERVER_URL_SETTING) : null + return resolveServerUrlForMode(mode, modeUrl || legacyUrl) +} + function ConnectionStatus({ status, error, @@ -422,36 +517,43 @@ function ConnectionStatus({ onTest: () => void }) { return ( -
+
{status === 'testing' ? ( -
+
) : status === 'ok' ? ( ) : ( - + )} - - {status === 'ok' ? 'Connected' - : status === 'testing' ? 'Testing...' - : status === 'error' ? error - : 'Not tested'} + + {status === 'ok' + ? 'Connected' + : status === 'testing' + ? 'Testing...' + : status === 'error' + ? error + : 'Not tested'}
-