feat(voice): animate the mascot's mouth during realtime voice calls

app/src/features/human/HumanPage.tsx

@@ -1,4 +1,4 @@
-import { useMemo } from 'react';
+import { useMemo, useRef, useState } from 'react';
 
 import { useT } from '../../lib/i18n/I18nContext';
 import { useAppDispatch, useAppSelector } from '../../store/hooks';
@@ -22,6 +22,8 @@ import {
 import { useMascotManifest } from './Mascot/manifest/useMascotManifest';
 import RealtimeVoiceControls from './RealtimeVoiceControls';
 import { useHumanMascot } from './useHumanMascot';
+import { IDLE_REALTIME_VOICE_AUDIO, type RealtimeVoiceAudio } from './voice/amplitudeLipsync';
+import { useAmplitudeLipsync } from './voice/useAmplitudeLipsync';
 import { resolveHumanVoiceEntry } from './voiceEntry';
 
 const HumanPage = () => {
@@ -35,6 +37,24 @@ const HumanPage = () => {
   const speakReplies = useAppSelector(selectSpeakReplies);
 
   const { face, visemeCode } = useHumanMascot({ speakReplies });
+
+  // Lip-sync for the realtime voice session. The session lives inside
+  // RealtimeVoiceControls (which owns its own ConversationProvider), so it
+  // publishes its output-loudness accessor into this ref and the mascot samples
+  // it per frame — a 60fps signal must not travel through React state.
+  const realtimeAudioRef = useRef<RealtimeVoiceAudio>({ ...IDLE_REALTIME_VOICE_AUDIO });
+  // The agent's speaking edge, lifted out of RealtimeVoiceControls so it can gate
+  // the lip-sync loop below. Flips a couple of times per turn, so it is cheap as
+  // state (the 60fps amplitude stays in the ref). While it is false — an idle
+  // realtime session, or the classic voice path that never mounts the control —
+  // the loop schedules no frames at all.
+  const [realtimeSpeaking, setRealtimeSpeaking] = useState(false);
+  const realtimeLipsync = useAmplitudeLipsync(realtimeAudioRef, realtimeSpeaking);
+
+  // While the agent is speaking its own audio drives the mouth; otherwise the
+  // classic path keeps ownership, so the two never fight over the same frame.
+  const mascotFace = realtimeLipsync.active ? 'speaking' : face;
+  const mascotVisemeCode = realtimeLipsync.active ? realtimeLipsync.visemeCode : visemeCode;
   const mascotColor = useAppSelector(selectMascotColor);
   const customPrimary = useAppSelector(selectCustomPrimaryColor);
   const customSecondary = useAppSelector(selectCustomSecondaryColor);
@@ -76,7 +96,14 @@ const HumanPage = () => {
       <Conversations
         variant="sidebar"
         composer="mic-cloud"
-        voiceChatControl={voiceEntry === 'realtime' ? <RealtimeVoiceControls /> : null}
+        voiceChatControl={
+          voiceEntry === 'realtime' ? (
+            <RealtimeVoiceControls
+              audioRef={realtimeAudioRef}
+              onSpeakingChange={setRealtimeSpeaking}
+            />
+          ) : null
+        }
         showMicComposer={voiceEntry !== 'realtime'}
         projectThreadList
       />
@@ -97,23 +124,23 @@ const HumanPage = () => {
       <div className="absolute inset-y-0 left-0 right-[436px] flex items-center justify-center">
         <div className="relative w-[min(80vh,90%)] aspect-square">
           {customMascotGifUrl ? (
-            <CustomGifMascot src={customMascotGifUrl} face={face} />
+            <CustomGifMascot src={customMascotGifUrl} face={mascotFace} />
           ) : mascotEntry ? (
             <ManifestRiveMascot
               key={mascotEntry.id}
               entry={mascotEntry}
-              face={face}
+              face={mascotFace}
               primaryColor={primaryColor}
               secondaryColor={secondaryColor}
-              visemeCode={visemeCode}
+              visemeCode={mascotVisemeCode}
               idlePoseRotation
             />
           ) : (
             <RiveMascot
-              face={face}
+              face={mascotFace}
               primaryColor={primaryColor}
               secondaryColor={secondaryColor}
-              visemeCode={visemeCode}
+              visemeCode={mascotVisemeCode}
               idlePoseRotation
             />
           )}
@@ -125,7 +152,10 @@ const HumanPage = () => {
           tap-and-speak rather than a second button stacked on it. */}
       {voiceEntry === 'both' && (
         <div className="absolute bottom-8 left-0 right-[436px] z-10 flex justify-center">
-          <RealtimeVoiceControls />
+          <RealtimeVoiceControls
+            audioRef={realtimeAudioRef}
+            onSpeakingChange={setRealtimeSpeaking}
+          />
         </div>
       )}
 

app/src/features/human/RealtimeVoiceControls.test.tsx

@@ -6,12 +6,13 @@
  */
 import { configureStore } from '@reduxjs/toolkit';
 import { fireEvent, render, screen } from '@testing-library/react';
-import type { ReactNode } from 'react';
+import type { ReactNode, RefObject } from 'react';
 import { Provider } from 'react-redux';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 
 import mascotReducer from '../../store/mascotSlice';
 import RealtimeVoiceControls from './RealtimeVoiceControls';
+import type { RealtimeVoiceAudio } from './voice/amplitudeLipsync';
 import type { RealtimeVoiceSession } from './voice/useRealtimeVoiceSession';
 
 // `@elevenlabs/react`'s ConversationProvider is only a context shell here — pass
@@ -31,6 +32,7 @@ function makeSession(overrides: Partial<RealtimeVoiceSession> = {}): RealtimeVoi
     state: 'idle',
     isSpeaking: false,
     mode: 'listening',
+    getOutputVolume: () => 0,
     error: null,
     start,
     stop,
@@ -48,11 +50,14 @@ const LABEL = {
   speaking: 'Speaking',
 } as const;
 
-function renderControls() {
+function renderControls(
+  onSpeakingChange?: (speaking: boolean) => void,
+  audioRef?: RefObject<RealtimeVoiceAudio>
+) {
   const store = configureStore({ reducer: { mascot: mascotReducer } });
   return render(
     <Provider store={store}>
-      <RealtimeVoiceControls />
+      <RealtimeVoiceControls onSpeakingChange={onSpeakingChange} audioRef={audioRef} />
     </Provider>
   );
 }
@@ -112,4 +117,55 @@ describe('RealtimeVoiceControls', () => {
     expect(stop).toHaveBeenCalledTimes(1);
     expect(start).not.toHaveBeenCalled();
   });
+
+  // The speaking edge gates the mascot's lip-sync loop on the page
+  // (useAmplitudeLipsync's `enabled`), so it must reflect `active && isSpeaking`,
+  // not either half alone.
+  it('reports not-speaking to onSpeakingChange while idle', () => {
+    const onSpeakingChange = vi.fn();
+    renderControls(onSpeakingChange);
+    expect(onSpeakingChange).toHaveBeenLastCalledWith(false);
+  });
+
+  it('reports speaking only when the session is active and the agent speaks', () => {
+    const onSpeakingChange = vi.fn();
+    session = makeSession({ state: 'active', isSpeaking: true });
+    renderControls(onSpeakingChange);
+    expect(onSpeakingChange).toHaveBeenLastCalledWith(true);
+  });
+
+  it('reports not-speaking when active but the agent is silent', () => {
+    const onSpeakingChange = vi.fn();
+    session = makeSession({ state: 'active', isSpeaking: false });
+    renderControls(onSpeakingChange);
+    expect(onSpeakingChange).toHaveBeenLastCalledWith(false);
+  });
+
+  // The lip-sync loop reads the SDK accessor and speaking flag straight out of
+  // this ref (see useAmplitudeLipsync); a regression that stopped publishing
+  // them would freeze the mascot's mouth mid-turn without failing any of the
+  // presentational tests above, so pin the wiring directly.
+  it('publishes the loudness accessor and speaking flag into audioRef', () => {
+    const getOutputVolume = () => 0.5;
+    session = makeSession({ state: 'active', isSpeaking: true, getOutputVolume });
+    const audioRef: RefObject<RealtimeVoiceAudio> = {
+      current: { getOutputVolume: null, speaking: false },
+    };
+    renderControls(undefined, audioRef);
+    expect(audioRef.current?.getOutputVolume).toBe(getOutputVolume);
+    expect(audioRef.current?.speaking).toBe(true);
+  });
+
+  it('clears the audioRef when the session ends (unmount)', () => {
+    session = makeSession({ state: 'active', isSpeaking: true, getOutputVolume: () => 0.5 });
+    const audioRef: RefObject<RealtimeVoiceAudio> = {
+      current: { getOutputVolume: null, speaking: false },
+    };
+    const { unmount } = renderControls(undefined, audioRef);
+    expect(audioRef.current?.speaking).toBe(true);
+
+    unmount();
+    expect(audioRef.current?.getOutputVolume).toBeNull();
+    expect(audioRef.current?.speaking).toBe(false);
+  });
 });

app/src/features/human/RealtimeVoiceControls.tsx

@@ -1,9 +1,11 @@
 import { ConversationProvider } from '@elevenlabs/react';
+import { type RefObject, useEffect } from 'react';
 
 import Button from '../../components/ui/Button';
 import { useT } from '../../lib/i18n/I18nContext';
 import { useAppSelector } from '../../store/hooks';
 import { selectEffectiveMascotVoiceId } from '../../store/mascotSlice';
+import type { RealtimeVoiceAudio } from './voice/amplitudeLipsync';
 import { useRealtimeVoiceSession } from './voice/useRealtimeVoiceSession';
 
 /**
@@ -12,14 +14,53 @@ import { useRealtimeVoiceSession } from './voice/useRealtimeVoiceSession';
  * Wraps its own `ConversationProvider` (required by `@elevenlabs/react`) so it
  * stays self-contained and adds no context to the rest of the app.
  */
-function RealtimeVoiceControlsInner() {
+function RealtimeVoiceControlsInner({
+  audioRef,
+  onSpeakingChange,
+}: {
+  audioRef?: RefObject<RealtimeVoiceAudio>;
+  onSpeakingChange?: (speaking: boolean) => void;
+}) {
   const { t } = useT();
   const voiceId = useAppSelector(selectEffectiveMascotVoiceId);
   const session = useRealtimeVoiceSession({ voiceId });
 
   const active = session.state === 'active';
   const connecting = session.state === 'connecting';
 
+  // Publish the output-loudness accessor for the mascot's lip-sync. Written into
+  // a ref rather than lifted into state because the mascot samples it once per
+  // animation frame; see `useAmplitudeLipsync`. The session lives under this
+  // component's own ConversationProvider, so this is the only place that can
+  // reach it.
+  const { getOutputVolume, isSpeaking } = session;
+  const speaking = active && isSpeaking;
+  useEffect(() => {
+    if (audioRef?.current) {
+      audioRef.current.getOutputVolume = active ? getOutputVolume : null;
+      audioRef.current.speaking = speaking;
+    }
+    // Surface the speaking edge so the page can gate the mascot's lip-sync rAF
+    // loop (`useAmplitudeLipsync`'s `enabled`) — an idle or classic Human tab
+    // then schedules no frames. Unlike the 60fps amplitude above, this flips a
+    // couple of times per turn, so it is cheap to lift into React state.
+    onSpeakingChange?.(speaking);
+  }, [audioRef, active, speaking, getOutputVolume, onSpeakingChange]);
+
+  // A session that ends mid-speech would otherwise leave `speaking` true and the
+  // mouth frozen open — reset both the ref the mascot samples and the speaking
+  // edge the page gates its loop on.
+  useEffect(
+    () => () => {
+      if (audioRef?.current) {
+        audioRef.current.getOutputVolume = null;
+        audioRef.current.speaking = false;
+      }
+      onSpeakingChange?.(false);
+    },
+    [audioRef, onSpeakingChange]
+  );
+
   const label = connecting
     ? t('voice.mode.connecting')
     : active
@@ -51,10 +92,18 @@ function RealtimeVoiceControlsInner() {
   );
 }
 
-export default function RealtimeVoiceControls() {
+export default function RealtimeVoiceControls({
+  audioRef,
+  onSpeakingChange,
+}: {
+  /** Optional sink for the mascot's lip-sync signal (see RealtimeVoiceAudio). */
+  audioRef?: RefObject<RealtimeVoiceAudio>;
+  /** Notified with the agent's speaking edge so the page can gate the loop. */
+  onSpeakingChange?: (speaking: boolean) => void;
+}) {
   return (
     <ConversationProvider>
-      <RealtimeVoiceControlsInner />
+      <RealtimeVoiceControlsInner audioRef={audioRef} onSpeakingChange={onSpeakingChange} />
     </ConversationProvider>
   );
 }

app/src/features/human/voice/amplitudeLipsync.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, it } from 'vitest';
+
+import { amplitudeToVisemeCode, smoothAmplitude } from './amplitudeLipsync';
+
+describe('amplitudeToVisemeCode', () => {
+  it('rests the mouth on silence and room tone', () => {
+    expect(amplitudeToVisemeCode(0)).toBe('sil');
+    // Below the floor: the tail of a word, not speech. Holding the mouth open
+    // through these gaps is what makes naive amplitude lip-sync look slack.
+    expect(amplitudeToVisemeCode(0.03)).toBe('sil');
+  });
+
+  it('opens the mouth further as the signal gets louder', () => {
+    const codes = [0.08, 0.2, 0.6].map(amplitudeToVisemeCode);
+    expect(codes).toEqual(['I', 'E', 'aa']);
+  });
+
+  // A garbage reading (analyser torn down mid-frame) must not map to a wide-open
+  // mouth that then sticks for the rest of the call. Infinity rests for the same
+  // reason NaN does: it is a broken sample, not a loud one.
+  it('rests on a non-finite reading', () => {
+    expect(amplitudeToVisemeCode(Number.NaN)).toBe('sil');
+    expect(amplitudeToVisemeCode(Number.POSITIVE_INFINITY)).toBe('sil');
+  });
+});
+
+describe('smoothAmplitude', () => {
+  it('moves toward the new sample', () => {
+    expect(smoothAmplitude(0, 1)).toBeGreaterThan(0);
+    expect(smoothAmplitude(1, 0)).toBeLessThan(1);
+  });
+
+  // Asymmetric on purpose: consonant onsets must land on time, but the mouth
+  // must not snap shut inside a word.
+  it('opens faster than it closes', () => {
+    const opening = smoothAmplitude(0, 1) - 0;
+    const closing = 1 - smoothAmplitude(1, 0);
+    expect(opening).toBeGreaterThan(closing);
+  });
+
+  it('converges on a held level rather than oscillating', () => {
+    let level = 0;
+    for (let i = 0; i < 40; i += 1) level = smoothAmplitude(level, 0.5);
+    expect(level).toBeCloseTo(0.5, 2);
+  });
+});