feat(voice): animate the mascot's mouth during realtime voice calls

app/src/features/human/HumanPage.tsx

@@ -1,4 +1,4 @@
-import { useMemo } from 'react';
+import { useMemo, useRef } from 'react';
 
 import { useT } from '../../lib/i18n/I18nContext';
 import { useAppDispatch, useAppSelector } from '../../store/hooks';
@@ -22,6 +22,8 @@ import {
 import { useMascotManifest } from './Mascot/manifest/useMascotManifest';
 import RealtimeVoiceControls from './RealtimeVoiceControls';
 import { useHumanMascot } from './useHumanMascot';
+import { IDLE_REALTIME_VOICE_AUDIO, type RealtimeVoiceAudio } from './voice/amplitudeLipsync';
+import { useAmplitudeLipsync } from './voice/useAmplitudeLipsync';
 import { resolveHumanVoiceEntry } from './voiceEntry';
 
 const HumanPage = () => {
@@ -35,6 +37,18 @@ const HumanPage = () => {
   const speakReplies = useAppSelector(selectSpeakReplies);
 
   const { face, visemeCode } = useHumanMascot({ speakReplies });
+
+  // Lip-sync for the realtime voice session. The session lives inside
+  // RealtimeVoiceControls (which owns its own ConversationProvider), so it
+  // publishes its output-loudness accessor into this ref and the mascot samples
+  // it per frame — a 60fps signal must not travel through React state.
+  const realtimeAudioRef = useRef<RealtimeVoiceAudio>({ ...IDLE_REALTIME_VOICE_AUDIO });
+  const realtimeLipsync = useAmplitudeLipsync(realtimeAudioRef);
+
+  // While the agent is speaking its own audio drives the mouth; otherwise the
+  // classic path keeps ownership, so the two never fight over the same frame.
+  const mascotFace = realtimeLipsync.active ? 'speaking' : face;
+  const mascotVisemeCode = realtimeLipsync.active ? realtimeLipsync.visemeCode : visemeCode;
   const mascotColor = useAppSelector(selectMascotColor);
   const customPrimary = useAppSelector(selectCustomPrimaryColor);
   const customSecondary = useAppSelector(selectCustomSecondaryColor);
@@ -76,7 +90,9 @@ const HumanPage = () => {
       <Conversations
         variant="sidebar"
         composer="mic-cloud"
-        voiceChatControl={voiceEntry === 'realtime' ? <RealtimeVoiceControls /> : null}
+        voiceChatControl={
+          voiceEntry === 'realtime' ? <RealtimeVoiceControls audioRef={realtimeAudioRef} /> : null
+        }
         showMicComposer={voiceEntry !== 'realtime'}
         projectThreadList
       />
@@ -97,23 +113,23 @@ const HumanPage = () => {
       <div className="absolute inset-y-0 left-0 right-[436px] flex items-center justify-center">
         <div className="relative w-[min(80vh,90%)] aspect-square">
           {customMascotGifUrl ? (
-            <CustomGifMascot src={customMascotGifUrl} face={face} />
+            <CustomGifMascot src={customMascotGifUrl} face={mascotFace} />
           ) : mascotEntry ? (
             <ManifestRiveMascot
               key={mascotEntry.id}
               entry={mascotEntry}
-              face={face}
+              face={mascotFace}
               primaryColor={primaryColor}
               secondaryColor={secondaryColor}
-              visemeCode={visemeCode}
+              visemeCode={mascotVisemeCode}
               idlePoseRotation
             />
           ) : (
             <RiveMascot
-              face={face}
+              face={mascotFace}
               primaryColor={primaryColor}
               secondaryColor={secondaryColor}
-              visemeCode={visemeCode}
+              visemeCode={mascotVisemeCode}
               idlePoseRotation
             />
           )}
@@ -125,7 +141,7 @@ const HumanPage = () => {
           tap-and-speak rather than a second button stacked on it. */}
       {voiceEntry === 'both' && (
         <div className="absolute bottom-8 left-0 right-[436px] z-10 flex justify-center">
-          <RealtimeVoiceControls />
+          <RealtimeVoiceControls audioRef={realtimeAudioRef} />
         </div>
       )}
 

app/src/features/human/RealtimeVoiceControls.test.tsx

@@ -31,6 +31,7 @@ function makeSession(overrides: Partial<RealtimeVoiceSession> = {}): RealtimeVoi
     state: 'idle',
     isSpeaking: false,
     mode: 'listening',
+    getOutputVolume: () => 0,
     error: null,
     start,
     stop,

app/src/features/human/RealtimeVoiceControls.tsx

@@ -1,9 +1,11 @@
 import { ConversationProvider } from '@elevenlabs/react';
+import { type RefObject, useEffect } from 'react';
 
 import Button from '../../components/ui/Button';
 import { useT } from '../../lib/i18n/I18nContext';
 import { useAppSelector } from '../../store/hooks';
 import { selectEffectiveMascotVoiceId } from '../../store/mascotSlice';
+import type { RealtimeVoiceAudio } from './voice/amplitudeLipsync';
 import { useRealtimeVoiceSession } from './voice/useRealtimeVoiceSession';
 
 /**
@@ -12,14 +14,37 @@ import { useRealtimeVoiceSession } from './voice/useRealtimeVoiceSession';
  * Wraps its own `ConversationProvider` (required by `@elevenlabs/react`) so it
  * stays self-contained and adds no context to the rest of the app.
  */
-function RealtimeVoiceControlsInner() {
+function RealtimeVoiceControlsInner({ audioRef }: { audioRef?: RefObject<RealtimeVoiceAudio> }) {
   const { t } = useT();
   const voiceId = useAppSelector(selectEffectiveMascotVoiceId);
   const session = useRealtimeVoiceSession({ voiceId });
 
   const active = session.state === 'active';
   const connecting = session.state === 'connecting';
 
+  // Publish the output-loudness accessor for the mascot's lip-sync. Written into
+  // a ref rather than lifted into state because the mascot samples it once per
+  // animation frame; see `useAmplitudeLipsync`. The session lives under this
+  // component's own ConversationProvider, so this is the only place that can
+  // reach it.
+  const { getOutputVolume, isSpeaking } = session;
+  useEffect(() => {
+    if (!audioRef) return;
+    audioRef.current.getOutputVolume = active ? getOutputVolume : null;
+    audioRef.current.speaking = active && isSpeaking;
+  }, [audioRef, active, isSpeaking, getOutputVolume]);
+
+  // A session that ends mid-speech would otherwise leave `speaking` true and the
+  // mouth frozen open.
+  useEffect(
+    () => () => {
+      if (!audioRef) return;
+      audioRef.current.getOutputVolume = null;
+      audioRef.current.speaking = false;
+    },
+    [audioRef]
+  );
+
   const label = connecting
     ? t('voice.mode.connecting')
     : active
@@ -51,10 +76,15 @@ function RealtimeVoiceControlsInner() {
   );
 }
 
-export default function RealtimeVoiceControls() {
+export default function RealtimeVoiceControls({
+  audioRef,
+}: {
+  /** Optional sink for the mascot's lip-sync signal (see RealtimeVoiceAudio). */
+  audioRef?: RefObject<RealtimeVoiceAudio>;
+}) {
   return (
     <ConversationProvider>
-      <RealtimeVoiceControlsInner />
+      <RealtimeVoiceControlsInner audioRef={audioRef} />
     </ConversationProvider>
   );
 }

app/src/features/human/voice/amplitudeLipsync.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, it } from 'vitest';
+
+import { amplitudeToVisemeCode, smoothAmplitude } from './amplitudeLipsync';
+
+describe('amplitudeToVisemeCode', () => {
+  it('rests the mouth on silence and room tone', () => {
+    expect(amplitudeToVisemeCode(0)).toBe('sil');
+    // Below the floor: the tail of a word, not speech. Holding the mouth open
+    // through these gaps is what makes naive amplitude lip-sync look slack.
+    expect(amplitudeToVisemeCode(0.03)).toBe('sil');
+  });
+
+  it('opens the mouth further as the signal gets louder', () => {
+    const codes = [0.08, 0.2, 0.6].map(amplitudeToVisemeCode);
+    expect(codes).toEqual(['I', 'E', 'aa']);
+  });
+
+  // A garbage reading (analyser torn down mid-frame) must not map to a wide-open
+  // mouth that then sticks for the rest of the call. Infinity rests for the same
+  // reason NaN does: it is a broken sample, not a loud one.
+  it('rests on a non-finite reading', () => {
+    expect(amplitudeToVisemeCode(Number.NaN)).toBe('sil');
+    expect(amplitudeToVisemeCode(Number.POSITIVE_INFINITY)).toBe('sil');
+  });
+});
+
+describe('smoothAmplitude', () => {
+  it('moves toward the new sample', () => {
+    expect(smoothAmplitude(0, 1)).toBeGreaterThan(0);
+    expect(smoothAmplitude(1, 0)).toBeLessThan(1);
+  });
+
+  // Asymmetric on purpose: consonant onsets must land on time, but the mouth
+  // must not snap shut inside a word.
+  it('opens faster than it closes', () => {
+    const opening = smoothAmplitude(0, 1) - 0;
+    const closing = 1 - smoothAmplitude(1, 0);
+    expect(opening).toBeGreaterThan(closing);
+  });
+
+  it('converges on a held level rather than oscillating', () => {
+    let level = 0;
+    for (let i = 0; i < 40; i += 1) level = smoothAmplitude(level, 0.5);
+    expect(level).toBeCloseTo(0.5, 2);
+  });
+});

app/src/features/human/voice/amplitudeLipsync.ts

@@ -0,0 +1,83 @@
+/**
+ * Amplitude-driven lip-sync for the realtime voice session (#5399).
+ *
+ * The classic tap-and-speak path animates the mouth from a viseme *timeline*:
+ * frames of `{viseme, ms}` sampled against our own audio element's clock. That
+ * is not available here — the realtime SDK owns playback, so there is no
+ * `currentMs()` to sample and no per-character timing unless we subscribe to
+ * alignment events and reconstruct one.
+ *
+ * What the SDK does expose is the output signal itself (`getOutputVolume()`),
+ * so the mouth is driven from loudness instead. That is genuinely less accurate
+ * — it opens and closes with the envelope rather than forming phonemes, so no
+ * `M`/`F` closures — but it is in sync *by construction*, because it is the
+ * audio being played rather than a prediction of it. A frozen mouth while the
+ * agent talks reads as broken; an approximate one reads as alive.
+ *
+ * The viseme-timeline version is the follow-up, and this is the fallback it
+ * would keep for the case where alignment is absent or has run dry.
+ */
+
+/** Viseme codes, ordered by how open the mouth is. */
+const REST = 'sil';
+const NARROW = 'I'; // openness 0.30
+const MID = 'E'; // openness 0.45
+const OPEN = 'aa'; // openness ~0.95
+
+/**
+ * Below this the signal is room tone or the tail of a word, not speech. Holding
+ * the mouth open through those gaps is what makes naive amplitude lip-sync look
+ * slack-jawed, so anything under it rests.
+ */
+const SILENCE_FLOOR = 0.04;
+
+/** Where the mouth steps from narrow to mid, and from mid to wide open. */
+const MID_THRESHOLD = 0.12;
+const OPEN_THRESHOLD = 0.28;
+
+/**
+ * Smoothing applied to the raw reading, as the weight given to the new sample.
+ *
+ * `getOutputVolume()` is sampled per animation frame and is noisy at that rate:
+ * fed straight through it produces a chattering mouth that reads as a glitch
+ * rather than as speech. Asymmetric on purpose — opening tracks the signal
+ * quickly so consonant onsets land on time, closing lags so the mouth does not
+ * snap shut inside a word.
+ */
+const ATTACK = 0.55;
+const RELEASE = 0.18;
+
+/** Smooth one amplitude sample toward the previous level. Pure + unit-tested. */
+export function smoothAmplitude(previous: number, sample: number): number {
+  const weight = sample > previous ? ATTACK : RELEASE;
+  return previous + (sample - previous) * weight;
+}
+
+/**
+ * Map a smoothed amplitude (0..1) onto a viseme code. Steps rather than
+ * interpolates because the Rive mouth is driven by a code, not a scalar.
+ * Pure + unit-tested.
+ */
+export function amplitudeToVisemeCode(level: number): string {
+  if (!Number.isFinite(level) || level < SILENCE_FLOOR) return REST;
+  if (level < MID_THRESHOLD) return NARROW;
+  if (level < OPEN_THRESHOLD) return MID;
+  return OPEN;
+}
+
+/**
+ * What the realtime controls publish for the mascot to read. Held in a ref and
+ * mutated in place: the mascot samples it once per animation frame, and routing
+ * a 60fps signal through React state would re-render the page on every frame.
+ */
+export interface RealtimeVoiceAudio {
+  /** SDK accessor for output loudness, or null when no session is live. */
+  getOutputVolume: (() => number) | null;
+  /** Whether the agent is currently speaking (SDK `isSpeaking`). */
+  speaking: boolean;
+}
+
+export const IDLE_REALTIME_VOICE_AUDIO: RealtimeVoiceAudio = {
+  getOutputVolume: null,
+  speaking: false,
+};

app/src/features/human/voice/useAmplitudeLipsync.test.ts

@@ -0,0 +1,84 @@
+import { act, renderHook } from '@testing-library/react';
+import { createRef } from 'react';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { type RealtimeVoiceAudio } from './amplitudeLipsync';
+import { useAmplitudeLipsync } from './useAmplitudeLipsync';
+
+/** Drive the rAF loop by hand so frames are deterministic. */
+let frames: FrameRequestCallback[] = [];
+
+function flushFrames(count: number): void {
+  for (let i = 0; i < count; i += 1) {
+    const pending = frames;
+    frames = [];
+    act(() => pending.forEach(cb => cb(performance.now())));
+  }
+}
+
+function audioRef(overrides: Partial<RealtimeVoiceAudio> = {}) {
+  const ref = createRef<RealtimeVoiceAudio>() as { current: RealtimeVoiceAudio };
+  ref.current = { getOutputVolume: null, speaking: false, ...overrides };
+  return ref;
+}
+
+describe('useAmplitudeLipsync', () => {
+  beforeEach(() => {
+    frames = [];
+    vi.stubGlobal('requestAnimationFrame', (cb: FrameRequestCallback) => {
+      frames.push(cb);
+      return frames.length;
+    });
+    vi.stubGlobal('cancelAnimationFrame', () => {});
+  });
+  afterEach(() => vi.unstubAllGlobals());
+
+  it('stays inactive and rested while nothing is speaking', () => {
+    const { result } = renderHook(() => useAmplitudeLipsync(audioRef()));
+    flushFrames(3);
+    expect(result.current).toEqual({ active: false, visemeCode: 'sil' });
+  });
+
+  it('drives the mouth from output loudness while the agent speaks', () => {
+    const ref = audioRef({ speaking: true, getOutputVolume: () => 0.9 });
+    const { result } = renderHook(() => useAmplitudeLipsync(ref));
+    flushFrames(10);
+    expect(result.current.active).toBe(true);
+    expect(result.current.visemeCode).toBe('aa');
+  });
+
+  // The mouth must return to rest when the turn ends, not freeze on its last
+  // shape — a stuck-open mascot is worse than one that never moved.
+  it('rests the mouth when speaking stops', () => {
+    const ref = audioRef({ speaking: true, getOutputVolume: () => 0.9 });
+    const { result } = renderHook(() => useAmplitudeLipsync(ref));
+    flushFrames(10);
+    expect(result.current.visemeCode).toBe('aa');
+
+    ref.current.speaking = false;
+    flushFrames(2);
+    expect(result.current).toEqual({ active: false, visemeCode: 'sil' });
+  });
+
+  // The SDK reads a live analyser; a session torn down mid-frame throws rather
+  // than returning 0. An uncaught throw would kill the loop and freeze the mouth.
+  it('survives an accessor that throws mid-session', () => {
+    const ref = audioRef({
+      speaking: true,
+      getOutputVolume: () => {
+        throw new Error('analyser closed');
+      },
+    });
+    const { result } = renderHook(() => useAmplitudeLipsync(ref));
+    expect(() => flushFrames(5)).not.toThrow();
+    expect(result.current.visemeCode).toBe('sil');
+  });
+
+  it('cancels its frame loop on unmount', () => {
+    const cancel = vi.fn();
+    vi.stubGlobal('cancelAnimationFrame', cancel);
+    const { unmount } = renderHook(() => useAmplitudeLipsync(audioRef()));
+    unmount();
+    expect(cancel).toHaveBeenCalled();
+  });
+});