Claude/voice conversion realtime skbru

client/src/pages/Conversation/hooks/useVoiceConversion.ts

@@ -0,0 +1,306 @@
+/**
+ * Voice Conversion Hook
+ *
+ * Provides real-time voice conversion functionality using StreamVC-style
+ * architecture. Connects to the voice conversion WebSocket endpoint and
+ * streams audio for conversion.
+ *
+ * Usage:
+ *   const { start, stop, status, setTargetVoice } = useVoiceConversion({
+ *     onConvertedAudio: (audio) => playAudio(audio),
+ *   });
+ */
+
+import { useState, useCallback, useRef, useEffect } from "react";
+import Recorder from "opus-recorder";
+
+export type VCStatus =
+  | "idle"
+  | "connecting"
+  | "awaiting_reference"
+  | "reference_ready"
+  | "converting"
+  | "error"
+  | "disconnected";
+
+export interface VoiceConversionOptions {
+  /** Callback when converted audio is received */
+  onConvertedAudio?: (audioData: ArrayBuffer) => void;
+  /** Callback when status changes */
+  onStatusChange?: (status: VCStatus) => void;
+  /** Callback on error */
+  onError?: (error: string) => void;
+  /** Server URL (defaults to current host with /api/vc path) */
+  serverUrl?: string;
+  /** Sample rate (default: 24000) */
+  sampleRate?: number;
+}
+
+export interface VoiceConversionResult {
+  /** Current status */
+  status: VCStatus;
+  /** Start voice conversion with optional target voice */
+  start: (options?: { voice?: string; referenceMode?: boolean }) => Promise<void>;
+  /** Stop voice conversion */
+  stop: () => void;
+  /** Set target voice by name (requires voice embeddings on server) */
+  setTargetVoice: (voiceName: string) => void;
+  /** Send reference audio for voice cloning */
+  sendReferenceAudio: (audioData: ArrayBuffer) => void;
+  /** Signal end of reference audio collection */
+  endReferenceCollection: () => void;
+  /** Whether currently recording/converting */
+  isActive: boolean;
+  /** Available voices (fetched from server) */
+  availableVoices: string[];
+  /** Fetch available voices from server */
+  fetchVoices: () => Promise<string[]>;
+}
+
+export const useVoiceConversion = (
+  options: VoiceConversionOptions = {}
+): VoiceConversionResult => {
+  const {
+    onConvertedAudio,
+    onStatusChange,
+    onError,
+    serverUrl,
+    sampleRate = 24000,
+  } = options;
+
+  const [status, setStatus] = useState<VCStatus>("idle");
+  const [availableVoices, setAvailableVoices] = useState<string[]>([]);
+  const [isActive, setIsActive] = useState(false);
+
+  const socketRef = useRef<WebSocket | null>(null);
+  const recorderRef = useRef<Recorder | null>(null);
+  const targetVoiceRef = useRef<string | null>(null);
+
+  // Update status and notify
+  const updateStatus = useCallback(
+    (newStatus: VCStatus) => {
+      setStatus(newStatus);
+      onStatusChange?.(newStatus);
+    },
+    [onStatusChange]
+  );
+
+  // Get WebSocket URL
+  const getWsUrl = useCallback(
+    (voice?: string, referenceMode?: boolean) => {
+      const base =
+        serverUrl ||
+        `${window.location.protocol === "https:" ? "wss:" : "ws:"}//${window.location.host}/api/vc`;
+
+      const params = new URLSearchParams();
+      if (voice) params.set("voice", voice);
+      if (referenceMode) params.set("reference_mode", "true");
+
+      const queryString = params.toString();
+      return queryString ? `${base}?${queryString}` : base;
+    },
+    [serverUrl]
+  );
+
+  // Fetch available voices from server
+  const fetchVoices = useCallback(async (): Promise<string[]> => {
+    try {
+      const baseUrl = serverUrl?.replace(/^wss?:/, "http") || "";
+      const url = baseUrl
+        ? `${baseUrl.replace("/api/vc", "")}/api/vc/voices`
+        : "/api/vc/voices";
+
+      const response = await fetch(url);
+      if (!response.ok) {
+        throw new Error(`Failed to fetch voices: ${response.statusText}`);
+      }
+
+      const data = await response.json();
+      const voices = data.voices || [];
+      setAvailableVoices(voices);
+      return voices;
+    } catch (error) {
+      console.error("Failed to fetch voices:", error);
+      return [];
+    }
+  }, [serverUrl]);
+
+  // Set target voice
+  const setTargetVoice = useCallback((voiceName: string) => {
+    targetVoiceRef.current = voiceName;
+  }, []);
+
+  // Send reference audio
+  const sendReferenceAudio = useCallback((audioData: ArrayBuffer) => {
+    if (socketRef.current && socketRef.current.readyState === WebSocket.OPEN) {
+      // Prepend message type byte (0x01 for audio)
+      const message = new Uint8Array(audioData.byteLength + 1);
+      message[0] = 0x01;
+      message.set(new Uint8Array(audioData), 1);
+      socketRef.current.send(message.buffer);
+    }
+  }, []);
+
+  // Signal end of reference collection
+  const endReferenceCollection = useCallback(() => {
+    if (socketRef.current && socketRef.current.readyState === WebSocket.OPEN) {
+      // Send control message (0x03) with "end_reference"
+      const encoder = new TextEncoder();
+      const text = encoder.encode("end_reference");
+      const message = new Uint8Array(text.length + 1);
+      message[0] = 0x03;
+      message.set(text, 1);
+      socketRef.current.send(message.buffer);
+    }
+  }, []);
+
+  // Start voice conversion
+  const start = useCallback(
+    async (startOptions?: { voice?: string; referenceMode?: boolean }) => {
+      const voice = startOptions?.voice || targetVoiceRef.current || undefined;
+      const referenceMode = startOptions?.referenceMode || false;
+
+      try {
+        updateStatus("connecting");
+
+        // Create WebSocket connection
+        const wsUrl = getWsUrl(voice, referenceMode);
+        const ws = new WebSocket(wsUrl);
+        ws.binaryType = "arraybuffer";
+
+        ws.onopen = () => {
+          console.log("Voice conversion WebSocket connected");
+        };
+
+        ws.onmessage = (event) => {
+          const data = new Uint8Array(event.data);
+          if (data.length === 0) return;
+
+          const messageType = data[0];
+          const payload = data.slice(1);
+
+          switch (messageType) {
+            case 0x00: // Handshake
+              console.log("Voice conversion handshake received");
+              updateStatus(referenceMode ? "awaiting_reference" : "converting");
+              break;
+
+            case 0x01: // Audio
+              onConvertedAudio?.(payload.buffer);
+              break;
+
+            case 0x03: // Control
+              const controlMsg = new TextDecoder().decode(payload);
+              if (controlMsg === "awaiting_reference") {
+                updateStatus("awaiting_reference");
+              } else if (controlMsg === "reference_ready") {
+                updateStatus("converting");
+              }
+              break;
+
+            case 0x05: // Error
+              const errorMsg = new TextDecoder().decode(payload);
+              onError?.(errorMsg);
+              updateStatus("error");
+              break;
+          }
+        };
+
+        ws.onclose = () => {
+          console.log("Voice conversion WebSocket closed");
+          updateStatus("disconnected");
+          setIsActive(false);
+        };
+
+        ws.onerror = (error) => {
+          console.error("Voice conversion WebSocket error:", error);
+          onError?.("WebSocket connection error");
+          updateStatus("error");
+        };
+
+        socketRef.current = ws;
+
+        // Start microphone recording
+        const encoderPath = new URL(
+          "opus-recorder/dist/encoderWorker.min.js",
+          import.meta.url
+        ).href;
+
+        const recorder = new Recorder({
+          encoderPath,
+          encoderSampleRate: sampleRate,
+          encoderFrameSize: 20, // 20ms frames
+          maxFramesPerPage: 2, // 40ms packets
+          numberOfChannels: 1,
+          encoderApplication: 2049, // VOIP mode
+          encoderComplexity: 0, // Low CPU
+        });
+
+        recorder.ondataavailable = (opusData: ArrayBuffer) => {
+          if (
+            socketRef.current &&
+            socketRef.current.readyState === WebSocket.OPEN &&
+            status === "converting"
+          ) {
+            // Send audio with message type byte
+            const message = new Uint8Array(opusData.byteLength + 1);
+            message[0] = 0x01;
+            message.set(new Uint8Array(opusData), 1);
+            socketRef.current.send(message.buffer);
+          }
+        };
+
+        await recorder.start();
+        recorderRef.current = recorder;
+        setIsActive(true);
+
+        console.log("Voice conversion started");
+      } catch (error) {
+        console.error("Failed to start voice conversion:", error);
+        onError?.(error instanceof Error ? error.message : "Unknown error");
+        updateStatus("error");
+      }
+    },
+    [getWsUrl, onConvertedAudio, onError, sampleRate, status, updateStatus]
+  );
+
+  // Stop voice conversion
+  const stop = useCallback(() => {
+    // Stop recorder
+    if (recorderRef.current) {
+      recorderRef.current.stop();
+      recorderRef.current = null;
+    }
+
+    // Close WebSocket
+    if (socketRef.current) {
+      socketRef.current.close();
+      socketRef.current = null;
+    }
+
+    setIsActive(false);
+    updateStatus("idle");
+    console.log("Voice conversion stopped");
+  }, [updateStatus]);
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      stop();
+    };
+  }, [stop]);
+
+  return {
+    status,
+    start,
+    stop,
+    setTargetVoice,
+    sendReferenceAudio,
+    endReferenceCollection,
+    isActive,
+    availableVoices,
+    fetchVoices,
+  };
+};
+
+export default useVoiceConversion;