diff --git a/ai/ai-react-app/src/views/LiveView.module.css b/ai/ai-react-app/src/views/LiveView.module.css index 65396255c..bb4f5612f 100644 --- a/ai/ai-react-app/src/views/LiveView.module.css +++ b/ai/ai-react-app/src/views/LiveView.module.css @@ -88,6 +88,47 @@ white-space: pre-wrap; } +.controlsContainer { + display: flex; + flex-direction: column; + gap: 16px; + margin-top: 20px; + padding-top: 20px; + border-top: 1px solid var(--color-border-primary); + width: 100%; + max-width: 500px; + align-items: center; +} + +.videoControls { + display: flex; + gap: 10px; + align-items: center; + justify-content: center; + width: 100%; +} + +.videoSourceSelect { + padding: 12px 16px; + background-color: var(--color-surface-primary); + border: 1px solid var(--color-border-secondary); + color: var(--color-text-primary); + border-radius: 24px; + font-size: 1rem; + cursor: pointer; + transition: border-color 0.15s ease; +} + +.videoSourceSelect:hover { + border-color: var(--brand-gray-50); +} + +.videoSourceSelect:disabled { + opacity: 0.5; + cursor: not-allowed; + background-color: var(--color-surface-tertiary); +} + @keyframes pulse { 0% { box-shadow: 0 0 0 0 rgba(52, 168, 83, 0.7); @@ -98,4 +139,4 @@ 100% { box-shadow: 0 0 0 0 rgba(52, 168, 83, 0); } -} \ No newline at end of file +} diff --git a/ai/ai-react-app/src/views/LiveView.tsx b/ai/ai-react-app/src/views/LiveView.tsx index d328d94b3..9effb4fcc 100644 --- a/ai/ai-react-app/src/views/LiveView.tsx +++ b/ai/ai-react-app/src/views/LiveView.tsx @@ -4,7 +4,10 @@ import { AI, getLiveGenerativeModel, startAudioConversation, + startVideoRecording, AudioConversationController, + VideoRecordingController, + LiveSession, AIError, ResponseModality, } from "firebase/ai"; @@ -14,18 +17,27 @@ interface LiveViewProps { aiInstance: AI; } -type ConversationState = "idle" | "active" | "error"; +type SessionState = "idle" | "connecting" | "connected" | "error"; +type VideoSource = "camera" | "screen"; const LiveView: React.FC = ({ aiInstance }) => { - const [conversationState, setConversationState] = - useState("idle"); + const [sessionState, setSessionState] = useState("idle"); const [error, setError] = useState(null); - const [controller, setController] = + + const [liveSession, setLiveSession] = useState(null); + const [audioController, setAudioController] = useState(null); + const [videoController, setVideoController] = + useState(null); + + const [videoSource, setVideoSource] = useState("camera"); + + const isAudioRunning = audioController !== null; + const isVideoRunning = videoController !== null; - const handleStartConversation = useCallback(async () => { + const handleConnect = useCallback(async () => { setError(null); - setConversationState("active"); + setSessionState("connecting"); try { const modelName = LIVE_MODELS.get(aiInstance.backend.backendType)!; @@ -33,23 +45,17 @@ const LiveView: React.FC = ({ aiInstance }) => { const model = getLiveGenerativeModel(aiInstance, { model: modelName, generationConfig: { - responseModalities: [ResponseModality.AUDIO] - } + responseModalities: [ResponseModality.AUDIO], + }, }); console.log("[LiveView] Connecting to live session..."); - const liveSession = await model.connect(); - - console.log( - "[LiveView] Starting audio conversation. This will request microphone permissions.", - ); - - const newController = await startAudioConversation(liveSession); - - setController(newController); - console.log("[LiveView] Audio conversation started successfully."); + const session = await model.connect(); + setLiveSession(session); + setSessionState("connected"); + console.log("[LiveView] Live session connected successfully."); } catch (err: unknown) { - console.error("[LiveView] Failed to start conversation:", err); + console.error("[LiveView] Failed to connect to live session:", err); let errorMessage = "An unknown error occurred."; if (err instanceof AIError) { errorMessage = `Error (${err.code}): ${err.message}`; @@ -57,39 +63,104 @@ const LiveView: React.FC = ({ aiInstance }) => { errorMessage = err.message; } setError(errorMessage); - setConversationState("error"); - setController(null); // Ensure controller is cleared on error + setSessionState("error"); + setLiveSession(null); } }, [aiInstance]); - const handleStopConversation = useCallback(async () => { - if (!controller) return; + const handleDisconnect = useCallback(async () => { + console.log("[LiveView] Disconnecting live session..."); + if (audioController) { + await audioController.stop(); + setAudioController(null); + } + if (videoController) { + await videoController.stop(); + setVideoController(null); + } + // The liveSession does not have an explicit close() method in the public API. + // Resources are released when the controllers are stopped. + setLiveSession(null); + setSessionState("idle"); + console.log("[LiveView] Live session disconnected."); + }, [audioController, videoController]); + + const handleToggleAudio = useCallback(async () => { + if (!liveSession) return; + + if (isAudioRunning) { + console.log("[LiveView] Stopping audio conversation..."); + await audioController?.stop(); + setAudioController(null); + console.log("[LiveView] Audio conversation stopped."); + } else { + console.log( + "[LiveView] Starting audio conversation. This may request microphone permissions.", + ); + try { + const newController = await startAudioConversation(liveSession); + setAudioController(newController); + console.log("[LiveView] Audio conversation started successfully."); + } catch (err: unknown) { + console.error("[LiveView] Failed to start audio conversation:", err); + setError( + err instanceof Error + ? err.message + : "Failed to start audio conversation.", + ); + } + } + }, [liveSession, audioController, isAudioRunning]); - console.log("[LiveView] Stopping audio conversation..."); - await controller.stop(); - setController(null); - setConversationState("idle"); - console.log("[LiveView] Audio conversation stopped."); - }, [controller]); + const handleToggleVideo = useCallback(async () => { + if (!liveSession) return; - // Cleanup effect to stop the conversation if the component unmounts + if (isVideoRunning) { + console.log("[LiveView] Stopping video recording..."); + await videoController?.stop(); + setVideoController(null); + console.log("[LiveView] Video recording stopped."); + } else { + console.log( + `[LiveView] Starting video recording with source: ${videoSource}. This may request permissions.`, + ); + try { + const newController = await startVideoRecording(liveSession, { + videoSource, + }); + setVideoController(newController); + console.log("[LiveView] Video recording started successfully."); + } catch (err: unknown) { + console.error("[LiveView] Failed to start video recording:", err); + setError( + err instanceof Error + ? err.message + : "Failed to start video recording.", + ); + } + } + }, [liveSession, videoController, isVideoRunning, videoSource]); + + // Cleanup effect to disconnect if the component unmounts useEffect(() => { return () => { - if (controller) { + if (liveSession && liveSession.isClosed) { console.log( - "[LiveView] Component unmounting, stopping active conversation.", + "[LiveView] Component unmounting, disconnecting live session.", ); - controller.stop(); + handleDisconnect(); } }; - }, [controller]); + }, [liveSession, handleDisconnect]); const getStatusText = () => { - switch (conversationState) { + switch (sessionState) { case "idle": - return "Ready"; - case "active": - return "In Conversation"; + return "Idle"; + case "connecting": + return "Connecting..."; + case "connected": + return "Connected"; case "error": return "Error"; default: @@ -99,16 +170,16 @@ const LiveView: React.FC = ({ aiInstance }) => { return (
-

Live Conversation

+

Gemini Live

- Click the button below to start a real-time voice conversation with the - model. Your browser will ask for microphone permissions. + Connect to a live session, then start audio and/or video streams. + Ensure you grant microphone and camera/screen permissions when prompted.

Status: {getStatusText()} @@ -116,20 +187,55 @@ const LiveView: React.FC = ({ aiInstance }) => { + {sessionState === "connected" && ( +
+ {/* Audio Controls */} + + + {/* Video Controls */} +
+ + +
+
+ )} + {error &&
{error}
}
);