/** * Realtime Voice Session - React Browser Client * * Demonstrates: * - Creating a voice session in the browser * - Using WebRTC transport for low latency * - Handling audio I/O automatically * - Managing session lifecycle * - Displaying transcripts and tool calls * * IMPORTANT: Generate ephemeral API keys server-side, never expose your main API key */ import React, { useState, useEffect, useRef } from 'react'; import { RealtimeSession, RealtimeAgent } from '@openai/agents-realtime'; import { z } from 'zod'; // ======================================== // Voice Agent Definition // ======================================== import { tool } from '@openai/agents-realtime'; const weatherTool = tool({ name: 'get_weather', description: 'Get weather for a city', parameters: z.object({ city: z.string(), }), execute: async ({ city }) => { // Call your backend API const response = await fetch(`/api/weather?city=${city}`); const data = await response.json(); return data.weather; }, }); const voiceAgent = new RealtimeAgent({ name: 'Voice Assistant', instructions: 'You are a helpful voice assistant. Keep responses concise and friendly.', tools: [weatherTool], voice: 'alloy', }); // ======================================== // React Component // ======================================== interface Message { role: 'user' | 'assistant'; content: string; timestamp: Date; } interface ToolCall { name: string; arguments: Record; result?: any; } export function VoiceAssistant() { const [isConnected, setIsConnected] = useState(false); const [isListening, setIsListening] = useState(false); const [messages, setMessages] = useState([]); const [toolCalls, setToolCalls] = useState([]); const [error, setError] = useState(null); const sessionRef = useRef(null); // ======================================== // Initialize Session // ======================================== useEffect(() => { let session: RealtimeSession; async function initSession() { try { // Get ephemeral API key from your backend const response = await fetch('/api/generate-session-key'); const { apiKey } = await response.json(); // Create session with WebRTC transport (low latency) session = new RealtimeSession(voiceAgent, { apiKey, transport: 'webrtc', // or 'websocket' }); sessionRef.current = session; // ======================================== // Session Event Handlers // ======================================== session.on('connected', () => { console.log('βœ… Connected to voice session'); setIsConnected(true); setError(null); }); session.on('disconnected', () => { console.log('πŸ”Œ Disconnected from voice session'); setIsConnected(false); setIsListening(false); }); session.on('error', (err) => { console.error('❌ Session error:', err); setError(err.message); }); // ======================================== // Transcription Events // ======================================== session.on('audio.transcription.completed', (event) => { // User finished speaking setMessages(prev => [...prev, { role: 'user', content: event.transcript, timestamp: new Date(), }]); setIsListening(false); }); session.on('audio.transcription.started', () => { // User started speaking setIsListening(true); }); session.on('agent.audio.done', (event) => { // Agent finished speaking setMessages(prev => [...prev, { role: 'assistant', content: event.transcript, timestamp: new Date(), }]); }); // ======================================== // Tool Call Events // ======================================== session.on('tool.call', (event) => { console.log('πŸ› οΈ Tool call:', event.name, event.arguments); setToolCalls(prev => [...prev, { name: event.name, arguments: event.arguments, }]); }); session.on('tool.result', (event) => { console.log('βœ… Tool result:', event.result); setToolCalls(prev => prev.map(tc => tc.name === event.name ? { ...tc, result: event.result } : tc )); }); // Connect to start session await session.connect(); } catch (err: any) { console.error('Failed to initialize session:', err); setError(err.message); } } initSession(); // Cleanup on unmount return () => { if (session) { session.disconnect(); } }; }, []); // ======================================== // Manual Control Functions // ======================================== const handleInterrupt = () => { if (sessionRef.current) { sessionRef.current.interrupt(); } }; const handleDisconnect = () => { if (sessionRef.current) { sessionRef.current.disconnect(); } }; // ======================================== // Render UI // ======================================== return (
{isConnected ? '🟒 Connected' : 'πŸ”΄ Disconnected'}
{isListening &&
🎀 Listening...
}
{error && (
❌ Error: {error}
)}
{messages.map((msg, i) => (
{msg.role === 'user' ? 'πŸ‘€' : 'πŸ€–'}

{msg.content}

{msg.timestamp.toLocaleTimeString()}
))}
{toolCalls.length > 0 && (

πŸ› οΈ Tool Calls

{toolCalls.map((tc, i) => (
{tc.name}
{JSON.stringify(tc.arguments, null, 2)}
{tc.result && (
Result: {JSON.stringify(tc.result)}
)}
))}
)}
); } export default VoiceAssistant;