/** * useLiveKitRoom - Hook for LiveKit voice call with Julia AI * * IMPORTANT: This hook encapsulates ALL LiveKit logic. * The UI component should only use the returned state and actions. * * LOGGING: Maximum transparency - every step is logged! */ import { useState, useCallback, useRef, useEffect } from 'react'; import { Platform, AppState, AppStateStatus, NativeModules } from 'react-native'; import type { Room as RoomType } from 'livekit-client'; // Helper to detect iOS Simulator // Expo Go and production builds both work with this approach const isIOSSimulator = (): boolean => { if (Platform.OS !== 'ios') return false; // Check via DeviceInfo module if available const { PlatformConstants } = NativeModules; return PlatformConstants?.interfaceIdiom === 'simulator' || PlatformConstants?.isSimulator === true; }; import { getToken, VOICE_NAME } from '@/services/livekitService'; import { configureAudioForVoiceCall, stopAudioSession, reconfigureAudioForPlayback, } from '@/utils/audioSession'; // Connection states export type ConnectionState = | 'idle' | 'initializing' | 'configuring_audio' | 'requesting_token' | 'connecting' | 'connected' | 'reconnecting' | 'disconnecting' | 'disconnected' | 'error'; // Log entry type export interface LogEntry { timestamp: string; // Formatted time string (HH:MM:SS.mmm) level: 'info' | 'warn' | 'error' | 'success'; message: string; } // Hook options export interface UseLiveKitRoomOptions { userId: string; onTranscript?: (role: 'user' | 'assistant', text: string) => void; autoConnect?: boolean; } // Hook return type export interface UseLiveKitRoomReturn { // Connection state state: ConnectionState; error: string | null; // Call info roomName: string | null; callDuration: number; // Audio state isMuted: boolean; isAgentSpeaking: boolean; canPlayAudio: boolean; // Debug info logs: LogEntry[]; participantCount: number; // Actions connect: () => Promise; disconnect: () => Promise; toggleMute: () => Promise; clearLogs: () => void; } /** * Main hook for LiveKit voice calls */ export function useLiveKitRoom(options: UseLiveKitRoomOptions): UseLiveKitRoomReturn { const { userId, onTranscript, autoConnect = false } = options; // State const [state, setState] = useState('idle'); const [error, setError] = useState(null); const [roomName, setRoomName] = useState(null); const [callDuration, setCallDuration] = useState(0); const [isMuted, setIsMuted] = useState(false); const [isAgentSpeaking, setIsAgentSpeaking] = useState(false); const [canPlayAudio, setCanPlayAudio] = useState(false); const [logs, setLogs] = useState([]); const [participantCount, setParticipantCount] = useState(0); // Refs const roomRef = useRef(null); const callStartTimeRef = useRef(null); const connectionIdRef = useRef(0); const isUnmountingRef = useRef(false); const appStateRef = useRef(AppState.currentState); // =================== // LOGGING FUNCTIONS // =================== const log = useCallback((level: LogEntry['level'], message: string) => { const now = new Date(); const timestamp = now.toLocaleTimeString('en-US', { hour12: false, hour: '2-digit', minute: '2-digit', second: '2-digit', }) + '.' + now.getMilliseconds().toString().padStart(3, '0'); const entry: LogEntry = { timestamp, level, message, }; setLogs((prev) => [...prev, entry]); // Also log to console with color const prefix = `[LiveKit ${timestamp}]`; switch (level) { case 'error': console.error(`${prefix} ERROR: ${message}`); break; case 'warn': console.warn(`${prefix} WARN: ${message}`); break; case 'success': console.log(`${prefix} SUCCESS: ${message}`); break; default: console.log(`${prefix} INFO: ${message}`); } }, []); const logInfo = useCallback((msg: string) => log('info', msg), [log]); const logWarn = useCallback((msg: string) => log('warn', msg), [log]); const logError = useCallback((msg: string) => log('error', msg), [log]); const logSuccess = useCallback((msg: string) => log('success', msg), [log]); const clearLogs = useCallback(() => { setLogs([]); }, []); // =================== // CONNECT FUNCTION // =================== const connect = useCallback(async () => { // Prevent multiple concurrent connection attempts const currentConnectionId = ++connectionIdRef.current; logInfo('========== STARTING VOICE CALL =========='); logInfo(`User ID: ${userId}`); logInfo(`Platform: ${Platform.OS}`); logInfo(`Connection ID: ${currentConnectionId}`); // Check if already connected if (roomRef.current) { logWarn('Already connected to a room, disconnecting first...'); await roomRef.current.disconnect(); roomRef.current = null; } try { // ========== STEP 1: Initialize ========== setState('initializing'); logInfo('STEP 1/6: Initializing...'); // Detect simulator vs real device const isSimulator = isIOSSimulator(); logInfo(`Device type: ${isSimulator ? 'SIMULATOR' : 'REAL DEVICE'}`); logInfo(`Device model: ${Platform.OS} ${Platform.Version}`); if (isSimulator) { logWarn('⚠️ SIMULATOR DETECTED - Microphone will NOT work!'); logWarn('Simulator can only test: connection, token, agent presence, TTS playback'); logWarn('For full STT test, use a real iPhone device'); } // Check if connection was cancelled if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) { logWarn('Connection cancelled (component unmounting or new connection started)'); return; } // ========== STEP 2: Register WebRTC Globals ========== logInfo('STEP 2/6: Registering WebRTC globals...'); const { registerGlobals } = await import('@livekit/react-native'); if (typeof global.RTCPeerConnection === 'undefined') { logInfo('RTCPeerConnection not found, calling registerGlobals()...'); registerGlobals(); logSuccess('WebRTC globals registered!'); } else { logInfo('WebRTC globals already registered'); } // Check again if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) { logWarn('Connection cancelled after registerGlobals'); return; } // ========== STEP 3: Configure iOS Audio ========== setState('configuring_audio'); logInfo('STEP 3/6: Configuring iOS AudioSession...'); await configureAudioForVoiceCall(); logSuccess('iOS AudioSession configured!'); // Check again if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) { logWarn('Connection cancelled after audio config'); await stopAudioSession(); return; } // ========== STEP 4: Get Token ========== setState('requesting_token'); logInfo('STEP 4/6: Requesting token from server...'); const tokenResult = await getToken(userId); if (!tokenResult.success || !tokenResult.data) { const errorMsg = tokenResult.error || 'Failed to get token'; logError(`Token request failed: ${errorMsg}`); setError(errorMsg); setState('error'); return; } const { token, wsUrl, roomName: room } = tokenResult.data; setRoomName(room); logSuccess(`Token received!`); logInfo(` Room: ${room}`); logInfo(` WebSocket URL: ${wsUrl}`); logInfo(` Token length: ${token.length} chars`); // Check again if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) { logWarn('Connection cancelled after token'); await stopAudioSession(); return; } // ========== STEP 5: Import LiveKit and Create Room ========== logInfo('STEP 5/6: Creating LiveKit Room...'); const { Room, RoomEvent, ConnectionState: LKConnectionState, Track } = await import( 'livekit-client' ); logInfo(` Room class available: ${typeof Room === 'function'}`); logInfo(` RoomEvent available: ${typeof RoomEvent === 'object'}`); const lkRoom = new Room(); roomRef.current = lkRoom; logSuccess('Room instance created!'); // ========== Setup Event Listeners ========== logInfo('Setting up event listeners...'); // Connection state changes lkRoom.on(RoomEvent.ConnectionStateChanged, (newState) => { logInfo(`EVENT: ConnectionStateChanged -> ${newState}`); switch (newState) { case LKConnectionState.Connecting: setState('connecting'); break; case LKConnectionState.Connected: setState('connected'); logSuccess('Connected to room!'); if (!callStartTimeRef.current) { callStartTimeRef.current = Date.now(); logInfo('Call timer started'); } break; case LKConnectionState.Reconnecting: setState('reconnecting'); logWarn('Reconnecting...'); break; case LKConnectionState.Disconnected: setState('disconnected'); logInfo('Disconnected from room'); break; } }); // Track subscribed (audio from agent) lkRoom.on(RoomEvent.TrackSubscribed, async (track, publication, participant) => { logInfo(`EVENT: TrackSubscribed`); logInfo(` Track kind: ${track.kind}`); logInfo(` Track source: ${track.source}`); logInfo(` Participant: ${participant.identity}`); logInfo(` Publication SID: ${publication.trackSid}`); if (track.kind === Track.Kind.Audio) { logSuccess(`Audio track from ${participant.identity} - should hear voice now!`); setIsAgentSpeaking(true); // Reconfigure audio for playback logInfo('Reconfiguring audio for playback...'); await reconfigureAudioForPlayback(); } }); // Track unsubscribed lkRoom.on(RoomEvent.TrackUnsubscribed, (track, publication, participant) => { logInfo(`EVENT: TrackUnsubscribed`); logInfo(` Track kind: ${track.kind}`); logInfo(` Participant: ${participant.identity}`); if (track.kind === Track.Kind.Audio) { setIsAgentSpeaking(false); } }); // Track muted/unmuted lkRoom.on(RoomEvent.TrackMuted, (publication, participant) => { logInfo(`EVENT: TrackMuted - ${publication.trackSid} by ${participant.identity}`); }); lkRoom.on(RoomEvent.TrackUnmuted, (publication, participant) => { logInfo(`EVENT: TrackUnmuted - ${publication.trackSid} by ${participant.identity}`); }); // Participants lkRoom.on(RoomEvent.ParticipantConnected, (participant) => { logSuccess(`EVENT: ParticipantConnected - ${participant.identity}`); setParticipantCount((c) => c + 1); }); lkRoom.on(RoomEvent.ParticipantDisconnected, (participant) => { logInfo(`EVENT: ParticipantDisconnected - ${participant.identity}`); setParticipantCount((c) => Math.max(0, c - 1)); }); // Active speakers (voice activity) lkRoom.on(RoomEvent.ActiveSpeakersChanged, (speakers) => { if (speakers.length > 0) { const speakerNames = speakers.map((s: any) => s.identity).join(', '); logInfo(`EVENT: ActiveSpeakersChanged - ${speakerNames}`); // Check if agent is speaking const agentSpeaking = speakers.some((s: any) => s.identity.startsWith('agent')); setIsAgentSpeaking(agentSpeaking); } }); // Local track published (our mic) lkRoom.on(RoomEvent.LocalTrackPublished, (publication, participant) => { logSuccess(`EVENT: LocalTrackPublished`); logInfo(` Track: ${publication.trackSid}`); logInfo(` Kind: ${publication.kind}`); logInfo(` Source: ${publication.source}`); }); // Audio playback status lkRoom.on(RoomEvent.AudioPlaybackStatusChanged, () => { const canPlay = lkRoom.canPlaybackAudio; logInfo(`EVENT: AudioPlaybackStatusChanged - canPlaybackAudio: ${canPlay}`); setCanPlayAudio(canPlay); }); // Data received (transcripts) lkRoom.on(RoomEvent.DataReceived, (payload, participant) => { try { const data = JSON.parse(new TextDecoder().decode(payload)); logInfo(`EVENT: DataReceived from ${participant?.identity || 'unknown'}`); logInfo(` Type: ${data.type}`); if (data.type === 'transcript' && onTranscript) { logInfo(` Role: ${data.role}, Text: ${data.text?.substring(0, 50)}...`); onTranscript(data.role, data.text); } } catch (e) { // Non-JSON data, ignore } }); // Errors lkRoom.on(RoomEvent.Disconnected, (reason) => { logWarn(`EVENT: Disconnected - Reason: ${reason}`); }); // Check again before connect if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) { logWarn('Connection cancelled before room.connect()'); await stopAudioSession(); return; } // ========== STEP 6: Connect to Room ========== setState('connecting'); logInfo('STEP 6/6: Connecting to LiveKit room...'); logInfo(` URL: ${wsUrl}`); logInfo(` Room: ${room}`); await lkRoom.connect(wsUrl, token, { autoSubscribe: true, }); logSuccess('Connected to room!'); // Check if connection was cancelled after connect if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) { logWarn('Connection cancelled after room.connect()'); await lkRoom.disconnect(); await stopAudioSession(); return; } // ========== Enable Microphone ========== logInfo('Enabling microphone...'); try { await lkRoom.localParticipant.setMicrophoneEnabled(true); logSuccess('Microphone enabled!'); logInfo(` Local participant: ${lkRoom.localParticipant.identity}`); // Log track info - CRITICAL for debugging! const audioTracks = lkRoom.localParticipant.getTrackPublications(); logInfo(` Published tracks: ${audioTracks.length}`); let micTrackFound = false; audioTracks.forEach((pub) => { logInfo(` - ${pub.kind}: ${pub.trackSid} (${pub.source})`); logInfo(` isMuted: ${pub.isMuted}, isSubscribed: ${pub.isSubscribed}`); if (pub.kind === 'audio' && pub.source === 'microphone') { micTrackFound = true; const track = pub.track; if (track) { logInfo(` Track mediaStreamTrack: ${track.mediaStreamTrack?.readyState || 'N/A'}`); logInfo(` Track enabled: ${track.mediaStreamTrack?.enabled || 'N/A'}`); } else { logWarn(` WARNING: No track object on publication!`); } } }); if (!micTrackFound) { // Check if simulator const isSimulator = isIOSSimulator(); if (isSimulator) { logWarn('No microphone track - EXPECTED on simulator'); logInfo('Simulator test: check if Agent joined and TTS works'); } else { logError('CRITICAL: No microphone track published! STT will NOT work!'); logError('Possible causes: permissions denied, AudioSession not configured, hardware issue'); } } else { logSuccess('Microphone track found and published - STT should work'); } } catch (micError: any) { logError(`Failed to enable microphone: ${micError.message}`); logError(`Stack: ${micError.stack || 'N/A'}`); // This is CRITICAL - user must know! setError(`Microphone error: ${micError.message}`); } // Set initial participant count setParticipantCount(lkRoom.remoteParticipants.size); logInfo(`Remote participants: ${lkRoom.remoteParticipants.size}`); logSuccess('========== VOICE CALL STARTED =========='); } catch (err: any) { // Ignore errors if unmounting if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) { logWarn('Error ignored (component unmounting)'); return; } const errorMsg = err?.message || String(err); logError(`Connection failed: ${errorMsg}`); logError(`Stack: ${err?.stack || 'N/A'}`); setError(errorMsg); setState('error'); // Cleanup await stopAudioSession(); } }, [userId, onTranscript, logInfo, logWarn, logError, logSuccess]); // =================== // DISCONNECT FUNCTION // =================== const disconnect = useCallback(async () => { logInfo('========== DISCONNECTING =========='); setState('disconnecting'); try { if (roomRef.current) { logInfo('Disconnecting from room...'); await roomRef.current.disconnect(); roomRef.current = null; logSuccess('Disconnected from room'); } else { logInfo('No room to disconnect from'); } } catch (err: any) { logError(`Disconnect error: ${err.message}`); } logInfo('Stopping audio session...'); await stopAudioSession(); // Reset state setState('disconnected'); setRoomName(null); setIsMuted(false); setIsAgentSpeaking(false); setParticipantCount(0); callStartTimeRef.current = null; logSuccess('========== DISCONNECTED =========='); }, [logInfo, logError, logSuccess]); // =================== // TOGGLE MUTE // =================== const toggleMute = useCallback(async () => { if (!roomRef.current) { logWarn('Cannot toggle mute - not connected'); return; } const newMuted = !isMuted; logInfo(`Toggling mute: ${isMuted} -> ${newMuted}`); try { await roomRef.current.localParticipant.setMicrophoneEnabled(!newMuted); setIsMuted(newMuted); logSuccess(`Microphone ${newMuted ? 'muted' : 'unmuted'}`); } catch (err: any) { logError(`Failed to toggle mute: ${err.message}`); } }, [isMuted, logInfo, logWarn, logError, logSuccess]); // =================== // CALL DURATION TIMER // =================== useEffect(() => { if (state !== 'connected') return; const interval = setInterval(() => { if (callStartTimeRef.current) { const elapsed = Math.floor((Date.now() - callStartTimeRef.current) / 1000); setCallDuration(elapsed); } }, 1000); return () => clearInterval(interval); }, [state]); // =================== // APP STATE HANDLING // =================== useEffect(() => { const handleAppStateChange = (nextAppState: AppStateStatus) => { const prevState = appStateRef.current; appStateRef.current = nextAppState; if (prevState.match(/inactive|background/) && nextAppState === 'active') { logInfo('App returned to foreground'); } else if (prevState === 'active' && nextAppState.match(/inactive|background/)) { logInfo('App went to background - call continues in background'); } }; const subscription = AppState.addEventListener('change', handleAppStateChange); return () => subscription.remove(); }, [logInfo]); // =================== // CLEANUP ON UNMOUNT // =================== useEffect(() => { isUnmountingRef.current = false; return () => { isUnmountingRef.current = true; // Cleanup const cleanup = async () => { if (roomRef.current) { try { await roomRef.current.disconnect(); } catch (e) { // Ignore } roomRef.current = null; } await stopAudioSession(); }; cleanup(); }; }, []); // =================== // AUTO CONNECT // =================== useEffect(() => { if (autoConnect && state === 'idle') { connect(); } }, [autoConnect, state, connect]); // =================== // RETURN // =================== return { // Connection state state, error, // Call info roomName, callDuration, // Audio state isMuted, isAgentSpeaking, canPlayAudio, // Debug logs, participantCount, // Actions connect, disconnect, toggleMute, clearLogs, }; } export { VOICE_NAME };