wellnua-lite/hooks/useLiveKitRoom.ts
Sergei f2e633df99 Fix audio playback: add room.startAudio() call
Root cause: Audio from remote participant (Julia AI) was not playing
because room.startAudio() was never called after connecting.

This is REQUIRED by LiveKit WebRTC to enable audio playback.
The fix matches the working implementation in debug.tsx (Robert version).

Changes:
- Add room.startAudio() call after room.connect()
- Add canPlayAudio state tracking
- Add proper error handling for startAudio

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-25 18:03:56 -08:00

708 lines
22 KiB
TypeScript

/**
* useLiveKitRoom - Hook for LiveKit voice call with Julia AI
*
* IMPORTANT: This hook encapsulates ALL LiveKit logic.
* The UI component should only use the returned state and actions.
*
* LOGGING: Maximum transparency - every step is logged!
*/
import { useState, useCallback, useRef, useEffect } from 'react';
import { Platform, AppState, AppStateStatus, NativeModules } from 'react-native';
import type { Room as RoomType } from 'livekit-client';
// Helper to detect iOS Simulator
// Expo Go and production builds both work with this approach
const isIOSSimulator = (): boolean => {
if (Platform.OS !== 'ios') return false;
// Check via DeviceInfo module if available
const { PlatformConstants } = NativeModules;
return PlatformConstants?.interfaceIdiom === 'simulator' ||
PlatformConstants?.isSimulator === true;
};
import { getToken, VOICE_NAME, BeneficiaryData } from '@/services/livekitService';
import {
configureAudioForVoiceCall,
stopAudioSession,
reconfigureAudioForPlayback,
} from '@/utils/audioSession';
import { callManager } from '@/services/callManager';
// Connection states
export type ConnectionState =
| 'idle'
| 'initializing'
| 'configuring_audio'
| 'requesting_token'
| 'connecting'
| 'connected'
| 'reconnecting'
| 'disconnecting'
| 'disconnected'
| 'error';
// Log entry type
export interface LogEntry {
timestamp: string; // Formatted time string (HH:MM:SS.mmm)
level: 'info' | 'warn' | 'error' | 'success';
message: string;
}
// Hook options
export interface UseLiveKitRoomOptions {
userId: string;
beneficiaryData?: BeneficiaryData;
onTranscript?: (role: 'user' | 'assistant', text: string) => void;
autoConnect?: boolean;
}
// Hook return type
export interface UseLiveKitRoomReturn {
// Connection state
state: ConnectionState;
error: string | null;
// Call info
roomName: string | null;
callDuration: number;
// Audio state
isMuted: boolean;
isAgentSpeaking: boolean;
canPlayAudio: boolean;
// Debug info
logs: LogEntry[];
participantCount: number;
// Actions
connect: () => Promise<void>;
disconnect: () => Promise<void>;
toggleMute: () => Promise<void>;
clearLogs: () => void;
}
/**
* Main hook for LiveKit voice calls
*/
export function useLiveKitRoom(options: UseLiveKitRoomOptions): UseLiveKitRoomReturn {
const { userId, beneficiaryData, onTranscript, autoConnect = false } = options;
// State
const [state, setState] = useState<ConnectionState>('idle');
const [error, setError] = useState<string | null>(null);
const [roomName, setRoomName] = useState<string | null>(null);
const [callDuration, setCallDuration] = useState(0);
const [isMuted, setIsMuted] = useState(false);
const [isAgentSpeaking, setIsAgentSpeaking] = useState(false);
const [canPlayAudio, setCanPlayAudio] = useState(false);
const [logs, setLogs] = useState<LogEntry[]>([]);
const [participantCount, setParticipantCount] = useState(0);
// Refs
const roomRef = useRef<RoomType | null>(null);
const callStartTimeRef = useRef<number | null>(null);
const connectionIdRef = useRef(0);
const isUnmountingRef = useRef(false);
const appStateRef = useRef<AppStateStatus>(AppState.currentState);
const callIdRef = useRef<string | null>(null);
// ===================
// LOGGING FUNCTIONS
// ===================
const log = useCallback((level: LogEntry['level'], message: string) => {
const now = new Date();
const timestamp = now.toLocaleTimeString('en-US', {
hour12: false,
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
}) + '.' + now.getMilliseconds().toString().padStart(3, '0');
const entry: LogEntry = {
timestamp,
level,
message,
};
setLogs((prev) => [...prev, entry]);
// Also log to console with color
const prefix = `[LiveKit ${timestamp}]`;
switch (level) {
case 'error':
console.error(`${prefix} ERROR: ${message}`);
break;
case 'warn':
console.warn(`${prefix} WARN: ${message}`);
break;
case 'success':
console.log(`${prefix} SUCCESS: ${message}`);
break;
default:
console.log(`${prefix} INFO: ${message}`);
}
}, []);
const logInfo = useCallback((msg: string) => log('info', msg), [log]);
const logWarn = useCallback((msg: string) => log('warn', msg), [log]);
const logError = useCallback((msg: string) => log('error', msg), [log]);
const logSuccess = useCallback((msg: string) => log('success', msg), [log]);
const clearLogs = useCallback(() => {
setLogs([]);
}, []);
// ===================
// CONNECT FUNCTION
// ===================
const connect = useCallback(async () => {
// Prevent multiple concurrent connection attempts
const currentConnectionId = ++connectionIdRef.current;
// Generate unique call ID for this session
const callId = `call-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
callIdRef.current = callId;
logInfo('========== STARTING VOICE CALL ==========');
logInfo(`User ID: ${userId}`);
logInfo(`Platform: ${Platform.OS}`);
logInfo(`Connection ID: ${currentConnectionId}`);
logInfo(`Call ID: ${callId}`);
// Register with CallManager - this will disconnect any existing call
logInfo('Registering call with CallManager...');
await callManager.registerCall(callId, async () => {
logInfo('CallManager requested disconnect (another call starting)');
if (roomRef.current) {
await roomRef.current.disconnect();
roomRef.current = null;
}
await stopAudioSession();
});
logSuccess('Call registered with CallManager');
// Check if already connected
if (roomRef.current) {
logWarn('Already connected to a room, disconnecting first...');
await roomRef.current.disconnect();
roomRef.current = null;
}
try {
// ========== STEP 1: Initialize ==========
setState('initializing');
logInfo('STEP 1/6: Initializing...');
// Detect simulator vs real device
const isSimulator = isIOSSimulator();
logInfo(`Device type: ${isSimulator ? 'SIMULATOR' : 'REAL DEVICE'}`);
logInfo(`Device model: ${Platform.OS} ${Platform.Version}`);
if (isSimulator) {
logWarn('⚠️ SIMULATOR DETECTED - Microphone will NOT work!');
logWarn('Simulator can only test: connection, token, agent presence, TTS playback');
logWarn('For full STT test, use a real iPhone device');
}
// Check if connection was cancelled
if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) {
logWarn('Connection cancelled (component unmounting or new connection started)');
return;
}
// ========== STEP 2: Register WebRTC Globals ==========
logInfo('STEP 2/6: Registering WebRTC globals...');
const { registerGlobals } = await import('@livekit/react-native');
if (typeof global.RTCPeerConnection === 'undefined') {
logInfo('RTCPeerConnection not found, calling registerGlobals()...');
registerGlobals();
logSuccess('WebRTC globals registered!');
} else {
logInfo('WebRTC globals already registered');
}
// Check again
if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) {
logWarn('Connection cancelled after registerGlobals');
return;
}
// ========== STEP 3: Configure iOS Audio ==========
setState('configuring_audio');
logInfo('STEP 3/6: Configuring iOS AudioSession...');
await configureAudioForVoiceCall();
logSuccess('iOS AudioSession configured!');
// Check again
if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) {
logWarn('Connection cancelled after audio config');
await stopAudioSession();
return;
}
// ========== STEP 4: Get Token ==========
setState('requesting_token');
logInfo('STEP 4/6: Requesting token from server...');
const tokenResult = await getToken(userId, beneficiaryData);
if (!tokenResult.success || !tokenResult.data) {
const errorMsg = tokenResult.error || 'Failed to get token';
logError(`Token request failed: ${errorMsg}`);
setError(errorMsg);
setState('error');
return;
}
const { token, wsUrl, roomName: room } = tokenResult.data;
setRoomName(room);
logSuccess(`Token received!`);
logInfo(` Room: ${room}`);
logInfo(` WebSocket URL: ${wsUrl}`);
logInfo(` Token length: ${token.length} chars`);
// Check again
if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) {
logWarn('Connection cancelled after token');
await stopAudioSession();
return;
}
// ========== STEP 5: Import LiveKit and Create Room ==========
logInfo('STEP 5/6: Creating LiveKit Room...');
const { Room, RoomEvent, ConnectionState: LKConnectionState, Track } = await import(
'livekit-client'
);
logInfo(` Room class available: ${typeof Room === 'function'}`);
logInfo(` RoomEvent available: ${typeof RoomEvent === 'object'}`);
const lkRoom = new Room();
roomRef.current = lkRoom;
logSuccess('Room instance created!');
// ========== Setup Event Listeners ==========
logInfo('Setting up event listeners...');
// Connection state changes
lkRoom.on(RoomEvent.ConnectionStateChanged, (newState) => {
logInfo(`EVENT: ConnectionStateChanged -> ${newState}`);
switch (newState) {
case LKConnectionState.Connecting:
setState('connecting');
break;
case LKConnectionState.Connected:
setState('connected');
logSuccess('Connected to room!');
if (!callStartTimeRef.current) {
callStartTimeRef.current = Date.now();
logInfo('Call timer started');
}
break;
case LKConnectionState.Reconnecting:
setState('reconnecting');
logWarn('Reconnecting...');
break;
case LKConnectionState.Disconnected:
setState('disconnected');
logInfo('Disconnected from room');
break;
}
});
// Track subscribed (audio from agent)
lkRoom.on(RoomEvent.TrackSubscribed, async (track, publication, participant) => {
logInfo(`EVENT: TrackSubscribed`);
logInfo(` Track kind: ${track.kind}`);
logInfo(` Track source: ${track.source}`);
logInfo(` Participant: ${participant.identity}`);
logInfo(` Publication SID: ${publication.trackSid}`);
if (track.kind === Track.Kind.Audio) {
logSuccess(`Audio track from ${participant.identity} - should hear voice now!`);
setIsAgentSpeaking(true);
// Reconfigure audio for playback
logInfo('Reconfiguring audio for playback...');
await reconfigureAudioForPlayback();
}
});
// Track unsubscribed
lkRoom.on(RoomEvent.TrackUnsubscribed, (track, publication, participant) => {
logInfo(`EVENT: TrackUnsubscribed`);
logInfo(` Track kind: ${track.kind}`);
logInfo(` Participant: ${participant.identity}`);
if (track.kind === Track.Kind.Audio) {
setIsAgentSpeaking(false);
}
});
// Track muted/unmuted
lkRoom.on(RoomEvent.TrackMuted, (publication, participant) => {
logInfo(`EVENT: TrackMuted - ${publication.trackSid} by ${participant.identity}`);
});
lkRoom.on(RoomEvent.TrackUnmuted, (publication, participant) => {
logInfo(`EVENT: TrackUnmuted - ${publication.trackSid} by ${participant.identity}`);
});
// Participants
lkRoom.on(RoomEvent.ParticipantConnected, (participant) => {
logSuccess(`EVENT: ParticipantConnected - ${participant.identity}`);
setParticipantCount((c) => c + 1);
});
lkRoom.on(RoomEvent.ParticipantDisconnected, (participant) => {
logInfo(`EVENT: ParticipantDisconnected - ${participant.identity}`);
setParticipantCount((c) => Math.max(0, c - 1));
});
// Active speakers (voice activity)
lkRoom.on(RoomEvent.ActiveSpeakersChanged, (speakers) => {
if (speakers.length > 0) {
const speakerNames = speakers.map((s: any) => s.identity).join(', ');
logInfo(`EVENT: ActiveSpeakersChanged - ${speakerNames}`);
// Check if agent is speaking
const agentSpeaking = speakers.some((s: any) => s.identity.startsWith('agent'));
setIsAgentSpeaking(agentSpeaking);
}
});
// Local track published (our mic)
lkRoom.on(RoomEvent.LocalTrackPublished, (publication, participant) => {
logSuccess(`EVENT: LocalTrackPublished`);
logInfo(` Track: ${publication.trackSid}`);
logInfo(` Kind: ${publication.kind}`);
logInfo(` Source: ${publication.source}`);
});
// Audio playback status
lkRoom.on(RoomEvent.AudioPlaybackStatusChanged, () => {
const canPlay = lkRoom.canPlaybackAudio;
logInfo(`EVENT: AudioPlaybackStatusChanged - canPlaybackAudio: ${canPlay}`);
setCanPlayAudio(canPlay);
});
// Data received (transcripts)
lkRoom.on(RoomEvent.DataReceived, (payload, participant) => {
try {
const data = JSON.parse(new TextDecoder().decode(payload));
logInfo(`EVENT: DataReceived from ${participant?.identity || 'unknown'}`);
logInfo(` Type: ${data.type}`);
if (data.type === 'transcript' && onTranscript) {
logInfo(` Role: ${data.role}, Text: ${data.text?.substring(0, 50)}...`);
onTranscript(data.role, data.text);
}
} catch (e) {
// Non-JSON data, ignore
}
});
// Errors
lkRoom.on(RoomEvent.Disconnected, (reason) => {
logWarn(`EVENT: Disconnected - Reason: ${reason}`);
});
// Check again before connect
if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) {
logWarn('Connection cancelled before room.connect()');
await stopAudioSession();
return;
}
// ========== STEP 6: Connect to Room ==========
setState('connecting');
logInfo('STEP 6/6: Connecting to LiveKit room...');
logInfo(` URL: ${wsUrl}`);
logInfo(` Room: ${room}`);
await lkRoom.connect(wsUrl, token, {
autoSubscribe: true,
});
logSuccess('Connected to room!');
// ========== CRITICAL: Start Audio Playback ==========
// This is REQUIRED for audio to play on iOS and Android!
// Without this call, remote audio tracks will NOT be heard.
logInfo('Starting audio playback (room.startAudio)...');
try {
await lkRoom.startAudio();
logSuccess(`Audio playback started! canPlaybackAudio: ${lkRoom.canPlaybackAudio}`);
setCanPlayAudio(lkRoom.canPlaybackAudio);
} catch (audioPlaybackErr: any) {
logError(`startAudio failed: ${audioPlaybackErr.message}`);
// Don't fail the whole call - audio might still work on some platforms
}
// Check if connection was cancelled after connect
if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) {
logWarn('Connection cancelled after room.connect()');
await lkRoom.disconnect();
await stopAudioSession();
return;
}
// ========== Enable Microphone ==========
logInfo('Enabling microphone...');
try {
await lkRoom.localParticipant.setMicrophoneEnabled(true);
logSuccess('Microphone enabled!');
logInfo(` Local participant: ${lkRoom.localParticipant.identity}`);
// Log track info - CRITICAL for debugging!
const audioTracks = lkRoom.localParticipant.getTrackPublications();
logInfo(` Published tracks: ${audioTracks.length}`);
let micTrackFound = false;
audioTracks.forEach((pub) => {
logInfo(` - ${pub.kind}: ${pub.trackSid} (${pub.source})`);
logInfo(` isMuted: ${pub.isMuted}, isSubscribed: ${pub.isSubscribed}`);
if (pub.kind === 'audio' && pub.source === 'microphone') {
micTrackFound = true;
const track = pub.track;
if (track) {
logInfo(` Track mediaStreamTrack: ${track.mediaStreamTrack?.readyState || 'N/A'}`);
logInfo(` Track enabled: ${track.mediaStreamTrack?.enabled || 'N/A'}`);
} else {
logWarn(` WARNING: No track object on publication!`);
}
}
});
if (!micTrackFound) {
// Check if simulator
const isSimulator = isIOSSimulator();
if (isSimulator) {
logWarn('No microphone track - EXPECTED on simulator');
logInfo('Simulator test: check if Agent joined and TTS works');
} else {
logError('CRITICAL: No microphone track published! STT will NOT work!');
logError('Possible causes: permissions denied, AudioSession not configured, hardware issue');
}
} else {
logSuccess('Microphone track found and published - STT should work');
}
} catch (micError: any) {
logError(`Failed to enable microphone: ${micError.message}`);
logError(`Stack: ${micError.stack || 'N/A'}`);
// This is CRITICAL - user must know!
setError(`Microphone error: ${micError.message}`);
}
// Set initial participant count
setParticipantCount(lkRoom.remoteParticipants.size);
logInfo(`Remote participants: ${lkRoom.remoteParticipants.size}`);
logSuccess('========== VOICE CALL STARTED ==========');
} catch (err: any) {
// Ignore errors if unmounting
if (isUnmountingRef.current || currentConnectionId !== connectionIdRef.current) {
logWarn('Error ignored (component unmounting)');
return;
}
const errorMsg = err?.message || String(err);
logError(`Connection failed: ${errorMsg}`);
logError(`Stack: ${err?.stack || 'N/A'}`);
setError(errorMsg);
setState('error');
// Cleanup
await stopAudioSession();
}
}, [userId, beneficiaryData, onTranscript, logInfo, logWarn, logError, logSuccess]);
// ===================
// DISCONNECT FUNCTION
// ===================
const disconnect = useCallback(async () => {
logInfo('========== DISCONNECTING ==========');
setState('disconnecting');
// Unregister from CallManager
if (callIdRef.current) {
logInfo(`Unregistering call: ${callIdRef.current}`);
callManager.unregisterCall(callIdRef.current);
callIdRef.current = null;
}
try {
if (roomRef.current) {
logInfo('Disconnecting from room...');
await roomRef.current.disconnect();
roomRef.current = null;
logSuccess('Disconnected from room');
} else {
logInfo('No room to disconnect from');
}
} catch (err: any) {
logError(`Disconnect error: ${err.message}`);
}
logInfo('Stopping audio session...');
await stopAudioSession();
// Reset state
setState('disconnected');
setRoomName(null);
setIsMuted(false);
setIsAgentSpeaking(false);
setParticipantCount(0);
callStartTimeRef.current = null;
logSuccess('========== DISCONNECTED ==========');
}, [logInfo, logError, logSuccess]);
// ===================
// TOGGLE MUTE
// ===================
const toggleMute = useCallback(async () => {
if (!roomRef.current) {
logWarn('Cannot toggle mute - not connected');
return;
}
const newMuted = !isMuted;
logInfo(`Toggling mute: ${isMuted} -> ${newMuted}`);
try {
await roomRef.current.localParticipant.setMicrophoneEnabled(!newMuted);
setIsMuted(newMuted);
logSuccess(`Microphone ${newMuted ? 'muted' : 'unmuted'}`);
} catch (err: any) {
logError(`Failed to toggle mute: ${err.message}`);
}
}, [isMuted, logInfo, logWarn, logError, logSuccess]);
// ===================
// CALL DURATION TIMER
// ===================
useEffect(() => {
if (state !== 'connected') return;
const interval = setInterval(() => {
if (callStartTimeRef.current) {
const elapsed = Math.floor((Date.now() - callStartTimeRef.current) / 1000);
setCallDuration(elapsed);
}
}, 1000);
return () => clearInterval(interval);
}, [state]);
// ===================
// APP STATE HANDLING
// ===================
useEffect(() => {
const handleAppStateChange = (nextAppState: AppStateStatus) => {
const prevState = appStateRef.current;
appStateRef.current = nextAppState;
if (prevState.match(/inactive|background/) && nextAppState === 'active') {
logInfo('App returned to foreground');
} else if (prevState === 'active' && nextAppState.match(/inactive|background/)) {
logInfo('App went to background - call continues in background');
}
};
const subscription = AppState.addEventListener('change', handleAppStateChange);
return () => subscription.remove();
}, [logInfo]);
// ===================
// CLEANUP ON UNMOUNT
// ===================
useEffect(() => {
isUnmountingRef.current = false;
return () => {
isUnmountingRef.current = true;
// Cleanup
const cleanup = async () => {
// Unregister from CallManager
if (callIdRef.current) {
callManager.unregisterCall(callIdRef.current);
callIdRef.current = null;
}
if (roomRef.current) {
try {
await roomRef.current.disconnect();
} catch (e) {
// Ignore
}
roomRef.current = null;
}
await stopAudioSession();
};
cleanup();
};
}, []);
// ===================
// AUTO CONNECT
// ===================
useEffect(() => {
if (autoConnect && state === 'idle') {
connect();
}
}, [autoConnect, state, connect]);
// ===================
// RETURN
// ===================
return {
// Connection state
state,
error,
// Call info
roomName,
callDuration,
// Audio state
isMuted,
isAgentSpeaking,
canPlayAudio,
// Debug
logs,
participantCount,
// Actions
connect,
disconnect,
toggleMute,
clearLogs,
};
}
export { VOICE_NAME };