Sergei dde0ecb9cd Add Julia AI voice agent with LiveKit integration
Voice AI Features:
- LiveKit Agents integration for real-time voice calls
- Julia AI agent (Python) deployed to LiveKit Cloud
- Token server for authentication
- Debug screen with voice call testing
- Voice call screen with full-screen UI

Agent Configuration:
- STT: Deepgram Nova-2
- LLM: OpenAI GPT-4o
- TTS: Deepgram Aura Asteria (female voice)
- Turn Detection: LiveKit Multilingual Model
- VAD: Silero
- Noise Cancellation: LiveKit BVC

Files added:
- julia-agent/ - Complete agent code and token server
- app/voice-call.tsx - Full-screen voice call UI
- services/livekitService.ts - LiveKit client service
- contexts/VoiceTranscriptContext.tsx - Transcript state
- polyfills/livekit-globals.ts - WebRTC polyfills

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-17 17:58:31 -08:00

547 lines
16 KiB
TypeScript

/**
* Debug Screen - Voice Call Testing with Detailed Logs
*
* All-in-one screen for testing Julia AI voice:
* - Start/End call buttons
* - Real-time logs of all LiveKit events
* - Copy logs button
*/
import React, { useState, useEffect, useRef, useCallback } from 'react';
import {
View,
Text,
StyleSheet,
FlatList,
TouchableOpacity,
Platform,
Share,
AppState,
AppStateStatus,
} from 'react-native';
import { SafeAreaView } from 'react-native-safe-area-context';
import { Ionicons } from '@expo/vector-icons';
import * as Clipboard from 'expo-clipboard';
import { activateKeepAwakeAsync, deactivateKeepAwake } from 'expo-keep-awake';
import type { Room as RoomType } from 'livekit-client';
import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme';
import { getToken, VOICE_NAME } from '@/services/livekitService';
type LogEntry = {
id: string;
time: string;
message: string;
type: 'info' | 'success' | 'error' | 'event';
};
type CallState = 'idle' | 'connecting' | 'connected' | 'ending';
export default function DebugScreen() {
const [logs, setLogs] = useState<LogEntry[]>([]);
const [callState, setCallState] = useState<CallState>('idle');
const [callDuration, setCallDuration] = useState(0);
const flatListRef = useRef<FlatList>(null);
const roomRef = useRef<RoomType | null>(null);
const callStartTimeRef = useRef<number | null>(null);
const appStateRef = useRef<AppStateStatus>(AppState.currentState);
// Add log entry
const log = useCallback((message: string, type: LogEntry['type'] = 'info') => {
const time = new Date().toLocaleTimeString('en-US', { hour12: false, hour: '2-digit', minute: '2-digit', second: '2-digit' });
const ms = String(new Date().getMilliseconds()).padStart(3, '0');
setLogs(prev => [...prev, {
id: `${Date.now()}-${Math.random()}`,
time: `${time}.${ms}`,
message,
type,
}]);
}, []);
// Clear logs
const clearLogs = useCallback(() => {
setLogs([]);
}, []);
// Copy logs to clipboard
const copyLogs = useCallback(async () => {
const text = logs.map(l => `[${l.time}] ${l.message}`).join('\n');
await Clipboard.setStringAsync(text);
log('Logs copied to clipboard!', 'success');
}, [logs, log]);
// Share logs
const shareLogs = useCallback(async () => {
const text = logs.map(l => `[${l.time}] ${l.message}`).join('\n');
try {
await Share.share({ message: text, title: 'Voice Debug Logs' });
} catch (e) {
log(`Share failed: ${e}`, 'error');
}
}, [logs, log]);
// Auto-scroll to bottom
useEffect(() => {
if (logs.length > 0) {
setTimeout(() => flatListRef.current?.scrollToEnd({ animated: true }), 100);
}
}, [logs]);
// Call duration timer
useEffect(() => {
if (callState !== 'connected') return;
const interval = setInterval(() => {
if (callStartTimeRef.current) {
setCallDuration(Math.floor((Date.now() - callStartTimeRef.current) / 1000));
}
}, 1000);
return () => clearInterval(interval);
}, [callState]);
// Handle app background/foreground
useEffect(() => {
const subscription = AppState.addEventListener('change', (nextAppState) => {
if (appStateRef.current.match(/inactive|background/) && nextAppState === 'active') {
log('App returned to foreground', 'event');
} else if (appStateRef.current === 'active' && nextAppState.match(/inactive|background/)) {
log('App went to background - call continues', 'event');
}
appStateRef.current = nextAppState;
});
return () => subscription.remove();
}, [log]);
// Start call
const startCall = useCallback(async () => {
if (callState !== 'idle') return;
clearLogs();
setCallState('connecting');
setCallDuration(0);
callStartTimeRef.current = null;
try {
log('=== STARTING VOICE CALL ===', 'info');
// Keep screen awake
await activateKeepAwakeAsync('voiceCall').catch(() => {});
log('Screen keep-awake activated', 'info');
// Step 1: Register WebRTC globals
log('Step 1: Importing @livekit/react-native...', 'info');
const { registerGlobals, AudioSession } = await import('@livekit/react-native');
if (typeof global.RTCPeerConnection === 'undefined') {
log('Registering WebRTC globals...', 'info');
registerGlobals();
log('WebRTC globals registered', 'success');
} else {
log('WebRTC globals already registered', 'info');
}
// Step 2: Import livekit-client
log('Step 2: Importing livekit-client...', 'info');
const { Room, RoomEvent, ConnectionState, Track } = await import('livekit-client');
log('livekit-client imported', 'success');
// Step 3: Start iOS AudioSession
if (Platform.OS === 'ios') {
log('Step 3: Starting iOS AudioSession...', 'info');
await AudioSession.startAudioSession();
log('iOS AudioSession started', 'success');
}
// Step 4: Get token from server
log('Step 4: Requesting token from server...', 'info');
log(`Token server: wellnuo.smartlaunchhub.com/julia/token`, 'info');
const result = await getToken(`user-${Date.now()}`);
if (!result.success || !result.data) {
throw new Error(result.error || 'Failed to get token');
}
const { token, wsUrl, roomName } = result.data;
log(`Token received`, 'success');
log(`Room: ${roomName}`, 'info');
log(`WebSocket URL: ${wsUrl}`, 'info');
// Step 5: Create room and setup listeners
log('Step 5: Creating Room instance...', 'info');
const room = new Room();
roomRef.current = room;
log('Room instance created', 'success');
// Setup ALL event listeners
log('Step 6: Setting up event listeners...', 'info');
room.on(RoomEvent.ConnectionStateChanged, (state: any) => {
log(`EVENT: ConnectionStateChanged → ${state}`, 'event');
if (state === ConnectionState.Connected) {
setCallState('connected');
callStartTimeRef.current = Date.now();
} else if (state === ConnectionState.Disconnected) {
setCallState('idle');
}
});
room.on(RoomEvent.Connected, () => {
log('EVENT: Connected to room', 'success');
});
room.on(RoomEvent.Disconnected, (reason?: any) => {
log(`EVENT: Disconnected. Reason: ${reason || 'unknown'}`, 'event');
});
room.on(RoomEvent.Reconnecting, () => {
log('EVENT: Reconnecting...', 'event');
});
room.on(RoomEvent.Reconnected, () => {
log('EVENT: Reconnected', 'success');
});
room.on(RoomEvent.ParticipantConnected, (participant: any) => {
log(`EVENT: Participant connected: ${participant.identity}`, 'event');
});
room.on(RoomEvent.ParticipantDisconnected, (participant: any) => {
log(`EVENT: Participant disconnected: ${participant.identity}`, 'event');
});
room.on(RoomEvent.TrackSubscribed, (track: any, publication: any, participant: any) => {
log(`EVENT: Track subscribed: ${track.kind} from ${participant.identity}`, 'event');
if (track.kind === Track.Kind.Audio) {
log('Audio track from Julia AI - should hear voice now', 'success');
}
});
room.on(RoomEvent.TrackUnsubscribed, (track: any, publication: any, participant: any) => {
log(`EVENT: Track unsubscribed: ${track.kind} from ${participant.identity}`, 'event');
});
room.on(RoomEvent.TrackMuted, (publication: any, participant: any) => {
log(`EVENT: Track muted by ${participant.identity}`, 'event');
});
room.on(RoomEvent.TrackUnmuted, (publication: any, participant: any) => {
log(`EVENT: Track unmuted by ${participant.identity}`, 'event');
});
room.on(RoomEvent.ActiveSpeakersChanged, (speakers: any[]) => {
if (speakers.length > 0) {
log(`EVENT: Active speakers: ${speakers.map(s => s.identity).join(', ')}`, 'event');
}
});
room.on(RoomEvent.DataReceived, (payload: any, participant: any) => {
try {
const data = JSON.parse(new TextDecoder().decode(payload));
log(`EVENT: Data received: ${JSON.stringify(data).substring(0, 100)}`, 'event');
} catch (e) {
log(`EVENT: Data received (binary)`, 'event');
}
});
room.on(RoomEvent.AudioPlaybackStatusChanged, () => {
log(`EVENT: AudioPlaybackStatusChanged - canPlay: ${room.canPlaybackAudio}`, 'event');
});
room.on(RoomEvent.MediaDevicesError, (error: any) => {
log(`EVENT: MediaDevicesError: ${error?.message || error}`, 'error');
});
room.on(RoomEvent.RoomMetadataChanged, (metadata: string) => {
log(`EVENT: RoomMetadataChanged: ${metadata}`, 'event');
});
log('Event listeners set up', 'success');
// Step 7: Connect to room
log('Step 7: Connecting to LiveKit room...', 'info');
await room.connect(wsUrl, token, { autoSubscribe: true });
log('Connected to room', 'success');
// Step 8: Enable microphone
log('Step 8: Enabling microphone...', 'info');
await room.localParticipant.setMicrophoneEnabled(true);
log('Microphone enabled', 'success');
log(`Local participant: ${room.localParticipant.identity}`, 'info');
log('=== CALL ACTIVE ===', 'success');
} catch (err: any) {
log(`ERROR: ${err?.message || err}`, 'error');
log(`Stack: ${err?.stack?.substring(0, 200) || 'no stack'}`, 'error');
setCallState('idle');
deactivateKeepAwake('voiceCall');
}
}, [callState, log, clearLogs]);
// End call
const endCall = useCallback(async () => {
if (callState === 'idle') return;
log('=== ENDING CALL ===', 'info');
setCallState('ending');
try {
if (roomRef.current) {
log('Disconnecting from room...', 'info');
await roomRef.current.disconnect();
roomRef.current = null;
log('Disconnected from room', 'success');
}
if (Platform.OS === 'ios') {
log('Stopping iOS AudioSession...', 'info');
const { AudioSession } = await import('@livekit/react-native');
await AudioSession.stopAudioSession();
log('iOS AudioSession stopped', 'success');
}
deactivateKeepAwake('voiceCall');
log('Screen keep-awake deactivated', 'info');
} catch (err: any) {
log(`Error during cleanup: ${err?.message || err}`, 'error');
}
setCallState('idle');
log('=== CALL ENDED ===', 'info');
}, [callState, log]);
// Format duration
const formatDuration = (seconds: number): string => {
const mins = Math.floor(seconds / 60);
const secs = seconds % 60;
return `${mins}:${secs.toString().padStart(2, '0')}`;
};
// Get log color
const getLogColor = (type: LogEntry['type']): string => {
switch (type) {
case 'success': return '#4ade80';
case 'error': return '#f87171';
case 'event': return '#60a5fa';
default: return '#e5e5e5';
}
};
return (
<SafeAreaView style={styles.container} edges={['top']}>
{/* Header */}
<View style={styles.header}>
<Text style={styles.title}>Voice Debug</Text>
<Text style={styles.subtitle}>{VOICE_NAME}</Text>
</View>
{/* Call Status */}
<View style={styles.statusBar}>
<View style={styles.statusLeft}>
<View style={[
styles.statusDot,
{ backgroundColor: callState === 'connected' ? '#4ade80' : callState === 'connecting' ? '#fbbf24' : '#6b7280' }
]} />
<Text style={styles.statusText}>
{callState === 'idle' && 'Ready'}
{callState === 'connecting' && 'Connecting...'}
{callState === 'connected' && `Connected ${formatDuration(callDuration)}`}
{callState === 'ending' && 'Ending...'}
</Text>
</View>
<Text style={styles.logCount}>{logs.length} logs</Text>
</View>
{/* Control Buttons */}
<View style={styles.controls}>
{callState === 'idle' ? (
<TouchableOpacity style={styles.startButton} onPress={startCall}>
<Ionicons name="call" size={24} color="#fff" />
<Text style={styles.buttonText}>Start Call</Text>
</TouchableOpacity>
) : (
<TouchableOpacity
style={styles.endButton}
onPress={endCall}
disabled={callState === 'ending'}
>
<Ionicons name="call" size={24} color="#fff" style={{ transform: [{ rotate: '135deg' }] }} />
<Text style={styles.buttonText}>End Call</Text>
</TouchableOpacity>
)}
<TouchableOpacity style={styles.copyButton} onPress={copyLogs}>
<Ionicons name="copy" size={20} color="#fff" />
<Text style={styles.smallButtonText}>Copy</Text>
</TouchableOpacity>
<TouchableOpacity style={styles.shareButton} onPress={shareLogs}>
<Ionicons name="share" size={20} color="#fff" />
<Text style={styles.smallButtonText}>Share</Text>
</TouchableOpacity>
<TouchableOpacity style={styles.clearButton} onPress={clearLogs}>
<Ionicons name="trash" size={20} color="#fff" />
<Text style={styles.smallButtonText}>Clear</Text>
</TouchableOpacity>
</View>
{/* Logs */}
<FlatList
ref={flatListRef}
data={logs}
keyExtractor={(item) => item.id}
style={styles.logsList}
contentContainerStyle={styles.logsContent}
renderItem={({ item }) => (
<Text style={[styles.logEntry, { color: getLogColor(item.type) }]}>
<Text style={styles.logTime}>[{item.time}]</Text> {item.message}
</Text>
)}
ListEmptyComponent={
<View style={styles.emptyContainer}>
<Ionicons name="terminal" size={48} color="#6b7280" />
<Text style={styles.emptyText}>Press "Start Call" to begin</Text>
</View>
}
/>
</SafeAreaView>
);
}
const styles = StyleSheet.create({
container: {
flex: 1,
backgroundColor: '#0f0f0f',
},
header: {
padding: Spacing.md,
borderBottomWidth: 1,
borderBottomColor: '#333',
},
title: {
fontSize: 24,
fontWeight: '700',
color: '#fff',
},
subtitle: {
fontSize: 14,
color: '#888',
marginTop: 2,
},
statusBar: {
flexDirection: 'row',
justifyContent: 'space-between',
alignItems: 'center',
paddingHorizontal: Spacing.md,
paddingVertical: Spacing.sm,
backgroundColor: '#1a1a1a',
},
statusLeft: {
flexDirection: 'row',
alignItems: 'center',
},
statusDot: {
width: 10,
height: 10,
borderRadius: 5,
marginRight: 8,
},
statusText: {
color: '#fff',
fontSize: 14,
fontWeight: '500',
},
logCount: {
color: '#888',
fontSize: 12,
},
controls: {
flexDirection: 'row',
padding: Spacing.md,
gap: 10,
borderBottomWidth: 1,
borderBottomColor: '#333',
},
startButton: {
flex: 1,
flexDirection: 'row',
alignItems: 'center',
justifyContent: 'center',
backgroundColor: '#22c55e',
paddingVertical: 14,
borderRadius: 12,
gap: 8,
},
endButton: {
flex: 1,
flexDirection: 'row',
alignItems: 'center',
justifyContent: 'center',
backgroundColor: '#ef4444',
paddingVertical: 14,
borderRadius: 12,
gap: 8,
},
buttonText: {
color: '#fff',
fontSize: 16,
fontWeight: '600',
},
copyButton: {
alignItems: 'center',
justifyContent: 'center',
backgroundColor: '#3b82f6',
paddingVertical: 10,
paddingHorizontal: 12,
borderRadius: 10,
},
shareButton: {
alignItems: 'center',
justifyContent: 'center',
backgroundColor: '#8b5cf6',
paddingVertical: 10,
paddingHorizontal: 12,
borderRadius: 10,
},
clearButton: {
alignItems: 'center',
justifyContent: 'center',
backgroundColor: '#6b7280',
paddingVertical: 10,
paddingHorizontal: 12,
borderRadius: 10,
},
smallButtonText: {
color: '#fff',
fontSize: 10,
fontWeight: '500',
marginTop: 2,
},
logsList: {
flex: 1,
},
logsContent: {
padding: Spacing.sm,
paddingBottom: 100,
},
logEntry: {
fontSize: 12,
fontFamily: Platform.OS === 'ios' ? 'Menlo' : 'monospace',
lineHeight: 18,
marginBottom: 2,
},
logTime: {
color: '#888',
},
emptyContainer: {
alignItems: 'center',
justifyContent: 'center',
paddingTop: 100,
},
emptyText: {
color: '#6b7280',
fontSize: 16,
marginTop: 12,
},
});