diff --git a/app/(tabs)/debug.tsx b/app/(tabs)/debug.tsx index 6ab5395..5c6e1d2 100644 --- a/app/(tabs)/debug.tsx +++ b/app/(tabs)/debug.tsx @@ -20,6 +20,8 @@ import { Share, AppState, AppStateStatus, + TextInput, + KeyboardAvoidingView, } from 'react-native'; import { SafeAreaView } from 'react-native-safe-area-context'; import { Ionicons } from '@expo/vector-icons'; @@ -28,10 +30,11 @@ import { activateKeepAwakeAsync, deactivateKeepAwake } from 'expo-keep-awake'; import type { Room as RoomType } from 'livekit-client'; import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme'; import { getToken, VOICE_NAME } from '@/services/livekitService'; +import { api } from '@/services/api'; +import { useBeneficiary } from '@/contexts/BeneficiaryContext'; import { configureAudioForVoiceCall, stopAudioSession, - setAudioOutput, } from '@/utils/audioSession'; import { startVoiceCallService, @@ -56,12 +59,64 @@ export default function DebugScreen() { const [logs, setLogs] = useState([]); const [callState, setCallState] = useState('idle'); const [callDuration, setCallDuration] = useState(0); - const [isSpeakerOn, setIsSpeakerOn] = useState(true); // Default to speaker + const [agentState, setAgentState] = useState('—'); // listening/thinking/speaking + const [lastUserText, setLastUserText] = useState(''); // Последний распознанный текст пользователя + const [lastAgentText, setLastAgentText] = useState(''); // Последний ответ агента + const [micLevel, setMicLevel] = useState(0); // Уровень микрофона 0-100 + const [deploymentId, setDeploymentIdState] = useState(''); // Custom deployment ID + const [loadingBeneficiary, setLoadingBeneficiary] = useState(true); + const [accumulateResponses, setAccumulateResponses] = useState(true); // Накапливать chunks до полного ответа const flatListRef = useRef(null); + + // Refs для накопления chunks + const accumulatedUserTextRef = useRef(''); + const accumulatedAgentTextRef = useRef(''); + const lastUserSegmentIdRef = useRef(null); + const lastAgentSegmentIdRef = useRef(null); const roomRef = useRef(null); const callStartTimeRef = useRef(null); const appStateRef = useRef(AppState.currentState); + const { currentBeneficiary, setDebugDeploymentId } = useBeneficiary(); + + // Sync deploymentId with context for voice-call.tsx to use + const setDeploymentId = useCallback((id: string) => { + setDeploymentIdState(id); + // Update context so voice-call.tsx can access it + setDebugDeploymentId(id.trim() || null); + }, [setDebugDeploymentId]); + + // Load default deployment ID from first beneficiary + useEffect(() => { + const loadDefaultDeploymentId = async () => { + try { + // First check if currentBeneficiary is available + if (currentBeneficiary?.id) { + const id = currentBeneficiary.id.toString(); + setDeploymentIdState(id); + setDebugDeploymentId(id); // Also set in context + setLoadingBeneficiary(false); + return; + } + + // Otherwise load from API + const response = await api.getAllBeneficiaries(); + if (response.ok && response.data && response.data.length > 0) { + const firstBeneficiary = response.data[0]; + const id = firstBeneficiary.id.toString(); + setDeploymentIdState(id); + setDebugDeploymentId(id); // Also set in context + } + } catch (error) { + console.error('[Debug] Failed to load beneficiary:', error); + } finally { + setLoadingBeneficiary(false); + } + }; + + loadDefaultDeploymentId(); + }, [currentBeneficiary, setDebugDeploymentId]); + // Add log entry const log = useCallback((message: string, type: LogEntry['type'] = 'info') => { const time = new Date().toLocaleTimeString('en-US', { hour12: false, hour: '2-digit', minute: '2-digit', second: '2-digit' }); @@ -127,20 +182,6 @@ export default function DebugScreen() { return () => subscription.remove(); }, [log]); - // Toggle speaker - const toggleSpeaker = useCallback(async () => { - const newState = !isSpeakerOn; - log(`=== TOGGLING SPEAKER: ${isSpeakerOn ? 'ON' : 'OFF'} → ${newState ? 'ON' : 'OFF'} ===`, 'info'); - - try { - await setAudioOutput(newState); - setIsSpeakerOn(newState); - log(`Speaker toggled to ${newState ? 'ON (loud speaker)' : 'OFF (earpiece)'}`, 'success'); - } catch (err: any) { - log(`Speaker toggle error: ${err?.message || err}`, 'error'); - } - }, [isSpeakerOn, log]); - // Start call const startCall = useCallback(async () => { if (callState !== 'idle') return; @@ -148,7 +189,6 @@ export default function DebugScreen() { clearLogs(); setCallState('connecting'); setCallDuration(0); - setIsSpeakerOn(true); // Reset speaker state callStartTimeRef.current = null; try { @@ -205,7 +245,20 @@ export default function DebugScreen() { // Step 4: Get token from server log('Step 4: Requesting token from server...', 'info'); log(`Token server: wellnuo.smartlaunchhub.com/julia/token`, 'info'); - const result = await getToken(`user-${Date.now()}`); + + // Передаём deployment ID если указан + const beneficiaryData = deploymentId.trim() ? { + deploymentId: deploymentId.trim(), + beneficiaryNamesDict: {}, + } : undefined; + + if (beneficiaryData) { + log(`📋 Using custom Deployment ID: ${deploymentId}`, 'success'); + } else { + log(`📋 No Deployment ID specified (default mode)`, 'info'); + } + + const result = await getToken(`user-${Date.now()}`, beneficiaryData); if (!result.success || !result.data) { throw new Error(result.error || 'Failed to get token'); @@ -252,11 +305,54 @@ export default function DebugScreen() { }); newRoom.on(RoomEvent.ParticipantConnected, (participant: any) => { - log(`EVENT: Participant connected: ${participant.identity}`, 'event'); + log(`👋 PARTICIPANT CONNECTED: ${participant.identity}`, 'success'); + + // Подписаться на события этого участника (для агента Julia) + participant.on('isSpeakingChanged', (speaking: boolean) => { + if (speaking) { + log(`🔊 ${participant.identity} STARTED SPEAKING`, 'success'); + setAgentState('speaking'); + } else { + log(`🔇 ${participant.identity} stopped speaking`, 'info'); + } + }); + + participant.on('trackMuted', (pub: any) => { + log(`🔇 ${participant.identity} muted ${pub.kind}`, 'event'); + }); + + participant.on('trackUnmuted', (pub: any) => { + log(`🔊 ${participant.identity} unmuted ${pub.kind}`, 'event'); + }); + + participant.on('attributesChanged', (attrs: any) => { + log(`📋 ${participant.identity} ATTRIBUTES:`, 'event'); + Object.entries(attrs || {}).forEach(([k, v]) => { + log(` ${k}: ${v}`, 'info'); + if (k === 'lk.agent.state') { + setAgentState(String(v)); + } + }); + }); + + participant.on('transcriptionReceived', (segments: any[]) => { + log(`🤖 ${participant.identity} TRANSCRIPTION:`, 'success'); + segments.forEach((seg: any, i: number) => { + const text = seg.text || seg.final || ''; + log(` [${i}] "${text}"`, 'info'); + if (text) setLastAgentText(text); + }); + }); + + // Показать текущие атрибуты участника + const attrs = participant.attributes || {}; + if (Object.keys(attrs).length > 0) { + log(` Initial attributes: ${JSON.stringify(attrs)}`, 'info'); + } }); newRoom.on(RoomEvent.ParticipantDisconnected, (participant: any) => { - log(`EVENT: Participant disconnected: ${participant.identity}`, 'event'); + log(`👋 PARTICIPANT DISCONNECTED: ${participant.identity}`, 'event'); }); newRoom.on(RoomEvent.TrackSubscribed, (track: any, publication: any, participant: any) => { @@ -284,12 +380,45 @@ export default function DebugScreen() { } }); - newRoom.on(RoomEvent.DataReceived, (payload: any, participant: any) => { + newRoom.on(RoomEvent.DataReceived, (payload: any, participant: any, kind: any, topic: any) => { + log(`📩 DATA RECEIVED from ${participant?.identity || 'unknown'}`, 'event'); + log(` kind: ${kind}, topic: ${topic || 'none'}`, 'info'); try { - const data = JSON.parse(new TextDecoder().decode(payload)); - log(`EVENT: Data received: ${JSON.stringify(data).substring(0, 100)}`, 'event'); + const text = new TextDecoder().decode(payload); + const data = JSON.parse(text); + log(` type: ${data.type || 'unknown'}`, 'info'); + + // Подробное логирование разных типов сообщений + if (data.type === 'transcript' || data.type === 'transcription') { + log(` 🗣️ TRANSCRIPT: role=${data.role}`, 'success'); + const text = data.text || data.content || ''; + log(` 📝 TEXT: "${text}"`, 'success'); + // Обновить UI + if (data.role === 'user') { + setLastUserText(text); + } else if (data.role === 'assistant' || data.role === 'agent') { + setLastAgentText(text); + } + } else if (data.type === 'state' || data.type === 'agent_state') { + const stateValue = data.state || JSON.stringify(data); + log(` 🤖 AGENT STATE: ${stateValue}`, 'success'); + setAgentState(stateValue); + } else if (data.type === 'function_call' || data.type === 'tool_call') { + log(` 🔧 FUNCTION CALL: ${data.name || data.function || JSON.stringify(data)}`, 'event'); + } else if (data.type === 'function_result' || data.type === 'tool_result') { + log(` ✅ FUNCTION RESULT: ${JSON.stringify(data.result || data).substring(0, 200)}`, 'event'); + } else { + // Показать полный JSON для неизвестных типов + log(` 📦 FULL DATA: ${JSON.stringify(data)}`, 'info'); + } } catch (e) { - log(`EVENT: Data received (binary)`, 'event'); + // Попробовать показать как текст + try { + const text = new TextDecoder().decode(payload); + log(` 📄 RAW TEXT: "${text.substring(0, 300)}"`, 'info'); + } catch { + log(` 📎 BINARY DATA: ${payload.byteLength} bytes`, 'info'); + } } }); @@ -305,7 +434,267 @@ export default function DebugScreen() { log(`EVENT: RoomMetadataChanged: ${metadata}`, 'event'); }); - log('Event listeners set up', 'success'); + // =========================================== + // TRANSCRIPTION - распознанный текст (STT) + // =========================================== + newRoom.on(RoomEvent.TranscriptionReceived, (segments: any[], participant: any) => { + const isUser = participant?.identity === newRoom.localParticipant.identity; + const who = isUser ? '👤 USER' : '🤖 AGENT'; + + segments.forEach((segment: any, idx: number) => { + const text = segment.text || segment.final || ''; + const segmentId = segment.id || `seg-${Date.now()}`; + const isFinalFlag = segment.final !== undefined; + + if (accumulateResponses) { + // === РЕЖИМ НАКОПЛЕНИЯ: Показываем только финальные полные ответы === + if (isUser) { + // Новый сегмент или продолжение текущего + if (lastUserSegmentIdRef.current !== segmentId) { + // Если был предыдущий финальный - логируем его + if (accumulatedUserTextRef.current && lastUserSegmentIdRef.current) { + log(`👤 USER FINAL: "${accumulatedUserTextRef.current}"`, 'success'); + } + accumulatedUserTextRef.current = text; + lastUserSegmentIdRef.current = segmentId; + } else { + // Обновляем текущий сегмент + accumulatedUserTextRef.current = text; + } + + // Если финальный - логируем сразу + if (isFinalFlag && text) { + log(`👤 USER: "${text}"`, 'success'); + setLastUserText(text); + accumulatedUserTextRef.current = ''; + lastUserSegmentIdRef.current = null; + } + } else { + // AGENT + if (lastAgentSegmentIdRef.current !== segmentId) { + if (accumulatedAgentTextRef.current && lastAgentSegmentIdRef.current) { + log(`🤖 AGENT FINAL: "${accumulatedAgentTextRef.current}"`, 'success'); + } + accumulatedAgentTextRef.current = text; + lastAgentSegmentIdRef.current = segmentId; + } else { + accumulatedAgentTextRef.current = text; + } + + if (isFinalFlag && text) { + log(`🤖 JULIA: "${text}"`, 'success'); + setLastAgentText(text); + accumulatedAgentTextRef.current = ''; + lastAgentSegmentIdRef.current = null; + } + } + } else { + // === РЕЖИМ ПОЛНОГО ЛОГИРОВАНИЯ: Показываем каждый chunk === + const finalLabel = isFinalFlag ? '(FINAL)' : '(interim)'; + log(`🎤 TRANSCRIPTION from ${who} (${participant?.identity || 'unknown'})`, 'success'); + log(` [${idx}] ${finalLabel}: "${text}"`, 'event'); + if (segment.id) log(` segment.id: ${segment.id}`, 'info'); + if (segment.firstReceivedTime) log(` firstReceivedTime: ${segment.firstReceivedTime}`, 'info'); + if (segment.lastReceivedTime) log(` lastReceivedTime: ${segment.lastReceivedTime}`, 'info'); + + // Обновить UI с последним текстом + if (text && (isFinalFlag || !segment.final)) { + if (isUser) { + setLastUserText(text); + } else { + setLastAgentText(text); + } + } + } + }); + }); + + // =========================================== + // PARTICIPANT ATTRIBUTES - состояние агента + // =========================================== + newRoom.on(RoomEvent.ParticipantAttributesChanged, (changedAttributes: any, participant: any) => { + log(`👤 ATTRIBUTES CHANGED for ${participant?.identity || 'unknown'}`, 'event'); + Object.entries(changedAttributes || {}).forEach(([key, value]) => { + log(` ${key}: ${value}`, 'info'); + // Особенно важно: lk.agent.state показывает listening/thinking/speaking + if (key === 'lk.agent.state') { + log(` 🤖 AGENT STATE: ${value}`, 'success'); + // Обновить UI + setAgentState(String(value)); + } + }); + // Показать все текущие атрибуты + const attrs = participant?.attributes || {}; + if (Object.keys(attrs).length > 0) { + log(` All attributes: ${JSON.stringify(attrs)}`, 'info'); + } + }); + + // =========================================== + // SIGNAL CONNECTED/RECONNECTING + // =========================================== + newRoom.on(RoomEvent.SignalConnected, () => { + log('EVENT: SignalConnected - WebSocket подключен', 'success'); + }); + + newRoom.on(RoomEvent.SignalReconnecting, () => { + log('EVENT: SignalReconnecting - переподключение сигнала...', 'event'); + }); + + // =========================================== + // LOCAL TRACK UNPUBLISHED + // =========================================== + newRoom.on(RoomEvent.LocalTrackUnpublished, (publication: any, participant: any) => { + log(`EVENT: LocalTrackUnpublished - ${publication.trackSid}`, 'event'); + }); + + // =========================================== + // ДОПОЛНИТЕЛЬНЫЕ СОБЫТИЯ ДЛЯ ПОЛНОГО ДЕБАГА + // =========================================== + + // Качество соединения + newRoom.on(RoomEvent.ConnectionQualityChanged, (quality: any, participant: any) => { + const qualityEmoji = quality === 'excellent' ? '🟢' : quality === 'good' ? '🟡' : '🔴'; + log(`${qualityEmoji} CONNECTION QUALITY: ${participant?.identity || 'local'} → ${quality}`, 'event'); + }); + + // Изменение устройств (микрофон/камера подключены/отключены) + newRoom.on(RoomEvent.MediaDevicesChanged, () => { + log(`🔌 MEDIA DEVICES CHANGED - устройства обновились`, 'event'); + }); + + // Изменение активного устройства + newRoom.on(RoomEvent.ActiveDeviceChanged, (kind: any, deviceId: any) => { + log(`🎛️ ACTIVE DEVICE CHANGED: ${kind} → ${deviceId}`, 'event'); + }); + + // Ошибка подписки на трек + newRoom.on(RoomEvent.TrackSubscriptionFailed, (trackSid: any, participant: any, reason: any) => { + log(`❌ TRACK SUBSCRIPTION FAILED: ${trackSid} from ${participant?.identity}`, 'error'); + log(` Reason: ${reason}`, 'error'); + }); + + // Публикация трека (когда агент начинает говорить) + newRoom.on(RoomEvent.TrackPublished, (publication: any, participant: any) => { + log(`📢 TRACK PUBLISHED by ${participant?.identity}: ${publication.kind} (${publication.source})`, 'event'); + }); + + // Отмена публикации трека + newRoom.on(RoomEvent.TrackUnpublished, (publication: any, participant: any) => { + log(`📤 TRACK UNPUBLISHED by ${participant?.identity}: ${publication.kind}`, 'event'); + }); + + // Изменение метаданных участника + newRoom.on(RoomEvent.ParticipantMetadataChanged, (metadata: any, participant: any) => { + log(`📋 PARTICIPANT METADATA: ${participant?.identity}`, 'event'); + try { + const parsed = JSON.parse(metadata || '{}'); + log(` ${JSON.stringify(parsed)}`, 'info'); + } catch { + log(` ${metadata}`, 'info'); + } + }); + + // Изменение имени участника + newRoom.on(RoomEvent.ParticipantNameChanged, (name: any, participant: any) => { + log(`👤 PARTICIPANT NAME: ${participant?.identity} → ${name}`, 'event'); + }); + + // Статус записи (если комната записывается) + newRoom.on(RoomEvent.RecordingStatusChanged, (recording: any) => { + log(`⏺️ RECORDING STATUS: ${recording ? 'RECORDING' : 'NOT RECORDING'}`, recording ? 'success' : 'info'); + }); + + // Изменение статуса потока трека + newRoom.on(RoomEvent.TrackStreamStateChanged, (publication: any, streamState: any, participant: any) => { + log(`📊 TRACK STREAM STATE: ${participant?.identity}/${publication.trackSid} → ${streamState}`, 'event'); + }); + + // Разрешения на подписку трека + newRoom.on(RoomEvent.TrackSubscriptionPermissionChanged, (publication: any, status: any, participant: any) => { + log(`🔐 TRACK PERMISSION: ${participant?.identity}/${publication.trackSid} → ${status}`, 'event'); + }); + + // Статус подписки на трек + newRoom.on(RoomEvent.TrackSubscriptionStatusChanged, (publication: any, status: any, participant: any) => { + log(`📶 TRACK SUBSCRIPTION: ${participant?.identity}/${publication.trackSid} → ${status}`, 'event'); + }); + + // Разрешения участника изменились + newRoom.on(RoomEvent.ParticipantPermissionsChanged, (prevPermissions: any, participant: any) => { + log(`🔑 PARTICIPANT PERMISSIONS CHANGED: ${participant?.identity}`, 'event'); + log(` New permissions: ${JSON.stringify(participant?.permissions || {})}`, 'info'); + }); + + // ChatMessage - сообщения в чате комнаты + newRoom.on(RoomEvent.ChatMessage, (message: any, participant: any) => { + log(`💬 CHAT MESSAGE from ${participant?.identity || 'system'}:`, 'success'); + log(` ${message.message || JSON.stringify(message)}`, 'info'); + }); + + // SIP DTMF - телефонные сигналы + newRoom.on(RoomEvent.SipDTMFReceived, (dtmf: any, participant: any) => { + log(`📞 SIP DTMF: ${dtmf.code} from ${participant?.identity}`, 'event'); + }); + + // Детекция тишины микрофона + newRoom.on(RoomEvent.LocalAudioSilenceDetected, (publication: any) => { + log(`🔇 LOCAL AUDIO SILENCE DETECTED - микрофон молчит`, 'event'); + }); + + // Изменения буфера DataChannel + newRoom.on(RoomEvent.DCBufferStatusChanged, (isLow: any, kind: any) => { + log(`📦 DC BUFFER: ${kind} buffer is ${isLow ? 'LOW' : 'OK'}`, isLow ? 'event' : 'info'); + }); + + // Метрики производительности + newRoom.on(RoomEvent.MetricsReceived, (metrics: any) => { + log(`📈 METRICS RECEIVED:`, 'info'); + if (metrics.audioStats) { + log(` Audio: bitrate=${metrics.audioStats.bitrate}, packetsLost=${metrics.audioStats.packetsLost}`, 'info'); + } + if (metrics.videoStats) { + log(` Video: bitrate=${metrics.videoStats.bitrate}, fps=${metrics.videoStats.fps}`, 'info'); + } + }); + + // Статус воспроизведения видео (если есть) + newRoom.on(RoomEvent.VideoPlaybackStatusChanged, () => { + log(`🎬 VIDEO PLAYBACK STATUS CHANGED`, 'event'); + }); + + // Ошибка шифрования + newRoom.on(RoomEvent.EncryptionError, (error: any) => { + log(`🔒 ENCRYPTION ERROR: ${error?.message || error}`, 'error'); + }); + + // Статус шифрования участника + newRoom.on(RoomEvent.ParticipantEncryptionStatusChanged, (encrypted: any, participant: any) => { + log(`🔐 ENCRYPTION STATUS: ${participant?.identity} → ${encrypted ? 'encrypted' : 'not encrypted'}`, 'event'); + }); + + // Комната перемещена (редко) + newRoom.on(RoomEvent.Moved, (room: any) => { + log(`🚀 ROOM MOVED to new server`, 'event'); + }); + + // Участник стал активным + newRoom.on(RoomEvent.ParticipantActive, (participant: any) => { + log(`✅ PARTICIPANT ACTIVE: ${participant?.identity}`, 'success'); + + // Проверяем, что это агент Julia (не локальный участник) + const isAgent = participant?.identity?.startsWith('agent-') || + (participant?.attributes?.['lk.agent_name'] === 'julia-ai'); + + if (isAgent) { + log(``, 'success'); + log(`🟢🟢🟢 AGENT READY 🟢🟢🟢`, 'success'); + log(`🔊 Julia will now speak greeting...`, 'success'); + log(``, 'success'); + } + }); + + log('Event listeners set up (FULL DEBUG MODE)', 'success'); // Step 7: Connect to room log('Step 7: Connecting to LiveKit room...', 'info'); @@ -335,21 +724,117 @@ export default function DebugScreen() { } }); - // Listen for local track published + // =========================================== + // LOCAL PARTICIPANT EVENTS - события моего микрофона + // =========================================== newRoom.localParticipant.on('localTrackPublished', (pub: any) => { - log(`MY TRACK PUBLISHED: ${pub.kind} sid=${pub.trackSid}`, 'success'); + log(`🎤 MY TRACK PUBLISHED: ${pub.kind} sid=${pub.trackSid}`, 'success'); + }); + + newRoom.localParticipant.on('localTrackUnpublished', (pub: any) => { + log(`🎤 MY TRACK UNPUBLISHED: ${pub.kind} sid=${pub.trackSid}`, 'event'); + }); + + // IsSpeakingChanged - когда я начинаю/перестаю говорить + newRoom.localParticipant.on('isSpeakingChanged', (speaking: boolean) => { + if (speaking) { + log(`🗣️ >>> I STARTED SPEAKING <<<`, 'success'); + } else { + log(`🤐 I stopped speaking`, 'info'); + } + }); + + // Мой трек замьютился/размьютился + newRoom.localParticipant.on('trackMuted', (pub: any) => { + log(`🔇 MY TRACK MUTED: ${pub.kind}`, 'event'); + }); + + newRoom.localParticipant.on('trackUnmuted', (pub: any) => { + log(`🔊 MY TRACK UNMUTED: ${pub.kind}`, 'success'); + }); + + // Ошибка медиа устройства на моём участнике + newRoom.localParticipant.on('mediaDevicesError', (error: any) => { + log(`❌ MY MEDIA DEVICE ERROR: ${error?.message || error}`, 'error'); + }); + + // Аудио поток захвачен + newRoom.localParticipant.on('audioStreamAcquired', () => { + log(`🎙️ AUDIO STREAM ACQUIRED - микрофон захвачен!`, 'success'); + }); + + // Транскрипция на моём треке + newRoom.localParticipant.on('transcriptionReceived', (segments: any[]) => { + log(`🎤 MY TRANSCRIPTION (${segments.length} segments):`, 'success'); + segments.forEach((seg: any, i: number) => { + log(` [${i}] "${seg.text || seg.final}"`, 'info'); + }); }); // Listen when I become an active speaker (means mic is working) newRoom.on(RoomEvent.ActiveSpeakersChanged, (speakers: any[]) => { const iAmSpeaking = speakers.some(s => s.identity === newRoom.localParticipant.identity); if (iAmSpeaking) { - log(`*** I AM SPEAKING - MIC WORKS ***`, 'success'); + log(`🎙️ *** I AM SPEAKING - MIC WORKS! ***`, 'success'); } }); log(`Local participant: ${newRoom.localParticipant.identity}`, 'info'); + // =========================================== + // AUDIO LEVEL MONITORING - периодическая проверка уровня микрофона + // =========================================== + let audioLevelInterval: ReturnType | null = null; + let lastLoggedLevel = -1; + + const startAudioLevelMonitoring = () => { + if (audioLevelInterval) return; + + audioLevelInterval = setInterval(() => { + try { + // Найти microphone track среди всех публикаций + const audioTracks = newRoom.localParticipant.audioTrackPublications; + let localAudioTrack: any = null; + audioTracks.forEach((pub: any) => { + if (pub.source === 'microphone' || pub.kind === 'audio') { + localAudioTrack = pub; + } + }); + if (localAudioTrack?.track) { + // Получаем audio level через LiveKit API + const audioLevel = (localAudioTrack.track as any).audioLevel; + if (audioLevel !== undefined) { + const roundedLevel = Math.round(audioLevel * 100); + // Обновить UI + setMicLevel(roundedLevel); + // Логируем только когда уровень существенно изменился + if (Math.abs(roundedLevel - lastLoggedLevel) > 5) { + lastLoggedLevel = roundedLevel; + const bars = '▓'.repeat(Math.min(20, Math.round(audioLevel * 20))) + '░'.repeat(Math.max(0, 20 - Math.round(audioLevel * 20))); + log(`🎚️ MIC LEVEL: [${bars}] ${roundedLevel}%`, audioLevel > 0.1 ? 'success' : 'info'); + } + } + } + } catch (e) { + // Ignore errors + } + }, 200); // Проверять каждые 200мс для плавного UI + }; + + // Запустить мониторинг audio level после подключения + newRoom.on(RoomEvent.Connected, () => { + log('Starting audio level monitoring...', 'info'); + setTimeout(startAudioLevelMonitoring, 1000); + }); + + // Остановить при отключении + newRoom.on(RoomEvent.Disconnected, () => { + if (audioLevelInterval) { + clearInterval(audioLevelInterval); + audioLevelInterval = null; + } + }); + // Android: Start foreground service to keep call alive in background if (Platform.OS === 'android') { log('Android: Starting foreground service...', 'info'); @@ -463,6 +948,52 @@ export default function DebugScreen() { {logs.length} logs + {/* Deployment ID Input */} + {callState === 'idle' && ( + + Deployment ID (optional): + + {deploymentId.trim() && ( + setDeploymentId('')} + > + + + )} + + )} + + {/* Log Mode Toggle */} + + Log mode: + setAccumulateResponses(true)} + > + + Clean (final only) + + + setAccumulateResponses(false)} + > + + Verbose (all chunks) + + + + {/* Control Buttons - Row 1: Call controls */} {callState === 'idle' ? ( @@ -481,19 +1012,6 @@ export default function DebugScreen() { )} - {/* Speaker Toggle Button */} - - - {isSpeakerOn ? 'Speaker' : 'Ear'} - {/* Control Buttons - Row 2: Log controls */} @@ -518,6 +1036,54 @@ export default function DebugScreen() { + {/* ========== LIVE STATUS PANEL ========== */} + {callState === 'connected' && ( + + {/* Agent State */} + + 🤖 Agent: + + + {agentState === 'speaking' ? '🔊 SPEAKING' : + agentState === 'thinking' ? '🧠 THINKING' : + agentState === 'listening' ? '👂 LISTENING' : + agentState} + + + + + {/* Mic Level */} + + 🎙️ Mic: + + + + {micLevel}% + + + {/* Last User Text */} + {lastUserText ? ( + + 👤 You: + {lastUserText} + + ) : null} + + {/* Last Agent Text */} + {lastAgentText ? ( + + 🤖 Julia: + {lastAgentText} + + ) : null} + + )} + {/* Logs */} ([]); @@ -57,6 +57,16 @@ export default function VoiceCallScreen() { // Build beneficiaryData for voice agent const beneficiaryData = useMemo((): BeneficiaryData | undefined => { + // PRIORITY 1: If debugDeploymentId is set (from Debug screen), use it + if (debugDeploymentId) { + console.log('[VoiceCall] Using DEBUG deployment ID:', debugDeploymentId); + return { + deploymentId: debugDeploymentId, + beneficiaryNamesDict: {}, + }; + } + + // PRIORITY 2: Use beneficiaries from API // Safety check - ensure beneficiaries is an array if (!Array.isArray(beneficiaries) || beneficiaries.length === 0) { console.log('[VoiceCall] No beneficiaries yet, skipping beneficiaryData'); @@ -91,7 +101,7 @@ export default function VoiceCallScreen() { console.error('[VoiceCall] Error building beneficiaryData:', error); return undefined; } - }, [beneficiaries, currentBeneficiary]); + }, [beneficiaries, currentBeneficiary, debugDeploymentId]); // LiveKit hook - ALL logic is here const { @@ -126,14 +136,22 @@ export default function VoiceCallScreen() { // Track if connect has been called to prevent duplicate calls const connectCalledRef = useRef(false); - // Start call ONLY after beneficiaries are loaded AND beneficiaryData is ready + // Start call ONLY after beneficiaryData is ready // IMPORTANT: We must wait for beneficiaryData to be populated! // Without deploymentId, Julia AI agent won't know which beneficiary to talk about. useEffect(() => { // Prevent duplicate connect calls if (connectCalledRef.current) return; - // Only connect when beneficiaryData has a valid deploymentId + // If debugDeploymentId is set, connect immediately (don't wait for beneficiaries) + if (debugDeploymentId && beneficiaryData?.deploymentId) { + console.log('[VoiceCall] Starting call with DEBUG deploymentId:', debugDeploymentId); + connectCalledRef.current = true; + connect(); + return; + } + + // Otherwise, only connect when beneficiaries are loaded AND beneficiaryData is ready if (beneficiariesLoaded && beneficiaryData?.deploymentId) { console.log('[VoiceCall] Starting call with beneficiaryData:', JSON.stringify(beneficiaryData)); connectCalledRef.current = true; @@ -145,7 +163,7 @@ export default function VoiceCallScreen() { beneficiaryData: beneficiaryData ? JSON.stringify(beneficiaryData) : 'undefined' }); } - }, [beneficiariesLoaded, beneficiaryData, beneficiaries.length, connect]); + }, [beneficiariesLoaded, beneficiaryData, beneficiaries.length, connect, debugDeploymentId]); // Fallback: if beneficiaryData doesn't arrive in 5 seconds, connect anyway // This handles edge cases where API fails or user has no beneficiaries diff --git a/contexts/BeneficiaryContext.tsx b/contexts/BeneficiaryContext.tsx index f769228..c50efeb 100644 --- a/contexts/BeneficiaryContext.tsx +++ b/contexts/BeneficiaryContext.tsx @@ -7,12 +7,17 @@ interface BeneficiaryContextType { clearCurrentBeneficiary: () => void; // Helper to format beneficiary context for AI getBeneficiaryContext: () => string; + // Debug: Override deployment ID for testing (used by Debug screen) + debugDeploymentId: string | null; + setDebugDeploymentId: (id: string | null) => void; } const BeneficiaryContext = createContext(undefined); export function BeneficiaryProvider({ children }: { children: React.ReactNode }) { const [currentBeneficiary, setCurrentBeneficiary] = useState(null); + // Debug: Override deployment ID for testing purposes + const [debugDeploymentId, setDebugDeploymentId] = useState(null); const clearCurrentBeneficiary = useCallback(() => { setCurrentBeneficiary(null); @@ -70,6 +75,8 @@ export function BeneficiaryProvider({ children }: { children: React.ReactNode }) setCurrentBeneficiary, clearCurrentBeneficiary, getBeneficiaryContext, + debugDeploymentId, + setDebugDeploymentId, }} > {children}