From 89afe86f54814f318fcf42728a84815f7bb9a387 Mon Sep 17 00:00:00 2001 From: Sergei Date: Sat, 24 Jan 2026 20:27:40 -0800 Subject: [PATCH] feat: Integrate LiveKit voice calls into chat screen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add voice call button in chat input area - Implement LiveKit room connection with Julia AI agent - Create full-screen voice call modal with: - Visual avatar with speaking indicator - Call duration timer - Agent state display (listening/thinking/speaking) - Hang up button - Add real-time transcription tracking for voice calls - Keep screen awake during active calls - Integrate with existing VoiceTranscriptContext for history 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- app/(tabs)/chat.tsx | 381 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 379 insertions(+), 2 deletions(-) diff --git a/app/(tabs)/chat.tsx b/app/(tabs)/chat.tsx index 216bf00..68c3a0d 100644 --- a/app/(tabs)/chat.tsx +++ b/app/(tabs)/chat.tsx @@ -1,7 +1,7 @@ /** * Chat Screen - Text Chat with Julia AI * - * Clean text chat interface. + * Clean text chat interface with integrated voice calls. */ import React, { useState, useCallback, useRef, useEffect } from 'react'; @@ -16,17 +16,37 @@ import { ActivityIndicator, Keyboard, Platform, + Alert, } from 'react-native'; import { KeyboardAvoidingView } from 'react-native-keyboard-controller'; import { Ionicons } from '@expo/vector-icons'; import { SafeAreaView } from 'react-native-safe-area-context'; import { useRouter } from 'expo-router'; +import { activateKeepAwakeAsync, deactivateKeepAwake } from 'expo-keep-awake'; import { api } from '@/services/api'; import { useBeneficiary } from '@/contexts/BeneficiaryContext'; import { useVoiceTranscript } from '@/contexts/VoiceTranscriptContext'; import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme'; import type { Message, Beneficiary } from '@/types'; +// LiveKit imports +import { + registerGlobals, + LiveKitRoom, + useVoiceAssistant, + useConnectionState, + useRoomContext, + BarVisualizer, + useTrackTranscription, + useTracks, +} from '@livekit/react-native'; +import { ConnectionState, RoomEvent, Track, TranscriptionSegment } from 'livekit-client'; +import { getToken, type BeneficiaryData } from '@/services/livekitService'; +import { useAuth } from '@/contexts/AuthContext'; + +// Register LiveKit globals (must be called before using LiveKit) +registerGlobals(); + const API_URL = 'https://eluxnetworks.net/function/well-api/api'; // WellNuo API credentials (same as julia-agent) @@ -106,10 +126,233 @@ function normalizeQuestion(userMessage: string): string { return userMessage; } +// ============================================================================ +// Voice Call Overlay Component +// ============================================================================ + +interface VoiceCallOverlayProps { + onHangUp: () => void; + onTranscript: (role: 'user' | 'assistant', text: string) => void; + beneficiaryName?: string; +} + +function VoiceCallContent({ onHangUp, onTranscript, beneficiaryName }: VoiceCallOverlayProps) { + const room = useRoomContext(); + const connectionState = useConnectionState(); + const { state: agentState, audioTrack } = useVoiceAssistant(); + const [callDuration, setCallDuration] = useState(0); + const [lastProcessedId, setLastProcessedId] = useState(null); + + // Track all audio tracks for transcription + const tracks = useTracks([Track.Source.Microphone], { onlySubscribed: true }); + + // Get transcription from agent's audio track + const { segments: agentSegments } = useTrackTranscription(audioTrack); + + // Get transcription from user's microphone + const localTrack = tracks.find(t => t.participant?.isLocal); + const { segments: userSegments } = useTrackTranscription(localTrack); + + // Process agent transcription + useEffect(() => { + if (agentSegments && agentSegments.length > 0) { + const lastSegment = agentSegments[agentSegments.length - 1]; + if (lastSegment && lastSegment.final && lastSegment.id !== lastProcessedId) { + setLastProcessedId(lastSegment.id); + onTranscript('assistant', lastSegment.text); + console.log('[VoiceCall] Agent said:', lastSegment.text); + } + } + }, [agentSegments, lastProcessedId, onTranscript]); + + // Process user transcription + const [lastUserSegmentId, setLastUserSegmentId] = useState(null); + useEffect(() => { + if (userSegments && userSegments.length > 0) { + const lastSegment = userSegments[userSegments.length - 1]; + if (lastSegment && lastSegment.final && lastSegment.id !== lastUserSegmentId) { + setLastUserSegmentId(lastSegment.id); + onTranscript('user', lastSegment.text); + console.log('[VoiceCall] User said:', lastSegment.text); + } + } + }, [userSegments, lastUserSegmentId, onTranscript]); + + // Call duration timer + useEffect(() => { + if (connectionState === ConnectionState.Connected) { + const interval = setInterval(() => { + setCallDuration(prev => prev + 1); + }, 1000); + return () => clearInterval(interval); + } + }, [connectionState]); + + // Keep screen awake during call + useEffect(() => { + activateKeepAwakeAsync('voice-call'); + return () => { + deactivateKeepAwake('voice-call'); + }; + }, []); + + // Format duration as mm:ss + const formatDuration = (seconds: number) => { + const mins = Math.floor(seconds / 60); + const secs = seconds % 60; + return `${mins.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`; + }; + + // Get status text based on agent state + const getStatusText = () => { + if (connectionState === ConnectionState.Connecting) return 'Connecting...'; + if (connectionState === ConnectionState.Reconnecting) return 'Reconnecting...'; + if (connectionState !== ConnectionState.Connected) return 'Disconnected'; + + switch (agentState) { + case 'listening': return 'Listening...'; + case 'thinking': return 'Thinking...'; + case 'speaking': return 'Speaking...'; + case 'connecting': return 'Connecting to Julia...'; + case 'initializing': return 'Starting...'; + default: return 'Connected'; + } + }; + + return ( + + + {/* Avatar */} + + + J + + {agentState === 'speaking' && ( + + )} + + + {/* Name and status */} + Julia AI + {beneficiaryName && ( + About {beneficiaryName} + )} + {getStatusText()} + + {/* Duration */} + {connectionState === ConnectionState.Connected && ( + {formatDuration(callDuration)} + )} + + {/* Audio Visualizer */} + {audioTrack && agentState === 'speaking' && ( + + + + )} + + + {/* Hang up button */} + + + + + ); +} + +const voiceStyles = StyleSheet.create({ + container: { + flex: 1, + backgroundColor: 'rgba(0, 0, 0, 0.95)', + justifyContent: 'space-between', + alignItems: 'center', + paddingVertical: 60, + }, + content: { + flex: 1, + justifyContent: 'center', + alignItems: 'center', + }, + avatarContainer: { + position: 'relative', + marginBottom: Spacing.lg, + }, + avatar: { + width: 120, + height: 120, + borderRadius: 60, + backgroundColor: AppColors.success, + justifyContent: 'center', + alignItems: 'center', + }, + avatarSpeaking: { + backgroundColor: AppColors.primary, + }, + avatarText: { + fontSize: 48, + fontWeight: '600', + color: AppColors.white, + }, + speakingRing: { + position: 'absolute', + top: -10, + left: -10, + right: -10, + bottom: -10, + borderRadius: 70, + borderWidth: 3, + borderColor: AppColors.primary, + opacity: 0.5, + }, + name: { + fontSize: FontSizes['2xl'], + fontWeight: '600', + color: AppColors.white, + marginBottom: Spacing.xs, + }, + beneficiary: { + fontSize: FontSizes.base, + color: 'rgba(255, 255, 255, 0.7)', + marginBottom: Spacing.sm, + }, + status: { + fontSize: FontSizes.base, + color: AppColors.success, + marginBottom: Spacing.md, + }, + duration: { + fontSize: FontSizes.lg, + color: 'rgba(255, 255, 255, 0.8)', + fontVariant: ['tabular-nums'], + }, + visualizerContainer: { + marginTop: Spacing.xl, + height: 60, + width: 200, + }, + hangUpButton: { + width: 72, + height: 72, + borderRadius: 36, + backgroundColor: AppColors.error, + justifyContent: 'center', + alignItems: 'center', + marginBottom: Spacing.xl, + }, +}); + export default function ChatScreen() { const router = useRouter(); const { currentBeneficiary, setCurrentBeneficiary } = useBeneficiary(); - const { getTranscriptAsMessages, hasNewTranscript, markTranscriptAsShown } = useVoiceTranscript(); + const { getTranscriptAsMessages, hasNewTranscript, markTranscriptAsShown, addTranscriptEntry, clearTranscript } = useVoiceTranscript(); + const { user } = useAuth(); // Chat state const [messages, setMessages] = useState([ @@ -121,6 +364,12 @@ export default function ChatScreen() { }, ]); + // Voice call state + const [isVoiceCallActive, setIsVoiceCallActive] = useState(false); + const [voiceToken, setVoiceToken] = useState(undefined); + const [voiceWsUrl, setVoiceWsUrl] = useState(undefined); + const [isConnectingVoice, setIsConnectingVoice] = useState(false); + // Add voice call transcript to messages when returning from call useEffect(() => { if (hasNewTranscript) { @@ -208,6 +457,69 @@ export default function ChatScreen() { setShowBeneficiaryPicker(false); }, [setCurrentBeneficiary]); + // ============================================================================ + // Voice Call Functions + // ============================================================================ + + // Start voice call + const startVoiceCall = useCallback(async () => { + if (isConnectingVoice || isVoiceCallActive) return; + + setIsConnectingVoice(true); + console.log('[Chat] Starting voice call...'); + + try { + // Build beneficiary data for the agent + const beneficiaryData: BeneficiaryData = { + deploymentId: currentBeneficiary?.id?.toString() || beneficiaries[0]?.id?.toString() || '21', + beneficiaryNamesDict: {}, + }; + + // Add names dict if not in single deployment mode + if (!SINGLE_DEPLOYMENT_MODE) { + beneficiaries.forEach(b => { + beneficiaryData.beneficiaryNamesDict[b.id.toString()] = b.name; + }); + } + + // Get LiveKit token + const userIdStr = user?.user_id?.toString() || 'user-' + Date.now(); + const tokenResponse = await getToken(userIdStr, beneficiaryData); + + if (!tokenResponse.success || !tokenResponse.data) { + throw new Error(tokenResponse.error || 'Failed to get voice token'); + } + + console.log('[Chat] Got voice token, connecting to room:', tokenResponse.data.roomName); + + // Clear previous transcript and start call + clearTranscript(); + setVoiceToken(tokenResponse.data.token); + setVoiceWsUrl(tokenResponse.data.wsUrl); + setIsVoiceCallActive(true); + } catch (error) { + console.error('[Chat] Voice call error:', error); + Alert.alert( + 'Voice Call Error', + error instanceof Error ? error.message : 'Failed to start voice call' + ); + } finally { + setIsConnectingVoice(false); + } + }, [isConnectingVoice, isVoiceCallActive, currentBeneficiary, beneficiaries, user, clearTranscript]); + + // End voice call + const endVoiceCall = useCallback(() => { + console.log('[Chat] Ending voice call...'); + setIsVoiceCallActive(false); + setVoiceToken(undefined); + setVoiceWsUrl(undefined); + }, []); + + // Handle voice transcript entries + const handleVoiceTranscript = useCallback((role: 'user' | 'assistant', text: string) => { + addTranscriptEntry(role, text); + }, [addTranscriptEntry]); // Cached API token for WellNuo const apiTokenRef = useRef(null); @@ -466,6 +778,19 @@ export default function ChatScreen() { {/* Input */} + {/* Voice Call Button */} + + {isConnectingVoice ? ( + + ) : ( + + )} + + + + {/* Voice Call Modal */} + + + {voiceToken && voiceWsUrl ? ( + console.log('[Chat] LiveKit connected')} + onDisconnected={endVoiceCall} + onError={(error) => { + console.error('[Chat] LiveKit error:', error); + Alert.alert('Voice Call Error', error.message); + endVoiceCall(); + }} + > + + + ) : ( + + + Connecting... + + )} + + ); } @@ -631,6 +994,20 @@ const styles = StyleSheet.create({ maxHeight: 100, marginRight: Spacing.sm, }, + voiceButton: { + width: 44, + height: 44, + borderRadius: BorderRadius.full, + backgroundColor: AppColors.surface, + justifyContent: 'center', + alignItems: 'center', + marginRight: Spacing.sm, + borderWidth: 1, + borderColor: AppColors.primary, + }, + voiceButtonConnecting: { + opacity: 0.6, + }, sendButton: { width: 44, height: 44,