feat: Integrate LiveKit voice calls into chat screen

- Add voice call button in chat input area
- Implement LiveKit room connection with Julia AI agent
- Create full-screen voice call modal with:
  - Visual avatar with speaking indicator
  - Call duration timer
  - Agent state display (listening/thinking/speaking)
  - Hang up button
- Add real-time transcription tracking for voice calls
- Keep screen awake during active calls
- Integrate with existing VoiceTranscriptContext for history

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Sergei 2026-01-24 20:27:40 -08:00
parent 6f7c79f601
commit 89afe86f54

View File

@ -1,7 +1,7 @@
/** /**
* Chat Screen - Text Chat with Julia AI * Chat Screen - Text Chat with Julia AI
* *
* Clean text chat interface. * Clean text chat interface with integrated voice calls.
*/ */
import React, { useState, useCallback, useRef, useEffect } from 'react'; import React, { useState, useCallback, useRef, useEffect } from 'react';
@ -16,17 +16,37 @@ import {
ActivityIndicator, ActivityIndicator,
Keyboard, Keyboard,
Platform, Platform,
Alert,
} from 'react-native'; } from 'react-native';
import { KeyboardAvoidingView } from 'react-native-keyboard-controller'; import { KeyboardAvoidingView } from 'react-native-keyboard-controller';
import { Ionicons } from '@expo/vector-icons'; import { Ionicons } from '@expo/vector-icons';
import { SafeAreaView } from 'react-native-safe-area-context'; import { SafeAreaView } from 'react-native-safe-area-context';
import { useRouter } from 'expo-router'; import { useRouter } from 'expo-router';
import { activateKeepAwakeAsync, deactivateKeepAwake } from 'expo-keep-awake';
import { api } from '@/services/api'; import { api } from '@/services/api';
import { useBeneficiary } from '@/contexts/BeneficiaryContext'; import { useBeneficiary } from '@/contexts/BeneficiaryContext';
import { useVoiceTranscript } from '@/contexts/VoiceTranscriptContext'; import { useVoiceTranscript } from '@/contexts/VoiceTranscriptContext';
import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme'; import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme';
import type { Message, Beneficiary } from '@/types'; import type { Message, Beneficiary } from '@/types';
// LiveKit imports
import {
registerGlobals,
LiveKitRoom,
useVoiceAssistant,
useConnectionState,
useRoomContext,
BarVisualizer,
useTrackTranscription,
useTracks,
} from '@livekit/react-native';
import { ConnectionState, RoomEvent, Track, TranscriptionSegment } from 'livekit-client';
import { getToken, type BeneficiaryData } from '@/services/livekitService';
import { useAuth } from '@/contexts/AuthContext';
// Register LiveKit globals (must be called before using LiveKit)
registerGlobals();
const API_URL = 'https://eluxnetworks.net/function/well-api/api'; const API_URL = 'https://eluxnetworks.net/function/well-api/api';
// WellNuo API credentials (same as julia-agent) // WellNuo API credentials (same as julia-agent)
@ -106,10 +126,233 @@ function normalizeQuestion(userMessage: string): string {
return userMessage; return userMessage;
} }
// ============================================================================
// Voice Call Overlay Component
// ============================================================================
interface VoiceCallOverlayProps {
onHangUp: () => void;
onTranscript: (role: 'user' | 'assistant', text: string) => void;
beneficiaryName?: string;
}
function VoiceCallContent({ onHangUp, onTranscript, beneficiaryName }: VoiceCallOverlayProps) {
const room = useRoomContext();
const connectionState = useConnectionState();
const { state: agentState, audioTrack } = useVoiceAssistant();
const [callDuration, setCallDuration] = useState(0);
const [lastProcessedId, setLastProcessedId] = useState<string | null>(null);
// Track all audio tracks for transcription
const tracks = useTracks([Track.Source.Microphone], { onlySubscribed: true });
// Get transcription from agent's audio track
const { segments: agentSegments } = useTrackTranscription(audioTrack);
// Get transcription from user's microphone
const localTrack = tracks.find(t => t.participant?.isLocal);
const { segments: userSegments } = useTrackTranscription(localTrack);
// Process agent transcription
useEffect(() => {
if (agentSegments && agentSegments.length > 0) {
const lastSegment = agentSegments[agentSegments.length - 1];
if (lastSegment && lastSegment.final && lastSegment.id !== lastProcessedId) {
setLastProcessedId(lastSegment.id);
onTranscript('assistant', lastSegment.text);
console.log('[VoiceCall] Agent said:', lastSegment.text);
}
}
}, [agentSegments, lastProcessedId, onTranscript]);
// Process user transcription
const [lastUserSegmentId, setLastUserSegmentId] = useState<string | null>(null);
useEffect(() => {
if (userSegments && userSegments.length > 0) {
const lastSegment = userSegments[userSegments.length - 1];
if (lastSegment && lastSegment.final && lastSegment.id !== lastUserSegmentId) {
setLastUserSegmentId(lastSegment.id);
onTranscript('user', lastSegment.text);
console.log('[VoiceCall] User said:', lastSegment.text);
}
}
}, [userSegments, lastUserSegmentId, onTranscript]);
// Call duration timer
useEffect(() => {
if (connectionState === ConnectionState.Connected) {
const interval = setInterval(() => {
setCallDuration(prev => prev + 1);
}, 1000);
return () => clearInterval(interval);
}
}, [connectionState]);
// Keep screen awake during call
useEffect(() => {
activateKeepAwakeAsync('voice-call');
return () => {
deactivateKeepAwake('voice-call');
};
}, []);
// Format duration as mm:ss
const formatDuration = (seconds: number) => {
const mins = Math.floor(seconds / 60);
const secs = seconds % 60;
return `${mins.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
};
// Get status text based on agent state
const getStatusText = () => {
if (connectionState === ConnectionState.Connecting) return 'Connecting...';
if (connectionState === ConnectionState.Reconnecting) return 'Reconnecting...';
if (connectionState !== ConnectionState.Connected) return 'Disconnected';
switch (agentState) {
case 'listening': return 'Listening...';
case 'thinking': return 'Thinking...';
case 'speaking': return 'Speaking...';
case 'connecting': return 'Connecting to Julia...';
case 'initializing': return 'Starting...';
default: return 'Connected';
}
};
return (
<View style={voiceStyles.container}>
<View style={voiceStyles.content}>
{/* Avatar */}
<View style={voiceStyles.avatarContainer}>
<View style={[
voiceStyles.avatar,
agentState === 'speaking' && voiceStyles.avatarSpeaking,
]}>
<Text style={voiceStyles.avatarText}>J</Text>
</View>
{agentState === 'speaking' && (
<View style={voiceStyles.speakingRing} />
)}
</View>
{/* Name and status */}
<Text style={voiceStyles.name}>Julia AI</Text>
{beneficiaryName && (
<Text style={voiceStyles.beneficiary}>About {beneficiaryName}</Text>
)}
<Text style={voiceStyles.status}>{getStatusText()}</Text>
{/* Duration */}
{connectionState === ConnectionState.Connected && (
<Text style={voiceStyles.duration}>{formatDuration(callDuration)}</Text>
)}
{/* Audio Visualizer */}
{audioTrack && agentState === 'speaking' && (
<View style={voiceStyles.visualizerContainer}>
<BarVisualizer
trackRef={{ participant: audioTrack.participant, source: Track.Source.Microphone, publication: audioTrack.publication }}
barCount={5}
options={{ minHeight: 10 }}
/>
</View>
)}
</View>
{/* Hang up button */}
<TouchableOpacity style={voiceStyles.hangUpButton} onPress={onHangUp}>
<Ionicons name="call" size={32} color={AppColors.white} style={{ transform: [{ rotate: '135deg' }] }} />
</TouchableOpacity>
</View>
);
}
const voiceStyles = StyleSheet.create({
container: {
flex: 1,
backgroundColor: 'rgba(0, 0, 0, 0.95)',
justifyContent: 'space-between',
alignItems: 'center',
paddingVertical: 60,
},
content: {
flex: 1,
justifyContent: 'center',
alignItems: 'center',
},
avatarContainer: {
position: 'relative',
marginBottom: Spacing.lg,
},
avatar: {
width: 120,
height: 120,
borderRadius: 60,
backgroundColor: AppColors.success,
justifyContent: 'center',
alignItems: 'center',
},
avatarSpeaking: {
backgroundColor: AppColors.primary,
},
avatarText: {
fontSize: 48,
fontWeight: '600',
color: AppColors.white,
},
speakingRing: {
position: 'absolute',
top: -10,
left: -10,
right: -10,
bottom: -10,
borderRadius: 70,
borderWidth: 3,
borderColor: AppColors.primary,
opacity: 0.5,
},
name: {
fontSize: FontSizes['2xl'],
fontWeight: '600',
color: AppColors.white,
marginBottom: Spacing.xs,
},
beneficiary: {
fontSize: FontSizes.base,
color: 'rgba(255, 255, 255, 0.7)',
marginBottom: Spacing.sm,
},
status: {
fontSize: FontSizes.base,
color: AppColors.success,
marginBottom: Spacing.md,
},
duration: {
fontSize: FontSizes.lg,
color: 'rgba(255, 255, 255, 0.8)',
fontVariant: ['tabular-nums'],
},
visualizerContainer: {
marginTop: Spacing.xl,
height: 60,
width: 200,
},
hangUpButton: {
width: 72,
height: 72,
borderRadius: 36,
backgroundColor: AppColors.error,
justifyContent: 'center',
alignItems: 'center',
marginBottom: Spacing.xl,
},
});
export default function ChatScreen() { export default function ChatScreen() {
const router = useRouter(); const router = useRouter();
const { currentBeneficiary, setCurrentBeneficiary } = useBeneficiary(); const { currentBeneficiary, setCurrentBeneficiary } = useBeneficiary();
const { getTranscriptAsMessages, hasNewTranscript, markTranscriptAsShown } = useVoiceTranscript(); const { getTranscriptAsMessages, hasNewTranscript, markTranscriptAsShown, addTranscriptEntry, clearTranscript } = useVoiceTranscript();
const { user } = useAuth();
// Chat state // Chat state
const [messages, setMessages] = useState<Message[]>([ const [messages, setMessages] = useState<Message[]>([
@ -121,6 +364,12 @@ export default function ChatScreen() {
}, },
]); ]);
// Voice call state
const [isVoiceCallActive, setIsVoiceCallActive] = useState(false);
const [voiceToken, setVoiceToken] = useState<string | undefined>(undefined);
const [voiceWsUrl, setVoiceWsUrl] = useState<string | undefined>(undefined);
const [isConnectingVoice, setIsConnectingVoice] = useState(false);
// Add voice call transcript to messages when returning from call // Add voice call transcript to messages when returning from call
useEffect(() => { useEffect(() => {
if (hasNewTranscript) { if (hasNewTranscript) {
@ -208,6 +457,69 @@ export default function ChatScreen() {
setShowBeneficiaryPicker(false); setShowBeneficiaryPicker(false);
}, [setCurrentBeneficiary]); }, [setCurrentBeneficiary]);
// ============================================================================
// Voice Call Functions
// ============================================================================
// Start voice call
const startVoiceCall = useCallback(async () => {
if (isConnectingVoice || isVoiceCallActive) return;
setIsConnectingVoice(true);
console.log('[Chat] Starting voice call...');
try {
// Build beneficiary data for the agent
const beneficiaryData: BeneficiaryData = {
deploymentId: currentBeneficiary?.id?.toString() || beneficiaries[0]?.id?.toString() || '21',
beneficiaryNamesDict: {},
};
// Add names dict if not in single deployment mode
if (!SINGLE_DEPLOYMENT_MODE) {
beneficiaries.forEach(b => {
beneficiaryData.beneficiaryNamesDict[b.id.toString()] = b.name;
});
}
// Get LiveKit token
const userIdStr = user?.user_id?.toString() || 'user-' + Date.now();
const tokenResponse = await getToken(userIdStr, beneficiaryData);
if (!tokenResponse.success || !tokenResponse.data) {
throw new Error(tokenResponse.error || 'Failed to get voice token');
}
console.log('[Chat] Got voice token, connecting to room:', tokenResponse.data.roomName);
// Clear previous transcript and start call
clearTranscript();
setVoiceToken(tokenResponse.data.token);
setVoiceWsUrl(tokenResponse.data.wsUrl);
setIsVoiceCallActive(true);
} catch (error) {
console.error('[Chat] Voice call error:', error);
Alert.alert(
'Voice Call Error',
error instanceof Error ? error.message : 'Failed to start voice call'
);
} finally {
setIsConnectingVoice(false);
}
}, [isConnectingVoice, isVoiceCallActive, currentBeneficiary, beneficiaries, user, clearTranscript]);
// End voice call
const endVoiceCall = useCallback(() => {
console.log('[Chat] Ending voice call...');
setIsVoiceCallActive(false);
setVoiceToken(undefined);
setVoiceWsUrl(undefined);
}, []);
// Handle voice transcript entries
const handleVoiceTranscript = useCallback((role: 'user' | 'assistant', text: string) => {
addTranscriptEntry(role, text);
}, [addTranscriptEntry]);
// Cached API token for WellNuo // Cached API token for WellNuo
const apiTokenRef = useRef<string | null>(null); const apiTokenRef = useRef<string | null>(null);
@ -466,6 +778,19 @@ export default function ChatScreen() {
{/* Input */} {/* Input */}
<View style={styles.inputContainer}> <View style={styles.inputContainer}>
{/* Voice Call Button */}
<TouchableOpacity
style={[styles.voiceButton, isConnectingVoice && styles.voiceButtonConnecting]}
onPress={startVoiceCall}
disabled={isConnectingVoice}
>
{isConnectingVoice ? (
<ActivityIndicator size="small" color={AppColors.primary} />
) : (
<Ionicons name="call" size={20} color={AppColors.primary} />
)}
</TouchableOpacity>
<TextInput <TextInput
style={styles.input} style={styles.input}
placeholder="Type a message..." placeholder="Type a message..."
@ -489,6 +814,44 @@ export default function ChatScreen() {
</TouchableOpacity> </TouchableOpacity>
</View> </View>
</KeyboardAvoidingView> </KeyboardAvoidingView>
{/* Voice Call Modal */}
<Modal
visible={isVoiceCallActive}
animationType="slide"
presentationStyle="fullScreen"
onRequestClose={endVoiceCall}
>
<SafeAreaView style={{ flex: 1, backgroundColor: 'black' }} edges={['top', 'bottom']}>
{voiceToken && voiceWsUrl ? (
<LiveKitRoom
serverUrl={voiceWsUrl}
token={voiceToken}
connect={true}
audio={true}
video={false}
onConnected={() => console.log('[Chat] LiveKit connected')}
onDisconnected={endVoiceCall}
onError={(error) => {
console.error('[Chat] LiveKit error:', error);
Alert.alert('Voice Call Error', error.message);
endVoiceCall();
}}
>
<VoiceCallContent
onHangUp={endVoiceCall}
onTranscript={handleVoiceTranscript}
beneficiaryName={currentBeneficiary?.name}
/>
</LiveKitRoom>
) : (
<View style={{ flex: 1, justifyContent: 'center', alignItems: 'center' }}>
<ActivityIndicator size="large" color={AppColors.primary} />
<Text style={{ color: 'white', marginTop: 16 }}>Connecting...</Text>
</View>
)}
</SafeAreaView>
</Modal>
</SafeAreaView> </SafeAreaView>
); );
} }
@ -631,6 +994,20 @@ const styles = StyleSheet.create({
maxHeight: 100, maxHeight: 100,
marginRight: Spacing.sm, marginRight: Spacing.sm,
}, },
voiceButton: {
width: 44,
height: 44,
borderRadius: BorderRadius.full,
backgroundColor: AppColors.surface,
justifyContent: 'center',
alignItems: 'center',
marginRight: Spacing.sm,
borderWidth: 1,
borderColor: AppColors.primary,
},
voiceButtonConnecting: {
opacity: 0.6,
},
sendButton: { sendButton: {
width: 44, width: 44,
height: 44, height: 44,