From c1380b55dd87ad99611be861cb64d28999710e87 Mon Sep 17 00:00:00 2001 From: Sergei Date: Fri, 16 Jan 2026 12:20:17 -0800 Subject: [PATCH] Add unified assistant with Ultravox voice AI Chat screen now supports both: - Text messaging (keyboard input) - High-quality Ultravox voice calls (WebRTC) Features: - Voice call button in input bar (phone icon) - Green status bar when call is active - Transcripts from voice calls appear in chat history - Voice badge on messages from voice conversation - Mute button during calls - Auto-end call when leaving screen Background audio configured for iOS (audio, voip modes) --- app.json | 15 +- app/(tabs)/_layout.tsx | 16 +- app/(tabs)/chat.tsx | 1052 ++++++++++------------- app/(tabs)/voice.tsx | 515 +++++++++++ assets/data/ferdinand_7days_events.json | 983 +++++++++++++++++++++ services/ultravoxService.ts | 272 ++++++ types/index.ts | 2 + 7 files changed, 2250 insertions(+), 605 deletions(-) create mode 100644 app/(tabs)/voice.tsx create mode 100644 assets/data/ferdinand_7days_events.json create mode 100644 services/ultravoxService.ts diff --git a/app.json b/app.json index deb6050..9d8182f 100644 --- a/app.json +++ b/app.json @@ -16,7 +16,8 @@ "infoPlist": { "ITSAppUsesNonExemptEncryption": false, "NSMicrophoneUsageDescription": "WellNuo needs access to your microphone for voice input to the AI assistant.", - "NSSpeechRecognitionUsageDescription": "WellNuo uses speech recognition to convert your voice to text for the AI assistant." + "NSSpeechRecognitionUsageDescription": "WellNuo uses speech recognition to convert your voice to text for the AI assistant.", + "UIBackgroundModes": ["audio", "voip"] } }, "android": { @@ -28,13 +29,21 @@ "monochromeImage": "./assets/images/android-icon-monochrome.png" }, "edgeToEdgeEnabled": true, - "predictiveBackGestureEnabled": false + "predictiveBackGestureEnabled": false, + "permissions": [ + "android.permission.RECORD_AUDIO", + "android.permission.FOREGROUND_SERVICE", + "android.permission.FOREGROUND_SERVICE_MICROPHONE", + "android.permission.WAKE_LOCK" + ] }, "web": { "output": "static", "favicon": "./assets/images/favicon.png" }, "plugins": [ + "@livekit/react-native-expo-plugin", + "@config-plugins/react-native-webrtc", "expo-router", [ "expo-splash-screen", @@ -66,6 +75,6 @@ "projectId": "4f415b4b-41c8-4b98-989c-32f6b3f97481" } }, - "owner": "serter2069ya" + "owner": "serter20692" } } diff --git a/app/(tabs)/_layout.tsx b/app/(tabs)/_layout.tsx index 77f48cd..53f515f 100644 --- a/app/(tabs)/_layout.tsx +++ b/app/(tabs)/_layout.tsx @@ -46,15 +46,23 @@ export default function TabLayout() { href: null, }} /> + {/* Chat with text + voice input - main assistant screen */} ( ), }} /> + {/* Voice-only screen hidden - Chat has both text and voice */} + + {/* Debug hidden */} ( - - ), + href: null, }} /> {/* Hide explore tab */} diff --git a/app/(tabs)/chat.tsx b/app/(tabs)/chat.tsx index 369768e..8cdd278 100644 --- a/app/(tabs)/chat.tsx +++ b/app/(tabs)/chat.tsx @@ -1,3 +1,13 @@ +/** + * Unified Chat Screen - Text Chat + Ultravox Voice AI + * + * Features: + * - Text messaging with AI (keyboard input) + * - High-quality Ultravox voice calls (WebRTC) + * - Chat history with transcripts from voice calls + * - Seamless switching between text and voice + */ + import React, { useState, useCallback, useRef, useEffect } from 'react'; import { View, @@ -12,33 +22,41 @@ import { ActivityIndicator, Keyboard, Animated, - Alert, - Linking, - ScrollView, + Easing, } from 'react-native'; -import { Ionicons } from '@expo/vector-icons'; +import { Ionicons, Feather } from '@expo/vector-icons'; import { SafeAreaView } from 'react-native-safe-area-context'; import * as SecureStore from 'expo-secure-store'; import { useRouter } from 'expo-router'; +import { useFocusEffect } from '@react-navigation/native'; +import { + useUltravox, + UltravoxSessionStatus, +} from 'ultravox-react-native'; import { api } from '@/services/api'; import { useBeneficiary } from '@/contexts/BeneficiaryContext'; import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme'; import type { Message, Beneficiary } from '@/types'; -import { useSpeechRecognition } from '@/hooks/useSpeechRecognition'; -import sherpaTTS from '@/services/sherpaTTS'; -import { VoiceIndicator } from '@/components/VoiceIndicator'; -import { TTSErrorBoundary } from '@/components/TTSErrorBoundary'; +import { + createCall, + getSystemPrompt, + VOICE_NAME, +} from '@/services/ultravoxService'; const API_URL = 'https://eluxnetworks.net/function/well-api/api'; -function ChatScreenContent() { +type VoiceCallState = 'idle' | 'connecting' | 'active' | 'ending'; + +export default function ChatScreen() { const router = useRouter(); - const { currentBeneficiary, setCurrentBeneficiary, getBeneficiaryContext } = useBeneficiary(); + const { currentBeneficiary, setCurrentBeneficiary } = useBeneficiary(); + + // Chat state const [messages, setMessages] = useState([ { id: '1', role: 'assistant', - content: 'Hello! I\'m Julia, your AI assistant. How can I help you today?', + content: 'Hello! I\'m Julia, your AI wellness assistant. You can type a message or tap the phone button to start a voice call.', timestamp: new Date(), }, ]); @@ -46,59 +64,103 @@ function ChatScreenContent() { const [isSending, setIsSending] = useState(false); const flatListRef = useRef(null); - // Voice state - const [isSpeaking, setIsSpeaking] = useState(false); - const [ttsInitialized, setTtsInitialized] = useState(false); - const [voiceFeedback, setVoiceFeedback] = useState(null); - const [isVoiceConversation, setIsVoiceConversation] = useState(false); // Auto-listen mode + // Voice call state (Ultravox) + const [voiceCallState, setVoiceCallState] = useState('idle'); + const [isMuted, setIsMuted] = useState(false); + + // Animations const pulseAnim = useRef(new Animated.Value(1)).current; + const rotateAnim = useRef(new Animated.Value(0)).current; - // Speech recognition hook - const { - isListening, - recognizedText, - startListening, - stopListening, - isAvailable: speechRecognitionAvailable, - requestPermission, - } = useSpeechRecognition(); - - // Beneficiary picker state + // Beneficiary picker const [showBeneficiaryPicker, setShowBeneficiaryPicker] = useState(false); const [beneficiaries, setBeneficiaries] = useState([]); const [loadingBeneficiaries, setLoadingBeneficiaries] = useState(false); - // Initialize TTS on mount - useEffect(() => { - const initTTS = async () => { - try { - const success = await sherpaTTS.initialize(); - setTtsInitialized(success); - console.log('[Chat] SherpaTTS initialized:', success); - } catch (error) { - console.log('[Chat] SherpaTTS init failed, will use fallback'); + // Tool implementations for Ultravox navigation + const toolImplementations = { + navigateToDashboard: () => { + console.log('[Chat] Tool: navigateToDashboard'); + router.push('/(tabs)/dashboard'); + return 'Navigating to Dashboard'; + }, + navigateToBeneficiaries: () => { + console.log('[Chat] Tool: navigateToBeneficiaries'); + router.push('/(tabs)/beneficiaries'); + return 'Navigating to Beneficiaries'; + }, + navigateToProfile: () => { + console.log('[Chat] Tool: navigateToProfile'); + router.push('/(tabs)/profile'); + return 'Navigating to Profile'; + }, + }; + + // Ultravox hook for voice calls + const { transcripts, joinCall, leaveCall, session } = useUltravox({ + tools: toolImplementations, + onStatusChange: (event) => { + console.log('[Chat] Ultravox status:', event.status); + + switch (event.status) { + case UltravoxSessionStatus.IDLE: + case UltravoxSessionStatus.DISCONNECTED: + setVoiceCallState('idle'); + break; + case UltravoxSessionStatus.CONNECTING: + setVoiceCallState('connecting'); + break; + case UltravoxSessionStatus.LISTENING: + case UltravoxSessionStatus.THINKING: + case UltravoxSessionStatus.SPEAKING: + setVoiceCallState('active'); + break; + case UltravoxSessionStatus.DISCONNECTING: + setVoiceCallState('ending'); + break; } - }; - initTTS(); + }, + }); - return () => { - sherpaTTS.deinitialize(); - }; - }, []); - - // Pulse animation for listening state + // Add voice transcripts to chat history useEffect(() => { - if (isListening) { + if (transcripts.length > 0) { + const lastTranscript = transcripts[transcripts.length - 1]; + + // Check if this transcript is already in messages (by content match) + const isDuplicate = messages.some( + m => m.content === lastTranscript.text && + m.role === (lastTranscript.speaker === 'agent' ? 'assistant' : 'user') + ); + + if (!isDuplicate && lastTranscript.text.trim()) { + const newMessage: Message = { + id: `voice-${Date.now()}`, + role: lastTranscript.speaker === 'agent' ? 'assistant' : 'user', + content: lastTranscript.text, + timestamp: new Date(), + isVoice: true, + }; + setMessages(prev => [...prev, newMessage]); + } + } + }, [transcripts]); + + // Pulse animation when voice call is active + useEffect(() => { + if (voiceCallState === 'active') { const pulse = Animated.loop( Animated.sequence([ Animated.timing(pulseAnim, { - toValue: 1.3, - duration: 500, + toValue: 1.15, + duration: 1000, + easing: Easing.inOut(Easing.ease), useNativeDriver: true, }), Animated.timing(pulseAnim, { toValue: 1, - duration: 500, + duration: 1000, + easing: Easing.inOut(Easing.ease), useNativeDriver: true, }), ]) @@ -108,239 +170,123 @@ function ChatScreenContent() { } else { pulseAnim.setValue(1); } - }, [isListening, pulseAnim]); + }, [voiceCallState, pulseAnim]); - // Track if we were just listening (to show feedback when stopped) - const wasListeningRef = useRef(false); - - // Auto-send when speech recognition completes + // Rotate animation when connecting useEffect(() => { - if (!isListening && wasListeningRef.current) { - // We just stopped listening - wasListeningRef.current = false; - - if (recognizedText.trim()) { - // We have text - send it - setInput(recognizedText); - setTimeout(() => { - if (recognizedText.trim()) { - handleVoiceSend(recognizedText.trim()); - } - }, 300); - } else { - // No text recognized (C4 scenario) - show brief feedback - setInput(''); - setVoiceFeedback("Didn't catch that. Try again."); - // Auto-hide after 2 seconds - setTimeout(() => setVoiceFeedback(null), 2000); - } - } - - if (isListening) { - wasListeningRef.current = true; - } - }, [isListening, recognizedText]); - - // Auto-start listening after TTS finishes - const autoStartListening = useCallback(async () => { - if (!isVoiceConversation) return; - - // IMPORTANT: Wait longer to ensure TTS audio has fully stopped - // This prevents the microphone from capturing TTS output - await new Promise(resolve => setTimeout(resolve, 800)); - - // Double-check we're not speaking anymore (TTS may have restarted) - const stillSpeaking = await sherpaTTS.isSpeaking().catch(() => false); - if (stillSpeaking) { - console.log('[Chat] TTS still speaking, not starting listening yet'); - return; - } - - const hasPermission = await requestPermission(); - if (hasPermission && isVoiceConversation) { - console.log('[Chat] Auto-starting listening after TTS'); - startListening({ continuous: false }); - } - }, [isVoiceConversation, requestPermission, startListening]); - - // TTS function - use SherpaTTS or fallback to expo-speech - const speakText = useCallback(async (text: string, shouldAutoListen = false) => { - if (isSpeaking) return; - - // CRITICAL: Stop any active listening BEFORE TTS starts - // This prevents the microphone from capturing TTS audio output - if (isListening) { - console.log('[Chat] Stopping listening before TTS'); - stopListening(); - wasListeningRef.current = false; // Prevent auto-send of any partial text - } - - setIsSpeaking(true); - if (shouldAutoListen) { - setIsVoiceConversation(true); - } - - const handleDone = () => { - setIsSpeaking(false); - // Auto-start listening if in voice conversation mode - if (shouldAutoListen || isVoiceConversation) { - autoStartListening(); - } - }; - - try { - if (ttsInitialized && sherpaTTS.isAvailable()) { - await sherpaTTS.speak(text, { - onDone: handleDone, - onError: (error) => { - console.error('[Chat] TTS speak error:', error); - setIsSpeaking(false); - }, - }); - } else { - console.warn('[Chat] TTS not available'); - setIsSpeaking(false); - } - } catch (error) { - console.error('[Chat] TTS error:', error); - setIsSpeaking(false); - } - }, [isSpeaking, ttsInitialized, isVoiceConversation, autoStartListening]); - - // Stop TTS only (without exiting voice mode) - const stopTTS = useCallback(() => { - if (ttsInitialized && sherpaTTS.isAvailable()) { - sherpaTTS.stop(); - } - setIsSpeaking(false); - }, [ttsInitialized]); - - // Stop TTS and exit voice conversation mode completely - const stopSpeaking = useCallback(() => { - stopTTS(); - setIsVoiceConversation(false); // Exit voice mode when user stops TTS - }, [stopTTS]); - - // Smart handler for VoiceIndicator tap - behavior depends on current mode - const handleVoiceIndicatorTap = useCallback(async (currentMode: 'listening' | 'speaking') => { - console.log('[Chat] VoiceIndicator tapped in mode:', currentMode); - - if (currentMode === 'listening') { - // User tapped while we're recording their voice - // Action: Cancel recording and exit voice mode completely - console.log('[Chat] Cancelling listening, exiting voice mode'); - stopListening(); - setIsVoiceConversation(false); - wasListeningRef.current = false; // Prevent auto-send of partial text - setInput(''); // Clear any partial text - } else if (currentMode === 'speaking') { - // User tapped while AI is speaking - // Action: Interrupt AI and immediately start listening to user (like interrupting in conversation) - console.log('[Chat] Interrupting AI speech, starting to listen'); - stopTTS(); - - // Small delay then start listening - await new Promise(resolve => setTimeout(resolve, 200)); - - const hasPermission = await requestPermission(); - if (hasPermission) { - startListening({ continuous: false }); - } - } - }, [stopListening, stopTTS, requestPermission, startListening]); - - // Show permission denied alert - const showPermissionDeniedAlert = useCallback(() => { - Alert.alert( - 'Microphone Access Required', - 'To use voice input, please allow microphone access in Settings.', - [ - { text: 'Cancel', style: 'cancel' }, - { - text: 'Open Settings', - onPress: () => Linking.openSettings(), - }, - ] - ); - }, []); - - // Handle voice input toggle - const handleVoiceToggle = useCallback(async () => { - if (isListening) { - // User tapped while listening - stop and check if we have text - stopListening(); - // Note: The useEffect below handles auto-send if recognizedText exists - // If no text was recognized, it just cancels (B3 scenario) + if (voiceCallState === 'connecting') { + const rotate = Animated.loop( + Animated.timing(rotateAnim, { + toValue: 1, + duration: 1500, + easing: Easing.linear, + useNativeDriver: true, + }) + ); + rotate.start(); + return () => rotate.stop(); } else { - // Stop any ongoing speech first - if (isSpeaking) { - stopSpeaking(); - } - - // Dismiss keyboard (E1 scenario) - Keyboard.dismiss(); - - // Request permission if needed - const hasPermission = await requestPermission(); - if (!hasPermission) { - // Show alert with option to open settings (C1 scenario) - showPermissionDeniedAlert(); - return; - } - - startListening({ continuous: false }); + rotateAnim.setValue(0); } - }, [isListening, isSpeaking, startListening, stopListening, stopSpeaking, requestPermission, showPermissionDeniedAlert]); + }, [voiceCallState, rotateAnim]); - // Handle sending voice message - const handleVoiceSend = useCallback(async (text: string) => { - if (!text.trim() || isSending) return; + // Start voice call with Ultravox + const startVoiceCall = useCallback(async () => { + setVoiceCallState('connecting'); + Keyboard.dismiss(); - // Mark that we're in voice conversation mode - setIsVoiceConversation(true); - - const userMessage: Message = { - id: Date.now().toString(), - role: 'user', - content: text, + // Add system message + const systemMsg: Message = { + id: `system-${Date.now()}`, + role: 'assistant', + content: '📞 Starting voice call...', timestamp: new Date(), + isSystem: true, }; + setMessages(prev => [...prev, systemMsg]); - setMessages((prev) => [...prev, userMessage]); - setInput(''); - setIsSending(true); + const systemPrompt = getSystemPrompt(); try { - const aiResponse = await sendWithContext(text); + const result = await createCall({ + systemPrompt, + firstSpeaker: 'FIRST_SPEAKER_AGENT', + }); - const assistantMessage: Message = { - id: (Date.now() + 1).toString(), - role: 'assistant', - content: aiResponse, - timestamp: new Date(), - }; - setMessages((prev) => [...prev, assistantMessage]); + if (!result.success) { + throw new Error(result.error); + } - // Speak the response with auto-listen enabled - speakText(aiResponse, true); - } catch (error) { - const errorText = `Sorry, I encountered an error: ${error instanceof Error ? error.message : 'Unknown error'}`; - const errorMessage: Message = { - id: (Date.now() + 1).toString(), - role: 'assistant', - content: errorText, - timestamp: new Date(), - }; - setMessages((prev) => [...prev, errorMessage]); + console.log('[Chat] Call created, joining...'); + await joinCall(result.data.joinUrl); - // Speak error message with auto-listen enabled - speakText(errorText, true); - } finally { - setIsSending(false); + // Update system message + setMessages(prev => prev.map(m => + m.id === systemMsg.id + ? { ...m, content: '📞 Voice call connected. Julia is listening...' } + : m + )); + } catch (err) { + console.error('[Chat] Failed to start voice call:', err); + setVoiceCallState('idle'); + + // Update with error + setMessages(prev => prev.map(m => + m.id === systemMsg.id + ? { ...m, content: `❌ Failed to connect: ${err instanceof Error ? err.message : 'Unknown error'}` } + : m + )); } - }, [isSending, speakText]); + }, [joinCall]); - // Load beneficiaries when picker opens + // End voice call + const endVoiceCall = useCallback(async () => { + setVoiceCallState('ending'); + try { + await leaveCall(); + } catch (err) { + console.error('[Chat] Error leaving call:', err); + } + setVoiceCallState('idle'); + + // Add end message + const endMsg: Message = { + id: `system-end-${Date.now()}`, + role: 'assistant', + content: '📞 Voice call ended.', + timestamp: new Date(), + isSystem: true, + }; + setMessages(prev => [...prev, endMsg]); + }, [leaveCall]); + + // Toggle mute + const toggleMute = useCallback(() => { + if (session) { + const newMuted = !isMuted; + if (newMuted) { + session.muteMic(); + } else { + session.unmuteMic(); + } + setIsMuted(newMuted); + } + }, [session, isMuted]); + + // End call when leaving screen + useFocusEffect( + useCallback(() => { + return () => { + if (voiceCallState === 'active' || voiceCallState === 'connecting') { + console.log('[Chat] Screen unfocused, ending voice call'); + leaveCall().catch(console.error); + setVoiceCallState('idle'); + } + }; + }, [voiceCallState, leaveCall]) + ); + + // Load beneficiaries const loadBeneficiaries = useCallback(async () => { setLoadingBeneficiaries(true); try { @@ -358,18 +304,17 @@ function ChatScreenContent() { } }, []); - // Auto-select first beneficiary on mount if none selected + // Auto-select first beneficiary useEffect(() => { - const autoSelectBeneficiary = async () => { + const autoSelect = async () => { if (!currentBeneficiary) { const loaded = await loadBeneficiaries(); if (loaded.length > 0) { setCurrentBeneficiary(loaded[0]); - console.log('Auto-selected first beneficiary:', loaded[0].name); } } }; - autoSelectBeneficiary(); + autoSelect(); }, []); const openBeneficiaryPicker = useCallback(() => { @@ -382,176 +327,11 @@ function ChatScreenContent() { setShowBeneficiaryPicker(false); }, [setCurrentBeneficiary]); - // Fetch activity data for context - const getActivityContext = async (token: string, userName: string, deploymentId: string): Promise => { - try { - const response = await fetch(API_URL, { - method: 'POST', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - body: new URLSearchParams({ - function: 'activities_report_details', - user_name: userName, - token: token, - deployment_id: deploymentId, - filter: '0', - }).toString(), - }); - - const data = await response.json(); - if (!data.chart_data || data.chart_data.length === 0) return ''; - - const weeklyData = data.chart_data.find((d: any) => d.name === 'Weekly'); - if (!weeklyData) return ''; - - const lines: string[] = []; - if (data.alert_text) lines.push(`Alert status: ${data.alert_text}`); - - const todayStats: string[] = []; - for (const room of weeklyData.rooms) { - const todayData = room.data[room.data.length - 1]; - if (todayData && todayData.hours > 0) { - todayStats.push(`${room.name}: ${todayData.hours.toFixed(1)} hours (${todayData.events} events)`); - } - } - if (todayStats.length > 0) lines.push(`Today's activity: ${todayStats.join(', ')}`); - - const weeklyStats: string[] = []; - for (const room of weeklyData.rooms) { - const totalHours = room.data.reduce((sum: number, d: any) => sum + d.hours, 0); - if (totalHours > 0) { - weeklyStats.push(`${room.name}: ${totalHours.toFixed(1)} hours total this week`); - } - } - if (weeklyStats.length > 0) lines.push(`Weekly summary: ${weeklyStats.join(', ')}`); - - return lines.join('. '); - } catch (error) { - console.log('Failed to fetch activity context:', error); - return ''; - } - }; - - // Fetch dashboard data as fallback context - const getDashboardContext = async (token: string, userName: string, deploymentId: string): Promise => { - try { - const today = new Date().toISOString().split('T')[0]; - - const response = await fetch(API_URL, { - method: 'POST', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - body: new URLSearchParams({ - function: 'dashboard_single', - user_name: userName, - token: token, - deployment_id: deploymentId, - date: today, - }).toString(), - }); - - const data = await response.json(); - if (!data.result_list || data.result_list.length === 0) return ''; - - const info = data.result_list[0]; - const lines: string[] = []; - - if (info.wellness_descriptor) lines.push(`Current wellness: ${info.wellness_descriptor}`); - if (info.wellness_score_percent) lines.push(`Wellness score: ${info.wellness_score_percent}%`); - if (info.last_location) lines.push(`Last seen in: ${info.last_location}`); - if (info.last_detected_time) lines.push(`Last activity: ${info.last_detected_time}`); - if (info.sleep_hours) lines.push(`Sleep hours: ${info.sleep_hours}`); - if (info.temperature) lines.push(`Temperature: ${info.temperature}${info.units === 'F' ? '°F' : '°C'}`); - - return lines.join('. '); - } catch (error) { - console.log('Failed to fetch dashboard context:', error); - return ''; - } - }; - - // Send message with full context - fetches context in parallel for speed - const sendWithContext = async (question: string): Promise => { - const token = await SecureStore.getItemAsync('accessToken'); - const userName = await SecureStore.getItemAsync('userName'); - - if (!token || !userName) throw new Error('Please log in'); - - // Auto-select first beneficiary if none selected - let beneficiary = currentBeneficiary; - if (!beneficiary?.id) { - console.log('[Chat] No beneficiary selected, auto-loading first one...'); - const loaded = await loadBeneficiaries(); - if (loaded.length > 0) { - beneficiary = loaded[0]; - setCurrentBeneficiary(beneficiary); - console.log('[Chat] Auto-selected beneficiary:', beneficiary.name); - } else { - throw new Error('No beneficiaries found. Please add one first.'); - } - } - - const beneficiaryName = beneficiary.name || 'the patient'; - const deploymentId = beneficiary.id.toString(); - - // Fetch both contexts in PARALLEL for speed - const [activityContext, dashboardContext] = await Promise.all([ - getActivityContext(token, userName, deploymentId), - getDashboardContext(token, userName, deploymentId), - ]); - - // Use activity context, fallback to dashboard - const context = activityContext || dashboardContext; - - // Build the question with embedded context - let enhancedQuestion: string; - if (context) { - enhancedQuestion = `You are a caring assistant helping monitor ${beneficiaryName}'s wellbeing. - -Here is the current data about ${beneficiaryName}: -${context} - -Based on this data, please answer the following question: ${question}`; - } else { - enhancedQuestion = `You are a caring assistant helping monitor ${beneficiaryName}'s wellbeing. Please answer: ${question}`; - } - - // Call API - const requestBody = new URLSearchParams({ - function: 'voice_ask', - clientId: '001', - user_name: userName, - token: token, - question: enhancedQuestion, - deployment_id: deploymentId, - context: context || '', - }).toString(); - - const response = await fetch(API_URL, { - method: 'POST', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - body: requestBody, - }); - - const data = await response.json(); - - if (data.ok && data.response?.body) { - return data.response.body; - } else if (data.status === '401 Unauthorized') { - throw new Error('Session expired. Please log in again.'); - } else { - throw new Error('Could not get response'); - } - }; - - const handleSend = useCallback(async () => { + // Text chat - send message via API + const sendTextMessage = useCallback(async () => { const trimmedInput = input.trim(); if (!trimmedInput || isSending) return; - // If no beneficiary selected, auto-selection should have happened - // but if still none, just proceed without context - if (!currentBeneficiary?.id) { - console.log('No beneficiary selected, proceeding without context'); - } - const userMessage: Message = { id: Date.now().toString(), role: 'user', @@ -559,61 +339,102 @@ Based on this data, please answer the following question: ${question}`; timestamp: new Date(), }; - setMessages((prev) => [...prev, userMessage]); + setMessages(prev => [...prev, userMessage]); setInput(''); setIsSending(true); Keyboard.dismiss(); try { - const aiResponse = await sendWithContext(trimmedInput); + const token = await SecureStore.getItemAsync('accessToken'); + const userName = await SecureStore.getItemAsync('userName'); - const assistantMessage: Message = { - id: (Date.now() + 1).toString(), - role: 'assistant', - content: aiResponse, - timestamp: new Date(), - }; - setMessages((prev) => [...prev, assistantMessage]); + if (!token || !userName) { + throw new Error('Please log in'); + } + + // Get beneficiary context + let beneficiary = currentBeneficiary; + if (!beneficiary?.id) { + const loaded = await loadBeneficiaries(); + if (loaded.length > 0) { + beneficiary = loaded[0]; + setCurrentBeneficiary(beneficiary); + } + } + + const beneficiaryName = beneficiary?.name || 'the patient'; + const deploymentId = beneficiary?.id?.toString() || ''; + + // Call API + const response = await fetch(API_URL, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ + function: 'voice_ask', + clientId: '001', + user_name: userName, + token: token, + question: `You are Julia, a caring assistant helping monitor ${beneficiaryName}'s wellbeing. Answer: ${trimmedInput}`, + deployment_id: deploymentId, + context: '', + }).toString(), + }); + + const data = await response.json(); + + if (data.ok && data.response?.body) { + const assistantMessage: Message = { + id: (Date.now() + 1).toString(), + role: 'assistant', + content: data.response.body, + timestamp: new Date(), + }; + setMessages(prev => [...prev, assistantMessage]); + } else { + throw new Error(data.status === '401 Unauthorized' ? 'Session expired' : 'Could not get response'); + } } catch (error) { const errorMessage: Message = { id: (Date.now() + 1).toString(), role: 'assistant', - content: `Sorry, I encountered an error: ${error instanceof Error ? error.message : 'Unknown error'}. Please try again.`, + content: `Sorry, I encountered an error: ${error instanceof Error ? error.message : 'Unknown error'}`, timestamp: new Date(), }; - setMessages((prev) => [...prev, errorMessage]); + setMessages(prev => [...prev, errorMessage]); } finally { setIsSending(false); } - }, [input, isSending, currentBeneficiary]); + }, [input, isSending, currentBeneficiary, loadBeneficiaries, setCurrentBeneficiary]); + // Render message bubble const renderMessage = ({ item }: { item: Message }) => { const isUser = item.role === 'user'; + const isSystem = (item as any).isSystem; + const isVoice = (item as any).isVoice; + + if (isSystem) { + return ( + + {item.content} + + ); + } return ( - + {!isUser && ( J )} - - + + {isVoice && ( + + + Voice + + )} + {item.content} @@ -624,15 +445,52 @@ Based on this data, please answer the following question: ${question}`; ); }; + // Voice call button with animations + const renderVoiceCallButton = () => { + const spin = rotateAnim.interpolate({ + inputRange: [0, 1], + outputRange: ['0deg', '360deg'], + }); + + if (voiceCallState === 'connecting') { + return ( + + + + ); + } + + if (voiceCallState === 'active') { + return ( + + + + + + ); + } + + if (voiceCallState === 'ending') { + return ( + + + + ); + } + + // Idle state + return ( + + + + ); + }; + return ( {/* Header */} - router.push('/(tabs)/dashboard')} - hitSlop={{ top: 10, bottom: 10, left: 10, right: 10 }} - > + router.push('/(tabs)')}> @@ -642,21 +500,45 @@ Based on this data, please answer the following question: ${question}`; Julia AI - {isSending - ? 'Typing...' - : currentBeneficiary - ? `About ${currentBeneficiary.name}` - : 'Online'} + {voiceCallState === 'active' + ? `In call • ${VOICE_NAME}` + : voiceCallState === 'connecting' + ? 'Connecting...' + : currentBeneficiary + ? `About ${currentBeneficiary.name}` + : 'Online'} + {voiceCallState === 'active' && ( + + + + )} - + + {/* Voice call indicator bar */} + {voiceCallState === 'active' && ( + + + + Voice call active + + + End + + + )} + {/* Beneficiary Picker Modal */} - Loading beneficiaries... ) : beneficiaries.length === 0 ? ( @@ -701,9 +582,6 @@ Based on this data, please answer the following question: ${question}`; {item.name} - {item.email && ( - {item.email} - )} {currentBeneficiary?.id === item.id && ( @@ -717,7 +595,6 @@ Based on this data, please answer the following question: ${question}`; - {/* Messages */} flatListRef.current?.scrollToEnd({ animated: true })} /> - {/* Voice Feedback Text (for errors) */} - {voiceFeedback && !isListening && !isSpeaking && ( - - {voiceFeedback} - - )} - - {/* Beautiful Voice Indicator Animation */} - {(isListening || isSpeaking) && ( - - )} - {/* Input */} - {/* Microphone / Stop Button */} - - - - - + {/* Voice Call Button */} + {renderVoiceCallButton()} @@ -833,6 +664,7 @@ const styles = StyleSheet.create({ marginRight: Spacing.sm, }, headerInfo: { + flex: 1, flexDirection: 'row', alignItems: 'center', }, @@ -859,9 +691,48 @@ const styles = StyleSheet.create({ fontSize: FontSizes.sm, color: AppColors.success, }, + headerButtons: { + flexDirection: 'row', + gap: Spacing.xs, + }, headerButton: { padding: Spacing.xs, }, + voiceCallBar: { + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'space-between', + backgroundColor: AppColors.success, + paddingHorizontal: Spacing.md, + paddingVertical: Spacing.sm, + }, + voiceCallBarLeft: { + flexDirection: 'row', + alignItems: 'center', + }, + voiceCallBarDot: { + width: 8, + height: 8, + borderRadius: 4, + backgroundColor: AppColors.white, + marginRight: Spacing.sm, + }, + voiceCallBarText: { + fontSize: FontSizes.sm, + fontWeight: '500', + color: AppColors.white, + }, + voiceCallBarEnd: { + paddingHorizontal: Spacing.md, + paddingVertical: Spacing.xs, + backgroundColor: 'rgba(255,255,255,0.2)', + borderRadius: BorderRadius.md, + }, + voiceCallBarEndText: { + fontSize: FontSizes.sm, + fontWeight: '600', + color: AppColors.white, + }, chatContainer: { flex: 1, }, @@ -880,6 +751,15 @@ const styles = StyleSheet.create({ assistantMessageContainer: { justifyContent: 'flex-start', }, + systemMessageContainer: { + alignItems: 'center', + marginVertical: Spacing.sm, + }, + systemMessageText: { + fontSize: FontSizes.sm, + color: AppColors.textMuted, + fontStyle: 'italic', + }, avatarContainer: { width: 32, height: 32, @@ -907,6 +787,16 @@ const styles = StyleSheet.create({ backgroundColor: AppColors.background, borderBottomLeftRadius: BorderRadius.sm, }, + voiceBadge: { + flexDirection: 'row', + alignItems: 'center', + marginBottom: Spacing.xs, + }, + voiceBadgeText: { + fontSize: FontSizes.xs, + color: AppColors.primary, + marginLeft: 4, + }, messageText: { fontSize: FontSizes.base, lineHeight: 22, @@ -934,6 +824,34 @@ const styles = StyleSheet.create({ borderTopWidth: 1, borderTopColor: AppColors.border, }, + voiceCallButton: { + width: 44, + height: 44, + borderRadius: 22, + justifyContent: 'center', + alignItems: 'center', + marginRight: Spacing.sm, + }, + voiceCallButtonInner: { + width: '100%', + height: '100%', + justifyContent: 'center', + alignItems: 'center', + }, + voiceCallIdle: { + backgroundColor: AppColors.surface, + borderWidth: 1, + borderColor: AppColors.primary, + }, + voiceCallConnecting: { + backgroundColor: AppColors.warning || '#FF9800', + }, + voiceCallActive: { + backgroundColor: AppColors.success, + }, + voiceCallEnding: { + backgroundColor: AppColors.textMuted, + }, input: { flex: 1, backgroundColor: AppColors.surface, @@ -986,11 +904,6 @@ const styles = StyleSheet.create({ padding: Spacing.xl, alignItems: 'center', }, - loadingText: { - marginTop: Spacing.md, - fontSize: FontSizes.base, - color: AppColors.textSecondary, - }, modalEmpty: { padding: Spacing.xl, alignItems: 'center', @@ -1037,59 +950,4 @@ const styles = StyleSheet.create({ fontWeight: '500', color: AppColors.textPrimary, }, - beneficiaryEmail: { - fontSize: FontSizes.sm, - color: AppColors.textSecondary, - marginTop: 2, - }, - // Voice UI styles - voiceFeedbackContainer: { - paddingHorizontal: Spacing.md, - paddingVertical: Spacing.sm, - backgroundColor: 'rgba(255, 152, 0, 0.1)', - borderRadius: BorderRadius.md, - marginHorizontal: Spacing.md, - marginBottom: Spacing.sm, - }, - voiceFeedbackText: { - fontSize: FontSizes.sm, - color: AppColors.warning || '#FF9800', - textAlign: 'center', - }, - micButton: { - width: 44, - height: 44, - borderRadius: 22, - backgroundColor: AppColors.surface, - justifyContent: 'center', - alignItems: 'center', - marginRight: Spacing.sm, - borderWidth: 1, - borderColor: AppColors.border, - }, - micButtonActive: { - backgroundColor: AppColors.primary, - borderColor: AppColors.primary, - }, - micButtonSpeaking: { - backgroundColor: AppColors.error || '#E53935', - borderColor: AppColors.error || '#E53935', - }, - micButtonDisabled: { - opacity: 0.5, - }, - // Header buttons (for beneficiary picker) - headerButtons: { - flexDirection: 'row', - gap: Spacing.xs, - }, }); - -// Wrap with TTSErrorBoundary to catch TTS crashes -export default function ChatScreen() { - return ( - - - - ); -} diff --git a/app/(tabs)/voice.tsx b/app/(tabs)/voice.tsx new file mode 100644 index 0000000..88f091d --- /dev/null +++ b/app/(tabs)/voice.tsx @@ -0,0 +1,515 @@ +/** + * Voice Screen - Ultravox Voice AI Integration + * Real-time voice conversation with Julia AI using WebRTC + * Ferdinand context is automatically loaded + */ + +import React, { useState, useCallback, useRef, useEffect } from 'react'; +import { + View, + Text, + StyleSheet, + TouchableOpacity, + ActivityIndicator, + Animated, + Easing, +} from 'react-native'; +import { SafeAreaView } from 'react-native-safe-area-context'; +import { Ionicons, Feather } from '@expo/vector-icons'; +import { useRouter } from 'expo-router'; +import { useFocusEffect } from '@react-navigation/native'; +import { + useUltravox, + UltravoxSessionStatus, + type Transcript, +} from 'ultravox-react-native'; +import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme'; +import { + createCall, + getSystemPrompt, + VOICE_NAME, +} from '@/services/ultravoxService'; + +type CallState = 'idle' | 'connecting' | 'active' | 'ending' | 'error'; + +export default function VoiceScreen() { + const router = useRouter(); + + // Call state + const [callState, setCallState] = useState('idle'); + const [error, setError] = useState(null); + const [isMuted, setIsMuted] = useState(false); + + // Animation for the voice button + const pulseAnim = useRef(new Animated.Value(1)).current; + const rotateAnim = useRef(new Animated.Value(0)).current; + + // Tool implementations for navigation (client-side) + const toolImplementations = { + navigateToDashboard: () => { + console.log('[Voice] Tool: navigateToDashboard'); + router.push('/(tabs)/dashboard'); + return 'Navigating to Dashboard'; + }, + navigateToBeneficiaries: () => { + console.log('[Voice] Tool: navigateToBeneficiaries'); + router.push('/(tabs)/beneficiaries'); + return 'Navigating to Beneficiaries'; + }, + navigateToProfile: () => { + console.log('[Voice] Tool: navigateToProfile'); + router.push('/(tabs)/profile'); + return 'Navigating to Profile'; + }, + }; + + // Ultravox hook - proper way to use the SDK + const { transcripts, joinCall, leaveCall, session } = useUltravox({ + tools: toolImplementations, + onStatusChange: (event) => { + console.log('[Voice] Status changed:', event.status); + + switch (event.status) { + case UltravoxSessionStatus.IDLE: + case UltravoxSessionStatus.DISCONNECTED: + setCallState('idle'); + break; + case UltravoxSessionStatus.CONNECTING: + setCallState('connecting'); + break; + case UltravoxSessionStatus.LISTENING: + case UltravoxSessionStatus.THINKING: + case UltravoxSessionStatus.SPEAKING: + setCallState('active'); + break; + case UltravoxSessionStatus.DISCONNECTING: + setCallState('ending'); + break; + } + }, + }); + + // Pulse animation when active + useEffect(() => { + if (callState === 'active') { + const pulse = Animated.loop( + Animated.sequence([ + Animated.timing(pulseAnim, { + toValue: 1.15, + duration: 1000, + easing: Easing.inOut(Easing.ease), + useNativeDriver: true, + }), + Animated.timing(pulseAnim, { + toValue: 1, + duration: 1000, + easing: Easing.inOut(Easing.ease), + useNativeDriver: true, + }), + ]) + ); + pulse.start(); + return () => pulse.stop(); + } else { + pulseAnim.setValue(1); + } + }, [callState, pulseAnim]); + + // Rotate animation when connecting + useEffect(() => { + if (callState === 'connecting') { + const rotate = Animated.loop( + Animated.timing(rotateAnim, { + toValue: 1, + duration: 1500, + easing: Easing.linear, + useNativeDriver: true, + }) + ); + rotate.start(); + return () => rotate.stop(); + } else { + rotateAnim.setValue(0); + } + }, [callState, rotateAnim]); + + // Start voice call + const startCall = useCallback(async () => { + setError(null); + setCallState('connecting'); + + // Get system prompt with Ferdinand context + const systemPrompt = getSystemPrompt(); + + try { + // Create call via API + const result = await createCall({ + systemPrompt, + firstSpeaker: 'FIRST_SPEAKER_AGENT', + }); + + if (!result.success) { + throw new Error(result.error); + } + + console.log('[Voice] Call created, joinUrl:', result.data.joinUrl); + + // Join the call using the hook's joinCall + await joinCall(result.data.joinUrl); + console.log('[Voice] Joined call'); + + } catch (err) { + console.error('[Voice] Failed to start call:', err); + setError(err instanceof Error ? err.message : 'Failed to start call'); + setCallState('error'); + } + }, [joinCall]); + + // End voice call + const endCall = useCallback(async () => { + setCallState('ending'); + try { + await leaveCall(); + } catch (err) { + console.error('[Voice] Error leaving call:', err); + } + setCallState('idle'); + }, [leaveCall]); + + // Toggle mute + const toggleMute = useCallback(() => { + if (session) { + const newMuted = !isMuted; + if (newMuted) { + session.muteMic(); + } else { + session.unmuteMic(); + } + setIsMuted(newMuted); + } + }, [session, isMuted]); + + // End call when leaving the screen (switching tabs) + useFocusEffect( + useCallback(() => { + // Screen focused - do nothing special + return () => { + // Screen unfocused - end the call if active + if (callState === 'active' || callState === 'connecting') { + console.log('[Voice] Screen unfocused, ending call'); + leaveCall().catch(console.error); + setCallState('idle'); + } + }; + }, [callState, leaveCall]) + ); + + // Get last transcript for display + const lastTranscript = transcripts[transcripts.length - 1]; + + // Render voice button based on state + const renderVoiceButton = () => { + const spin = rotateAnim.interpolate({ + inputRange: [0, 1], + outputRange: ['0deg', '360deg'], + }); + + switch (callState) { + case 'connecting': + return ( + + + + ); + + case 'active': + return ( + + + + + + ); + + case 'ending': + return ( + + + + ); + + case 'error': + return ( + + + + ); + + default: // idle + return ( + + + + ); + } + }; + + return ( + + {/* Header */} + + router.push('/(tabs)/dashboard')} + > + + + + Julia AI + + {callState === 'active' ? 'In call' : callState === 'connecting' ? 'Connecting...' : `Voice: ${VOICE_NAME}`} + + + + {callState === 'active' && ( + + + + )} + + + + {/* Main content */} + + {/* Avatar and status */} + + + + J + + {callState === 'active' && ( + + )} + + Julia + Ferdinand Zmrzli's Wellness Assistant + + + {/* Transcript display */} + {lastTranscript && callState === 'active' && ( + + + {lastTranscript.speaker === 'agent' ? 'Julia' : 'You'}: + + + {lastTranscript.text} + + + )} + + {/* Error display */} + {error && ( + + + {error} + + )} + + {/* Voice button */} + + {renderVoiceButton()} + + {callState === 'idle' && 'Tap to start voice call'} + {callState === 'connecting' && 'Connecting...'} + {callState === 'active' && 'Tap to end call'} + {callState === 'ending' && 'Ending call...'} + {callState === 'error' && 'Tap to retry'} + + + + {/* Info text */} + {callState === 'idle' && ( + + + Ask Julia about Ferdinand's wellness status, alerts, or say "show me the dashboard" to navigate. + + + )} + + + ); +} + +const styles = StyleSheet.create({ + container: { + flex: 1, + backgroundColor: AppColors.background, + }, + header: { + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'space-between', + paddingHorizontal: Spacing.md, + paddingVertical: Spacing.sm, + borderBottomWidth: 1, + borderBottomColor: AppColors.border, + }, + backButton: { + padding: Spacing.xs, + }, + headerCenter: { + alignItems: 'center', + }, + headerTitle: { + fontSize: FontSizes.lg, + fontWeight: '600', + color: AppColors.textPrimary, + }, + headerSubtitle: { + fontSize: FontSizes.sm, + color: AppColors.success, + marginTop: 2, + }, + headerRight: { + width: 44, + alignItems: 'flex-end', + }, + muteButton: { + padding: Spacing.xs, + }, + content: { + flex: 1, + alignItems: 'center', + justifyContent: 'space-between', + paddingVertical: Spacing.xl, + }, + avatarSection: { + alignItems: 'center', + paddingTop: Spacing.xl, + }, + avatarContainer: { + position: 'relative', + }, + avatar: { + width: 120, + height: 120, + borderRadius: 60, + backgroundColor: AppColors.success, + justifyContent: 'center', + alignItems: 'center', + }, + avatarText: { + fontSize: 48, + fontWeight: '600', + color: AppColors.white, + }, + statusDot: { + position: 'absolute', + bottom: 8, + right: 8, + width: 24, + height: 24, + borderRadius: 12, + backgroundColor: AppColors.success, + borderWidth: 3, + borderColor: AppColors.background, + }, + assistantName: { + fontSize: FontSizes.xxl, + fontWeight: '700', + color: AppColors.textPrimary, + marginTop: Spacing.md, + }, + assistantRole: { + fontSize: FontSizes.base, + color: AppColors.textSecondary, + marginTop: Spacing.xs, + }, + transcriptContainer: { + backgroundColor: AppColors.surface, + borderRadius: BorderRadius.lg, + padding: Spacing.md, + marginHorizontal: Spacing.lg, + maxWidth: '90%', + }, + transcriptLabel: { + fontSize: FontSizes.sm, + fontWeight: '600', + color: AppColors.primary, + marginBottom: Spacing.xs, + }, + transcriptText: { + fontSize: FontSizes.base, + color: AppColors.textPrimary, + lineHeight: 22, + }, + errorContainer: { + flexDirection: 'row', + alignItems: 'center', + backgroundColor: 'rgba(229, 57, 53, 0.1)', + borderRadius: BorderRadius.md, + padding: Spacing.md, + marginHorizontal: Spacing.lg, + }, + errorText: { + fontSize: FontSizes.sm, + color: AppColors.error, + marginLeft: Spacing.sm, + flex: 1, + }, + buttonSection: { + alignItems: 'center', + }, + voiceButton: { + width: 120, + height: 120, + borderRadius: 60, + justifyContent: 'center', + alignItems: 'center', + shadowColor: '#000', + shadowOffset: { width: 0, height: 4 }, + shadowOpacity: 0.3, + shadowRadius: 8, + elevation: 8, + }, + voiceButtonInner: { + width: '100%', + height: '100%', + justifyContent: 'center', + alignItems: 'center', + }, + voiceButtonIdle: { + backgroundColor: AppColors.primary, + }, + voiceButtonConnecting: { + backgroundColor: AppColors.warning || '#FF9800', + }, + voiceButtonActive: { + backgroundColor: AppColors.success, + }, + voiceButtonEnding: { + backgroundColor: AppColors.textMuted, + }, + voiceButtonError: { + backgroundColor: AppColors.error, + }, + buttonHint: { + fontSize: FontSizes.sm, + color: AppColors.textSecondary, + marginTop: Spacing.md, + }, + infoContainer: { + paddingHorizontal: Spacing.xl, + paddingBottom: Spacing.lg, + }, + infoText: { + fontSize: FontSizes.sm, + color: AppColors.textMuted, + textAlign: 'center', + lineHeight: 20, + }, +}); diff --git a/assets/data/ferdinand_7days_events.json b/assets/data/ferdinand_7days_events.json new file mode 100644 index 0000000..2356f6d --- /dev/null +++ b/assets/data/ferdinand_7days_events.json @@ -0,0 +1,983 @@ +{ + "client": { + "id": "fz-001", + "name": "Ferdinand Zmrzli", + "address": "661 Encore Way" + }, + "period": "last_7_days", + + "days": [ + { + "date": "6_days_ago", + "day": "Thursday", + "alerts": [ + { + "type": "high_bathroom_frequency", + "time": "15:00", + "count": 7, + "severity": "medium", + "note": "Visited bathroom 7 times before 3pm (normal: 4-5)" + }, + { + "type": "late_to_bed", + "time": "23:30", + "severity": "low", + "note": "Still awake at 23:30 (usual bedtime: 20:30-21:00)" + } + ], + "events": [ + {"time": "00:15", "event": "position_change", "location": "bedroom"}, + {"time": "00:45", "event": "position_change", "location": "bedroom"}, + {"time": "01:30", "event": "position_change", "location": "bedroom"}, + {"time": "02:15", "event": "position_change", "location": "bedroom"}, + {"time": "03:10", "event": "woke_up", "location": "bedroom"}, + {"time": "03:12", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "03:14", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "03:15", "event": "urination", "location": "bathroom"}, + {"time": "03:17", "event": "toilet_flush", "location": "bathroom"}, + {"time": "03:18", "event": "hand_wash", "location": "bathroom"}, + {"time": "03:20", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "03:22", "event": "fell_asleep", "location": "bedroom"}, + {"time": "04:00", "event": "position_change", "location": "bedroom"}, + {"time": "05:15", "event": "position_change", "location": "bedroom"}, + {"time": "06:30", "event": "woke_up", "location": "bedroom"}, + {"time": "06:32", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "06:35", "event": "diaper_check", "location": "bedroom"}, + {"time": "06:38", "event": "diaper_change", "location": "bedroom"}, + {"time": "06:42", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "06:44", "event": "face_wash", "location": "bathroom"}, + {"time": "06:47", "event": "teeth_brushing", "location": "bathroom"}, + {"time": "06:50", "event": "urination", "location": "bathroom"}, + {"time": "06:52", "event": "toilet_flush", "location": "bathroom"}, + {"time": "06:53", "event": "hand_wash", "location": "bathroom"}, + {"time": "06:55", "event": "left_bathroom", "location": "bathroom"}, + {"time": "06:57", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "06:58", "event": "fridge_opened", "location": "kitchen"}, + {"time": "07:00", "event": "fridge_closed", "location": "kitchen"}, + {"time": "07:02", "event": "coffee_machine_on", "location": "kitchen"}, + {"time": "07:05", "event": "coffee_ready", "location": "kitchen"}, + {"time": "07:06", "event": "toaster_on", "location": "kitchen"}, + {"time": "07:09", "event": "toast_ready", "location": "kitchen"}, + {"time": "07:10", "event": "fridge_opened", "location": "kitchen"}, + {"time": "07:11", "event": "fridge_closed", "location": "kitchen"}, + {"time": "07:12", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "07:13", "event": "medication_taken", "location": "kitchen"}, + {"time": "07:15", "event": "eating", "location": "kitchen"}, + {"time": "07:35", "event": "finished_eating", "location": "kitchen"}, + {"time": "07:47", "event": "water_running", "location": "kitchen"}, + {"time": "07:52", "event": "left_kitchen", "location": "kitchen"}, + {"time": "07:54", "event": "entered_living_room", "location": "living_room"}, + {"time": "07:56", "event": "sat_down", "location": "living_room"}, + {"time": "09:30", "event": "stood_up", "location": "living_room"}, + {"time": "09:31", "event": "window_opened", "location": "living_room"}, + {"time": "09:45", "event": "window_closed", "location": "living_room"}, + {"time": "09:47", "event": "stood_up", "location": "living_room"}, + {"time": "09:48", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "09:50", "event": "urination", "location": "bathroom"}, + {"time": "09:52", "event": "toilet_flush", "location": "bathroom"}, + {"time": "09:53", "event": "hand_wash", "location": "bathroom"}, + {"time": "09:55", "event": "entered_living_room", "location": "living_room"}, + {"time": "09:56", "event": "sat_down", "location": "living_room"}, + {"time": "10:30", "event": "stood_up", "location": "living_room"}, + {"time": "10:32", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "10:33", "event": "fridge_opened", "location": "kitchen"}, + {"time": "10:34", "event": "fridge_closed", "location": "kitchen"}, + {"time": "10:35", "event": "eating", "location": "kitchen"}, + {"time": "10:40", "event": "finished_eating", "location": "kitchen"}, + {"time": "10:42", "event": "left_kitchen", "location": "kitchen"}, + {"time": "10:44", "event": "entered_living_room", "location": "living_room"}, + {"time": "10:45", "event": "sat_down", "location": "living_room"}, + {"time": "12:00", "event": "stood_up", "location": "living_room"}, + {"time": "12:02", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "12:03", "event": "urination", "location": "bathroom"}, + {"time": "12:05", "event": "toilet_flush", "location": "bathroom"}, + {"time": "12:06", "event": "hand_wash", "location": "bathroom"}, + {"time": "12:08", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "12:10", "event": "fridge_opened", "location": "kitchen"}, + {"time": "12:11", "event": "fridge_closed", "location": "kitchen"}, + {"time": "12:12", "event": "microwave_on", "location": "kitchen"}, + {"time": "12:15", "event": "microwave_off", "location": "kitchen"}, + {"time": "12:17", "event": "eating", "location": "kitchen"}, + {"time": "12:40", "event": "finished_eating", "location": "kitchen"}, + {"time": "12:45", "event": "water_running", "location": "kitchen"}, + {"time": "12:50", "event": "left_kitchen", "location": "kitchen"}, + {"time": "12:52", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "12:55", "event": "lay_down", "location": "bedroom"}, + {"time": "13:00", "event": "fell_asleep", "location": "bedroom"}, + {"time": "13:30", "event": "position_change", "location": "bedroom"}, + {"time": "14:15", "event": "position_change", "location": "bedroom"}, + {"time": "14:30", "event": "woke_up", "location": "bedroom"}, + {"time": "14:32", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "14:34", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "14:35", "event": "urination", "location": "bathroom"}, + {"time": "14:37", "event": "toilet_flush", "location": "bathroom"}, + {"time": "14:38", "event": "hand_wash", "location": "bathroom"}, + {"time": "14:40", "event": "entered_living_room", "location": "living_room"}, + {"time": "14:42", "event": "sat_down", "location": "living_room"}, + {"time": "15:30", "event": "stood_up", "location": "living_room"}, + {"time": "15:32", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "15:33", "event": "urination", "location": "bathroom"}, + {"time": "15:35", "event": "toilet_flush", "location": "bathroom"}, + {"time": "15:36", "event": "hand_wash", "location": "bathroom"}, + {"time": "15:38", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "15:40", "event": "kettle_on", "location": "kitchen"}, + {"time": "15:44", "event": "kettle_off", "location": "kitchen"}, + {"time": "15:48", "event": "eating", "location": "kitchen"}, + {"time": "16:15", "event": "finished_eating", "location": "kitchen"}, + {"time": "16:17", "event": "entered_living_room", "location": "living_room"}, + {"time": "16:20", "event": "sat_down", "location": "living_room"}, + {"time": "17:30", "event": "stood_up", "location": "living_room"}, + {"time": "17:33", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "17:35", "event": "fridge_opened", "location": "kitchen"}, + {"time": "17:36", "event": "fridge_closed", "location": "kitchen"}, + {"time": "17:45", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "17:46", "event": "medication_taken", "location": "kitchen"}, + {"time": "17:48", "event": "eating", "location": "kitchen"}, + {"time": "18:10", "event": "finished_eating", "location": "kitchen"}, + {"time": "18:15", "event": "water_running", "location": "kitchen"}, + {"time": "18:20", "event": "entered_living_room", "location": "living_room"}, + {"time": "18:23", "event": "sat_down", "location": "living_room"}, + {"time": "20:00", "event": "stood_up", "location": "living_room"}, + {"time": "20:03", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "20:05", "event": "urination", "location": "bathroom"}, + {"time": "20:07", "event": "toilet_flush", "location": "bathroom"}, + {"time": "20:08", "event": "teeth_brushing", "location": "bathroom"}, + {"time": "20:12", "event": "face_wash", "location": "bathroom"}, + {"time": "20:15", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "20:18", "event": "diaper_put_on", "location": "bedroom"}, + {"time": "20:22", "event": "lay_down", "location": "bedroom"}, + {"time": "20:50", "event": "fell_asleep", "location": "bedroom"}, + {"time": "21:30", "event": "position_change", "location": "bedroom"}, + {"time": "22:15", "event": "position_change", "location": "bedroom"}, + {"time": "23:00", "event": "position_change", "location": "bedroom"}, + {"time": "23:45", "event": "position_change", "location": "bedroom"} + ] + }, + + { + "date": "5_days_ago", + "day": "Friday", + "alerts": [ + { + "type": "stove_left_on", + "time": "12:45", + "duration_minutes": 47, + "severity": "high", + "resolved": true, + "resolved_time": "13:32" + }, + { + "type": "low_activity", + "time": "14:00", + "duration_minutes": 120, + "severity": "medium", + "note": "No movement for 2 hours in living room" + } + ], + "events": [ + {"time": "00:20", "event": "position_change", "location": "bedroom"}, + {"time": "01:00", "event": "position_change", "location": "bedroom"}, + {"time": "01:45", "event": "position_change", "location": "bedroom"}, + {"time": "02:30", "event": "position_change", "location": "bedroom"}, + {"time": "03:15", "event": "woke_up", "location": "bedroom"}, + {"time": "03:17", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "03:19", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "03:20", "event": "urination", "location": "bathroom"}, + {"time": "03:22", "event": "toilet_flush", "location": "bathroom"}, + {"time": "03:23", "event": "hand_wash", "location": "bathroom"}, + {"time": "03:25", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "03:28", "event": "fell_asleep", "location": "bedroom"}, + {"time": "04:15", "event": "position_change", "location": "bedroom"}, + {"time": "05:00", "event": "position_change", "location": "bedroom"}, + {"time": "05:45", "event": "position_change", "location": "bedroom"}, + {"time": "06:45", "event": "woke_up", "location": "bedroom"}, + {"time": "06:48", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "06:50", "event": "diaper_check", "location": "bedroom"}, + {"time": "06:53", "event": "diaper_change", "location": "bedroom"}, + {"time": "06:58", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "07:00", "event": "face_wash", "location": "bathroom"}, + {"time": "07:03", "event": "teeth_brushing", "location": "bathroom"}, + {"time": "07:06", "event": "urination", "location": "bathroom"}, + {"time": "07:08", "event": "toilet_flush", "location": "bathroom"}, + {"time": "07:09", "event": "hand_wash", "location": "bathroom"}, + {"time": "07:12", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "07:14", "event": "fridge_opened", "location": "kitchen"}, + {"time": "07:15", "event": "fridge_closed", "location": "kitchen"}, + {"time": "07:17", "event": "coffee_machine_on", "location": "kitchen"}, + {"time": "07:20", "event": "coffee_ready", "location": "kitchen"}, + {"time": "07:21", "event": "toaster_on", "location": "kitchen"}, + {"time": "07:24", "event": "toast_ready", "location": "kitchen"}, + {"time": "07:25", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "07:26", "event": "medication_taken", "location": "kitchen"}, + {"time": "07:28", "event": "eating", "location": "kitchen"}, + {"time": "07:50", "event": "finished_eating", "location": "kitchen"}, + {"time": "08:00", "event": "water_running", "location": "kitchen"}, + {"time": "08:05", "event": "entered_living_room", "location": "living_room"}, + {"time": "08:08", "event": "sat_down", "location": "living_room"}, + {"time": "09:45", "event": "stood_up", "location": "living_room"}, + {"time": "09:47", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "09:48", "event": "urination", "location": "bathroom"}, + {"time": "09:50", "event": "toilet_flush", "location": "bathroom"}, + {"time": "09:51", "event": "hand_wash", "location": "bathroom"}, + {"time": "09:53", "event": "entered_living_room", "location": "living_room"}, + {"time": "09:55", "event": "sat_down", "location": "living_room"}, + {"time": "10:30", "event": "stood_up", "location": "living_room"}, + {"time": "10:32", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "10:33", "event": "fridge_opened", "location": "kitchen"}, + {"time": "10:34", "event": "fridge_closed", "location": "kitchen"}, + {"time": "10:35", "event": "eating", "location": "kitchen"}, + {"time": "10:42", "event": "finished_eating", "location": "kitchen"}, + {"time": "10:45", "event": "entered_living_room", "location": "living_room"}, + {"time": "10:47", "event": "sat_down", "location": "living_room"}, + {"time": "12:00", "event": "stood_up", "location": "living_room"}, + {"time": "12:02", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "12:03", "event": "urination", "location": "bathroom"}, + {"time": "12:05", "event": "toilet_flush", "location": "bathroom"}, + {"time": "12:06", "event": "hand_wash", "location": "bathroom"}, + {"time": "12:08", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "12:10", "event": "fridge_opened", "location": "kitchen"}, + {"time": "12:12", "event": "fridge_closed", "location": "kitchen"}, + {"time": "12:15", "event": "stove_on", "location": "kitchen"}, + {"time": "12:18", "event": "cooking", "location": "kitchen"}, + {"time": "12:35", "event": "eating", "location": "kitchen"}, + {"time": "12:45", "event": "left_kitchen", "location": "kitchen"}, + {"time": "12:45", "event": "ALERT_stove_still_on", "location": "kitchen"}, + {"time": "12:47", "event": "entered_living_room", "location": "living_room"}, + {"time": "12:48", "event": "sat_down", "location": "living_room"}, + {"time": "12:50", "event": "fell_asleep", "location": "living_room"}, + {"time": "13:00", "event": "ALERT_stove_on_15min", "location": "kitchen"}, + {"time": "13:15", "event": "ALERT_stove_on_30min", "location": "kitchen"}, + {"time": "13:30", "event": "woke_up", "location": "living_room"}, + {"time": "13:32", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "13:32", "event": "stove_off", "location": "kitchen"}, + {"time": "13:32", "event": "ALERT_resolved", "location": "kitchen"}, + {"time": "13:35", "event": "window_opened", "location": "kitchen"}, + {"time": "13:50", "event": "window_closed", "location": "kitchen"}, + {"time": "13:55", "event": "entered_living_room", "location": "living_room"}, + {"time": "13:57", "event": "sat_down", "location": "living_room"}, + {"time": "15:30", "event": "stood_up", "location": "living_room"}, + {"time": "15:32", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "15:33", "event": "urination", "location": "bathroom"}, + {"time": "15:35", "event": "toilet_flush", "location": "bathroom"}, + {"time": "15:36", "event": "hand_wash", "location": "bathroom"}, + {"time": "15:38", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "15:40", "event": "kettle_on", "location": "kitchen"}, + {"time": "15:44", "event": "kettle_off", "location": "kitchen"}, + {"time": "15:50", "event": "eating", "location": "kitchen"}, + {"time": "16:05", "event": "finished_eating", "location": "kitchen"}, + {"time": "16:20", "event": "entered_living_room", "location": "living_room"}, + {"time": "16:22", "event": "sat_down", "location": "living_room"}, + {"time": "17:45", "event": "stood_up", "location": "living_room"}, + {"time": "17:49", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "17:50", "event": "fridge_opened", "location": "kitchen"}, + {"time": "17:52", "event": "fridge_closed", "location": "kitchen"}, + {"time": "17:55", "event": "microwave_on", "location": "kitchen"}, + {"time": "17:58", "event": "microwave_off", "location": "kitchen"}, + {"time": "18:00", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "18:01", "event": "medication_taken", "location": "kitchen"}, + {"time": "18:03", "event": "eating", "location": "kitchen"}, + {"time": "18:25", "event": "finished_eating", "location": "kitchen"}, + {"time": "18:28", "event": "water_running", "location": "kitchen"}, + {"time": "18:35", "event": "entered_living_room", "location": "living_room"}, + {"time": "18:38", "event": "sat_down", "location": "living_room"}, + {"time": "20:15", "event": "stood_up", "location": "living_room"}, + {"time": "20:18", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "20:20", "event": "urination", "location": "bathroom"}, + {"time": "20:22", "event": "toilet_flush", "location": "bathroom"}, + {"time": "20:23", "event": "teeth_brushing", "location": "bathroom"}, + {"time": "20:27", "event": "face_wash", "location": "bathroom"}, + {"time": "20:30", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "20:33", "event": "diaper_put_on", "location": "bedroom"}, + {"time": "20:37", "event": "lay_down", "location": "bedroom"}, + {"time": "21:00", "event": "fell_asleep", "location": "bedroom"}, + {"time": "21:45", "event": "position_change", "location": "bedroom"}, + {"time": "22:30", "event": "position_change", "location": "bedroom"}, + {"time": "23:15", "event": "position_change", "location": "bedroom"} + ] + }, + + { + "date": "4_days_ago", + "day": "Saturday", + "alerts": [ + { + "type": "unusual_wake_time", + "time": "03:00", + "severity": "low", + "note": "Woke up at 3:00 AM (normal wake: 6:00-7:00)" + }, + { + "type": "fridge_not_opened", + "time": "11:00", + "duration_hours": 4, + "severity": "medium", + "note": "Fridge not opened for 4 hours after breakfast" + } + ], + "events": [ + {"time": "00:00", "event": "position_change", "location": "bedroom"}, + {"time": "00:45", "event": "position_change", "location": "bedroom"}, + {"time": "01:30", "event": "position_change", "location": "bedroom"}, + {"time": "02:15", "event": "position_change", "location": "bedroom"}, + {"time": "03:00", "event": "woke_up", "location": "bedroom"}, + {"time": "03:02", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "03:04", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "03:05", "event": "urination", "location": "bathroom"}, + {"time": "03:07", "event": "toilet_flush", "location": "bathroom"}, + {"time": "03:08", "event": "hand_wash", "location": "bathroom"}, + {"time": "03:10", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "03:13", "event": "fell_asleep", "location": "bedroom"}, + {"time": "04:00", "event": "position_change", "location": "bedroom"}, + {"time": "04:50", "event": "position_change", "location": "bedroom"}, + {"time": "05:40", "event": "position_change", "location": "bedroom"}, + {"time": "06:30", "event": "woke_up", "location": "bedroom"}, + {"time": "06:33", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "06:35", "event": "diaper_check", "location": "bedroom"}, + {"time": "06:38", "event": "diaper_change", "location": "bedroom"}, + {"time": "06:42", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "06:44", "event": "shower_on", "location": "bathroom"}, + {"time": "06:58", "event": "shower_off", "location": "bathroom"}, + {"time": "07:02", "event": "teeth_brushing", "location": "bathroom"}, + {"time": "07:15", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "07:28", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "07:30", "event": "fridge_opened", "location": "kitchen"}, + {"time": "07:32", "event": "fridge_closed", "location": "kitchen"}, + {"time": "07:34", "event": "stove_on", "location": "kitchen"}, + {"time": "07:36", "event": "cooking", "location": "kitchen"}, + {"time": "07:42", "event": "stove_off", "location": "kitchen"}, + {"time": "07:43", "event": "toaster_on", "location": "kitchen"}, + {"time": "07:46", "event": "toast_ready", "location": "kitchen"}, + {"time": "07:47", "event": "coffee_machine_on", "location": "kitchen"}, + {"time": "07:50", "event": "coffee_ready", "location": "kitchen"}, + {"time": "07:52", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "07:53", "event": "medication_taken", "location": "kitchen"}, + {"time": "07:55", "event": "eating", "location": "kitchen"}, + {"time": "08:20", "event": "finished_eating", "location": "kitchen"}, + {"time": "08:30", "event": "water_running", "location": "kitchen"}, + {"time": "08:38", "event": "entered_living_room", "location": "living_room"}, + {"time": "08:55", "event": "window_opened", "location": "living_room"}, + {"time": "09:10", "event": "window_closed", "location": "living_room"}, + {"time": "09:15", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "09:16", "event": "urination", "location": "bathroom"}, + {"time": "09:18", "event": "toilet_flush", "location": "bathroom"}, + {"time": "09:19", "event": "hand_wash", "location": "bathroom"}, + {"time": "09:22", "event": "entered_living_room", "location": "living_room"}, + {"time": "09:25", "event": "sat_down", "location": "living_room"}, + {"time": "10:25", "event": "stood_up", "location": "living_room"}, + {"time": "10:28", "event": "front_door_opened", "location": "entrance"}, + {"time": "10:30", "event": "front_door_closed", "location": "entrance"}, + {"time": "10:35", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "10:40", "event": "fridge_opened", "location": "kitchen"}, + {"time": "10:42", "event": "fridge_closed", "location": "kitchen"}, + {"time": "10:50", "event": "stove_on", "location": "kitchen"}, + {"time": "10:55", "event": "cooking", "location": "kitchen"}, + {"time": "11:30", "event": "stove_off", "location": "kitchen"}, + {"time": "11:35", "event": "eating", "location": "kitchen"}, + {"time": "12:05", "event": "finished_eating", "location": "kitchen"}, + {"time": "12:10", "event": "water_running", "location": "kitchen"}, + {"time": "12:20", "event": "entered_living_room", "location": "living_room"}, + {"time": "12:25", "event": "sat_down", "location": "living_room"}, + {"time": "14:00", "event": "stood_up", "location": "living_room"}, + {"time": "14:02", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "14:03", "event": "urination", "location": "bathroom"}, + {"time": "14:05", "event": "toilet_flush", "location": "bathroom"}, + {"time": "14:06", "event": "hand_wash", "location": "bathroom"}, + {"time": "14:10", "event": "entered_entrance", "location": "entrance"}, + {"time": "14:15", "event": "front_door_opened", "location": "entrance"}, + {"time": "14:16", "event": "left_home", "location": "entrance"}, + {"time": "14:32", "event": "front_door_opened", "location": "entrance"}, + {"time": "14:33", "event": "returned_home", "location": "entrance"}, + {"time": "14:34", "event": "front_door_closed", "location": "entrance"}, + {"time": "14:38", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "14:40", "event": "kettle_on", "location": "kitchen"}, + {"time": "14:44", "event": "kettle_off", "location": "kitchen"}, + {"time": "14:48", "event": "eating", "location": "kitchen"}, + {"time": "15:05", "event": "finished_eating", "location": "kitchen"}, + {"time": "15:10", "event": "entered_living_room", "location": "living_room"}, + {"time": "15:12", "event": "sat_down", "location": "living_room"}, + {"time": "16:30", "event": "stood_up", "location": "living_room"}, + {"time": "16:32", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "16:33", "event": "urination", "location": "bathroom"}, + {"time": "16:35", "event": "toilet_flush", "location": "bathroom"}, + {"time": "16:36", "event": "hand_wash", "location": "bathroom"}, + {"time": "16:40", "event": "entered_entrance", "location": "entrance"}, + {"time": "16:42", "event": "front_door_opened", "location": "entrance"}, + {"time": "16:44", "event": "front_door_closed", "location": "entrance"}, + {"time": "16:48", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "16:50", "event": "fridge_opened", "location": "kitchen"}, + {"time": "16:52", "event": "fridge_closed", "location": "kitchen"}, + {"time": "16:55", "event": "microwave_on", "location": "kitchen"}, + {"time": "16:58", "event": "microwave_off", "location": "kitchen"}, + {"time": "17:00", "event": "eating", "location": "kitchen"}, + {"time": "17:25", "event": "finished_eating", "location": "kitchen"}, + {"time": "17:27", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "17:28", "event": "medication_taken", "location": "kitchen"}, + {"time": "17:30", "event": "water_running", "location": "kitchen"}, + {"time": "17:38", "event": "entered_living_room", "location": "living_room"}, + {"time": "17:42", "event": "sat_down", "location": "living_room"}, + {"time": "19:30", "event": "stood_up", "location": "living_room"}, + {"time": "19:32", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "19:33", "event": "urination", "location": "bathroom"}, + {"time": "19:35", "event": "toilet_flush", "location": "bathroom"}, + {"time": "19:36", "event": "hand_wash", "location": "bathroom"}, + {"time": "19:40", "event": "entered_living_room", "location": "living_room"}, + {"time": "21:00", "event": "stood_up", "location": "living_room"}, + {"time": "21:03", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "21:05", "event": "urination", "location": "bathroom"}, + {"time": "21:07", "event": "toilet_flush", "location": "bathroom"}, + {"time": "21:08", "event": "teeth_brushing", "location": "bathroom"}, + {"time": "21:12", "event": "face_wash", "location": "bathroom"}, + {"time": "21:15", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "21:18", "event": "diaper_put_on", "location": "bedroom"}, + {"time": "21:22", "event": "lay_down", "location": "bedroom"}, + {"time": "21:45", "event": "fell_asleep", "location": "bedroom"}, + {"time": "22:30", "event": "position_change", "location": "bedroom"}, + {"time": "23:15", "event": "position_change", "location": "bedroom"} + ] + }, + + { + "date": "3_days_ago", + "day": "Sunday", + "alerts": [ + { + "type": "no_shower", + "time": "08:00", + "note": "No shower detected for 24+ hours", + "severity": "low" + }, + { + "type": "prolonged_nap", + "time": "16:00", + "duration_minutes": 150, + "severity": "medium", + "note": "Afternoon nap lasted 2.5 hours (normal: 1-1.5 hours)" + } + ], + "events": [ + {"time": "00:00", "event": "position_change", "location": "bedroom"}, + {"time": "00:45", "event": "position_change", "location": "bedroom"}, + {"time": "01:30", "event": "position_change", "location": "bedroom"}, + {"time": "02:15", "event": "position_change", "location": "bedroom"}, + {"time": "03:00", "event": "woke_up", "location": "bedroom"}, + {"time": "03:02", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "03:04", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "03:05", "event": "urination", "location": "bathroom"}, + {"time": "03:07", "event": "toilet_flush", "location": "bathroom"}, + {"time": "03:08", "event": "hand_wash", "location": "bathroom"}, + {"time": "03:10", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "03:13", "event": "fell_asleep", "location": "bedroom"}, + {"time": "04:00", "event": "position_change", "location": "bedroom"}, + {"time": "04:50", "event": "position_change", "location": "bedroom"}, + {"time": "05:40", "event": "position_change", "location": "bedroom"}, + {"time": "07:00", "event": "woke_up", "location": "bedroom"}, + {"time": "07:03", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "07:05", "event": "diaper_check", "location": "bedroom"}, + {"time": "07:08", "event": "diaper_change", "location": "bedroom"}, + {"time": "07:12", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "07:14", "event": "face_wash", "location": "bathroom"}, + {"time": "07:17", "event": "teeth_brushing", "location": "bathroom"}, + {"time": "07:20", "event": "urination", "location": "bathroom"}, + {"time": "07:22", "event": "toilet_flush", "location": "bathroom"}, + {"time": "07:23", "event": "hand_wash", "location": "bathroom"}, + {"time": "07:26", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "07:28", "event": "coffee_machine_on", "location": "kitchen"}, + {"time": "07:31", "event": "coffee_ready", "location": "kitchen"}, + {"time": "07:33", "event": "fridge_opened", "location": "kitchen"}, + {"time": "07:35", "event": "fridge_closed", "location": "kitchen"}, + {"time": "07:38", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "07:39", "event": "medication_taken", "location": "kitchen"}, + {"time": "07:41", "event": "eating", "location": "kitchen"}, + {"time": "08:00", "event": "ALERT_no_shower_24h", "location": "bathroom"}, + {"time": "08:05", "event": "finished_eating", "location": "kitchen"}, + {"time": "08:15", "event": "water_running", "location": "kitchen"}, + {"time": "08:22", "event": "entered_living_room", "location": "living_room"}, + {"time": "08:25", "event": "sat_down", "location": "living_room"}, + {"time": "10:00", "event": "stood_up", "location": "living_room"}, + {"time": "10:02", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "10:03", "event": "urination", "location": "bathroom"}, + {"time": "10:05", "event": "toilet_flush", "location": "bathroom"}, + {"time": "10:06", "event": "hand_wash", "location": "bathroom"}, + {"time": "10:08", "event": "entered_living_room", "location": "living_room"}, + {"time": "10:10", "event": "sat_down", "location": "living_room"}, + {"time": "11:30", "event": "stood_up", "location": "living_room"}, + {"time": "11:32", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "11:34", "event": "kettle_on", "location": "kitchen"}, + {"time": "11:38", "event": "kettle_off", "location": "kitchen"}, + {"time": "11:45", "event": "eating", "location": "kitchen"}, + {"time": "11:55", "event": "entered_living_room", "location": "living_room"}, + {"time": "11:57", "event": "sat_down", "location": "living_room"}, + {"time": "12:30", "event": "stood_up", "location": "living_room"}, + {"time": "12:32", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "12:33", "event": "urination", "location": "bathroom"}, + {"time": "12:35", "event": "toilet_flush", "location": "bathroom"}, + {"time": "12:36", "event": "hand_wash", "location": "bathroom"}, + {"time": "12:38", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "12:40", "event": "fridge_opened", "location": "kitchen"}, + {"time": "12:42", "event": "fridge_closed", "location": "kitchen"}, + {"time": "12:44", "event": "microwave_on", "location": "kitchen"}, + {"time": "12:47", "event": "microwave_off", "location": "kitchen"}, + {"time": "12:49", "event": "eating", "location": "kitchen"}, + {"time": "13:15", "event": "finished_eating", "location": "kitchen"}, + {"time": "13:18", "event": "water_running", "location": "kitchen"}, + {"time": "13:25", "event": "entered_living_room", "location": "living_room"}, + {"time": "13:27", "event": "sat_down", "location": "living_room"}, + {"time": "14:30", "event": "fell_asleep", "location": "living_room"}, + {"time": "15:15", "event": "position_change", "location": "living_room"}, + {"time": "16:00", "event": "woke_up", "location": "living_room"}, + {"time": "16:02", "event": "stood_up", "location": "living_room"}, + {"time": "16:04", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "16:05", "event": "urination", "location": "bathroom"}, + {"time": "16:07", "event": "toilet_flush", "location": "bathroom"}, + {"time": "16:08", "event": "hand_wash", "location": "bathroom"}, + {"time": "16:10", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "16:12", "event": "fridge_opened", "location": "kitchen"}, + {"time": "16:13", "event": "fridge_closed", "location": "kitchen"}, + {"time": "16:15", "event": "eating", "location": "kitchen"}, + {"time": "16:30", "event": "entered_living_room", "location": "living_room"}, + {"time": "16:33", "event": "sat_down", "location": "living_room"}, + {"time": "18:00", "event": "stood_up", "location": "living_room"}, + {"time": "18:02", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "18:04", "event": "fridge_opened", "location": "kitchen"}, + {"time": "18:06", "event": "fridge_closed", "location": "kitchen"}, + {"time": "18:08", "event": "stove_on", "location": "kitchen"}, + {"time": "18:12", "event": "cooking", "location": "kitchen"}, + {"time": "18:25", "event": "stove_off", "location": "kitchen"}, + {"time": "18:27", "event": "eating", "location": "kitchen"}, + {"time": "18:50", "event": "finished_eating", "location": "kitchen"}, + {"time": "18:52", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "18:53", "event": "medication_taken", "location": "kitchen"}, + {"time": "18:55", "event": "water_running", "location": "kitchen"}, + {"time": "19:02", "event": "entered_living_room", "location": "living_room"}, + {"time": "19:04", "event": "sat_down", "location": "living_room"}, + {"time": "20:30", "event": "stood_up", "location": "living_room"}, + {"time": "20:33", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "20:35", "event": "urination", "location": "bathroom"}, + {"time": "20:37", "event": "toilet_flush", "location": "bathroom"}, + {"time": "20:38", "event": "teeth_brushing", "location": "bathroom"}, + {"time": "20:42", "event": "face_wash", "location": "bathroom"}, + {"time": "20:45", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "20:48", "event": "diaper_put_on", "location": "bedroom"}, + {"time": "20:52", "event": "lay_down", "location": "bedroom"}, + {"time": "21:15", "event": "fell_asleep", "location": "bedroom"}, + {"time": "22:00", "event": "position_change", "location": "bedroom"}, + {"time": "22:45", "event": "position_change", "location": "bedroom"}, + {"time": "23:30", "event": "position_change", "location": "bedroom"} + ] + }, + + { + "date": "2_days_ago", + "day": "Monday", + "alerts": [ + { + "type": "diaper_not_changed", + "time": "10:30", + "duration_hours": 4, + "severity": "medium", + "note": "Diaper not changed for 4+ hours after waking" + }, + { + "type": "no_breakfast", + "time": "09:00", + "severity": "medium", + "note": "No cooking or eating activity detected until 11:30" + } + ], + "events": [ + {"time": "00:15", "event": "position_change", "location": "bedroom"}, + {"time": "01:00", "event": "position_change", "location": "bedroom"}, + {"time": "01:45", "event": "position_change", "location": "bedroom"}, + {"time": "02:30", "event": "woke_up", "location": "bedroom"}, + {"time": "02:32", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "02:34", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "02:35", "event": "urination", "location": "bathroom"}, + {"time": "02:37", "event": "toilet_flush", "location": "bathroom"}, + {"time": "02:38", "event": "hand_wash", "location": "bathroom"}, + {"time": "02:40", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "02:43", "event": "fell_asleep", "location": "bedroom"}, + {"time": "03:30", "event": "position_change", "location": "bedroom"}, + {"time": "04:20", "event": "position_change", "location": "bedroom"}, + {"time": "05:10", "event": "position_change", "location": "bedroom"}, + {"time": "06:00", "event": "position_change", "location": "bedroom"}, + {"time": "06:30", "event": "woke_up", "location": "bedroom"}, + {"time": "06:33", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "06:35", "event": "diaper_check", "location": "bedroom"}, + {"time": "06:36", "event": "diaper_not_changed", "location": "bedroom"}, + {"time": "06:40", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "06:42", "event": "face_wash", "location": "bathroom"}, + {"time": "06:45", "event": "teeth_brushing", "location": "bathroom"}, + {"time": "06:48", "event": "urination", "location": "bathroom"}, + {"time": "06:50", "event": "toilet_flush", "location": "bathroom"}, + {"time": "06:51", "event": "hand_wash", "location": "bathroom"}, + {"time": "06:54", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "06:56", "event": "coffee_machine_on", "location": "kitchen"}, + {"time": "06:59", "event": "coffee_ready", "location": "kitchen"}, + {"time": "07:00", "event": "toaster_on", "location": "kitchen"}, + {"time": "07:03", "event": "toast_ready", "location": "kitchen"}, + {"time": "07:05", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "07:06", "event": "medication_taken", "location": "kitchen"}, + {"time": "07:08", "event": "eating", "location": "kitchen"}, + {"time": "07:30", "event": "finished_eating", "location": "kitchen"}, + {"time": "07:40", "event": "water_running", "location": "kitchen"}, + {"time": "07:48", "event": "entered_living_room", "location": "living_room"}, + {"time": "07:51", "event": "sat_down", "location": "living_room"}, + {"time": "08:30", "event": "ALERT_diaper_2h", "location": "bedroom"}, + {"time": "09:30", "event": "stood_up", "location": "living_room"}, + {"time": "09:32", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "09:33", "event": "urination", "location": "bathroom"}, + {"time": "09:35", "event": "toilet_flush", "location": "bathroom"}, + {"time": "09:36", "event": "hand_wash", "location": "bathroom"}, + {"time": "09:38", "event": "entered_living_room", "location": "living_room"}, + {"time": "09:40", "event": "sat_down", "location": "living_room"}, + {"time": "10:00", "event": "front_door_opened", "location": "entrance"}, + {"time": "10:02", "event": "front_door_closed", "location": "entrance"}, + {"time": "10:05", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "10:08", "event": "diaper_check", "location": "bedroom"}, + {"time": "10:10", "event": "diaper_change", "location": "bedroom"}, + {"time": "10:10", "event": "ALERT_resolved", "location": "bedroom"}, + {"time": "10:15", "event": "entered_living_room", "location": "living_room"}, + {"time": "10:28", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "11:00", "event": "entered_living_room", "location": "living_room"}, + {"time": "11:25", "event": "front_door_opened", "location": "entrance"}, + {"time": "11:27", "event": "front_door_closed", "location": "entrance"}, + {"time": "11:30", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "11:32", "event": "fridge_opened", "location": "kitchen"}, + {"time": "11:34", "event": "fridge_closed", "location": "kitchen"}, + {"time": "11:36", "event": "microwave_on", "location": "kitchen"}, + {"time": "11:39", "event": "microwave_off", "location": "kitchen"}, + {"time": "11:41", "event": "eating", "location": "kitchen"}, + {"time": "12:05", "event": "finished_eating", "location": "kitchen"}, + {"time": "12:08", "event": "water_running", "location": "kitchen"}, + {"time": "12:15", "event": "entered_living_room", "location": "living_room"}, + {"time": "12:17", "event": "sat_down", "location": "living_room"}, + {"time": "13:30", "event": "stood_up", "location": "living_room"}, + {"time": "13:32", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "13:33", "event": "urination", "location": "bathroom"}, + {"time": "13:35", "event": "toilet_flush", "location": "bathroom"}, + {"time": "13:36", "event": "hand_wash", "location": "bathroom"}, + {"time": "13:38", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "13:40", "event": "lay_down", "location": "bedroom"}, + {"time": "13:50", "event": "fell_asleep", "location": "bedroom"}, + {"time": "14:30", "event": "position_change", "location": "bedroom"}, + {"time": "15:15", "event": "woke_up", "location": "bedroom"}, + {"time": "15:18", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "15:20", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "15:21", "event": "urination", "location": "bathroom"}, + {"time": "15:23", "event": "toilet_flush", "location": "bathroom"}, + {"time": "15:24", "event": "hand_wash", "location": "bathroom"}, + {"time": "15:27", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "15:29", "event": "fridge_opened", "location": "kitchen"}, + {"time": "15:30", "event": "fridge_closed", "location": "kitchen"}, + {"time": "15:32", "event": "eating", "location": "kitchen"}, + {"time": "15:42", "event": "entered_living_room", "location": "living_room"}, + {"time": "15:45", "event": "sat_down", "location": "living_room"}, + {"time": "17:30", "event": "stood_up", "location": "living_room"}, + {"time": "17:34", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "17:36", "event": "fridge_opened", "location": "kitchen"}, + {"time": "17:38", "event": "fridge_closed", "location": "kitchen"}, + {"time": "17:48", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "17:49", "event": "medication_taken", "location": "kitchen"}, + {"time": "17:51", "event": "eating", "location": "kitchen"}, + {"time": "18:15", "event": "finished_eating", "location": "kitchen"}, + {"time": "18:18", "event": "water_running", "location": "kitchen"}, + {"time": "18:25", "event": "entered_living_room", "location": "living_room"}, + {"time": "18:28", "event": "sat_down", "location": "living_room"}, + {"time": "20:15", "event": "stood_up", "location": "living_room"}, + {"time": "20:18", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "20:20", "event": "urination", "location": "bathroom"}, + {"time": "20:22", "event": "toilet_flush", "location": "bathroom"}, + {"time": "20:23", "event": "teeth_brushing", "location": "bathroom"}, + {"time": "20:27", "event": "face_wash", "location": "bathroom"}, + {"time": "20:30", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "20:33", "event": "diaper_put_on", "location": "bedroom"}, + {"time": "20:37", "event": "lay_down", "location": "bedroom"}, + {"time": "21:00", "event": "fell_asleep", "location": "bedroom"}, + {"time": "21:45", "event": "position_change", "location": "bedroom"}, + {"time": "22:30", "event": "position_change", "location": "bedroom"}, + {"time": "23:15", "event": "position_change", "location": "bedroom"} + ] + }, + + { + "date": "yesterday", + "day": "Tuesday", + "alerts": [ + { + "type": "missed_medication", + "time": "08:00", + "severity": "high", + "note": "Morning medication not taken until 09:45" + }, + { + "type": "low_water_intake", + "time": "18:00", + "severity": "low", + "note": "Only 2 water/drink events detected today (normal: 5-6)" + } + ], + "events": [ + {"time": "00:00", "event": "position_change", "location": "bedroom"}, + {"time": "00:45", "event": "position_change", "location": "bedroom"}, + {"time": "01:30", "event": "position_change", "location": "bedroom"}, + {"time": "02:15", "event": "position_change", "location": "bedroom"}, + {"time": "03:00", "event": "woke_up", "location": "bedroom"}, + {"time": "03:02", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "03:04", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "03:05", "event": "urination", "location": "bathroom"}, + {"time": "03:07", "event": "toilet_flush", "location": "bathroom"}, + {"time": "03:08", "event": "hand_wash", "location": "bathroom"}, + {"time": "03:10", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "03:13", "event": "fell_asleep", "location": "bedroom"}, + {"time": "04:00", "event": "position_change", "location": "bedroom"}, + {"time": "04:50", "event": "position_change", "location": "bedroom"}, + {"time": "05:40", "event": "position_change", "location": "bedroom"}, + {"time": "07:00", "event": "woke_up", "location": "bedroom"}, + {"time": "07:03", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "07:05", "event": "diaper_check", "location": "bedroom"}, + {"time": "07:08", "event": "diaper_change", "location": "bedroom"}, + {"time": "07:12", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "07:14", "event": "face_wash", "location": "bathroom"}, + {"time": "07:17", "event": "teeth_brushing", "location": "bathroom"}, + {"time": "07:20", "event": "urination", "location": "bathroom"}, + {"time": "07:22", "event": "toilet_flush", "location": "bathroom"}, + {"time": "07:23", "event": "hand_wash", "location": "bathroom"}, + {"time": "07:26", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "07:28", "event": "coffee_machine_on", "location": "kitchen"}, + {"time": "07:31", "event": "coffee_ready", "location": "kitchen"}, + {"time": "07:33", "event": "toaster_on", "location": "kitchen"}, + {"time": "07:36", "event": "toast_ready", "location": "kitchen"}, + {"time": "07:38", "event": "eating", "location": "kitchen"}, + {"time": "08:00", "event": "ALERT_medication_not_taken", "location": "kitchen"}, + {"time": "08:05", "event": "finished_eating", "location": "kitchen"}, + {"time": "08:15", "event": "water_running", "location": "kitchen"}, + {"time": "08:22", "event": "entered_living_room", "location": "living_room"}, + {"time": "08:25", "event": "sat_down", "location": "living_room"}, + {"time": "08:30", "event": "ALERT_medication_30min", "location": "kitchen"}, + {"time": "09:00", "event": "ALERT_medication_1h", "location": "kitchen"}, + {"time": "09:30", "event": "stood_up", "location": "living_room"}, + {"time": "09:32", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "09:33", "event": "urination", "location": "bathroom"}, + {"time": "09:35", "event": "toilet_flush", "location": "bathroom"}, + {"time": "09:36", "event": "hand_wash", "location": "bathroom"}, + {"time": "09:38", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "09:43", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "09:44", "event": "medication_taken", "location": "kitchen"}, + {"time": "09:44", "event": "ALERT_resolved", "location": "kitchen"}, + {"time": "09:48", "event": "entered_living_room", "location": "living_room"}, + {"time": "09:50", "event": "sat_down", "location": "living_room"}, + {"time": "12:00", "event": "stood_up", "location": "living_room"}, + {"time": "12:02", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "12:03", "event": "urination", "location": "bathroom"}, + {"time": "12:05", "event": "toilet_flush", "location": "bathroom"}, + {"time": "12:06", "event": "hand_wash", "location": "bathroom"}, + {"time": "12:08", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "12:10", "event": "fridge_opened", "location": "kitchen"}, + {"time": "12:12", "event": "fridge_closed", "location": "kitchen"}, + {"time": "12:14", "event": "stove_on", "location": "kitchen"}, + {"time": "12:18", "event": "cooking", "location": "kitchen"}, + {"time": "12:28", "event": "stove_off", "location": "kitchen"}, + {"time": "12:30", "event": "eating", "location": "kitchen"}, + {"time": "12:55", "event": "finished_eating", "location": "kitchen"}, + {"time": "12:58", "event": "water_running", "location": "kitchen"}, + {"time": "13:05", "event": "entered_living_room", "location": "living_room"}, + {"time": "13:07", "event": "sat_down", "location": "living_room"}, + {"time": "13:45", "event": "fell_asleep", "location": "living_room"}, + {"time": "14:30", "event": "position_change", "location": "living_room"}, + {"time": "15:15", "event": "woke_up", "location": "living_room"}, + {"time": "15:17", "event": "stood_up", "location": "living_room"}, + {"time": "15:19", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "15:20", "event": "urination", "location": "bathroom"}, + {"time": "15:22", "event": "toilet_flush", "location": "bathroom"}, + {"time": "15:23", "event": "hand_wash", "location": "bathroom"}, + {"time": "15:26", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "15:28", "event": "kettle_on", "location": "kitchen"}, + {"time": "15:32", "event": "kettle_off", "location": "kitchen"}, + {"time": "15:37", "event": "eating", "location": "kitchen"}, + {"time": "15:58", "event": "entered_living_room", "location": "living_room"}, + {"time": "16:02", "event": "sat_down", "location": "living_room"}, + {"time": "17:45", "event": "stood_up", "location": "living_room"}, + {"time": "17:49", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "17:51", "event": "fridge_opened", "location": "kitchen"}, + {"time": "17:53", "event": "fridge_closed", "location": "kitchen"}, + {"time": "17:55", "event": "stove_on", "location": "kitchen"}, + {"time": "17:58", "event": "cooking", "location": "kitchen"}, + {"time": "18:10", "event": "stove_off", "location": "kitchen"}, + {"time": "18:12", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "18:13", "event": "medication_taken", "location": "kitchen"}, + {"time": "18:15", "event": "eating", "location": "kitchen"}, + {"time": "18:40", "event": "finished_eating", "location": "kitchen"}, + {"time": "18:43", "event": "water_running", "location": "kitchen"}, + {"time": "18:50", "event": "entered_living_room", "location": "living_room"}, + {"time": "18:53", "event": "sat_down", "location": "living_room"}, + {"time": "20:30", "event": "stood_up", "location": "living_room"}, + {"time": "20:33", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "20:35", "event": "urination", "location": "bathroom"}, + {"time": "20:37", "event": "toilet_flush", "location": "bathroom"}, + {"time": "20:38", "event": "teeth_brushing", "location": "bathroom"}, + {"time": "20:42", "event": "face_wash", "location": "bathroom"}, + {"time": "20:45", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "20:48", "event": "diaper_put_on", "location": "bedroom"}, + {"time": "20:52", "event": "lay_down", "location": "bedroom"}, + {"time": "21:15", "event": "fell_asleep", "location": "bedroom"}, + {"time": "22:00", "event": "position_change", "location": "bedroom"}, + {"time": "22:45", "event": "position_change", "location": "bedroom"}, + {"time": "23:30", "event": "position_change", "location": "bedroom"} + ] + }, + + { + "date": "today", + "day": "Wednesday", + "alerts": [ + { + "type": "fall_detected", + "time": "06:32", + "severity": "critical", + "location": "hallway" + }, + { + "type": "no_shower", + "time": "12:00", + "severity": "low", + "note": "No shower for 48+ hours (after fall incident)" + } + ], + "events": [ + {"time": "00:15", "event": "position_change", "location": "bedroom"}, + {"time": "01:00", "event": "position_change", "location": "bedroom"}, + {"time": "01:45", "event": "position_change", "location": "bedroom"}, + {"time": "02:30", "event": "position_change", "location": "bedroom"}, + {"time": "03:15", "event": "position_change", "location": "bedroom"}, + {"time": "04:00", "event": "position_change", "location": "bedroom"}, + {"time": "04:45", "event": "position_change", "location": "bedroom"}, + {"time": "05:30", "event": "position_change", "location": "bedroom"}, + {"time": "06:30", "event": "woke_up", "location": "bedroom"}, + {"time": "06:31", "event": "got_out_of_bed", "location": "bedroom"}, + {"time": "06:32", "event": "FALL_DETECTED", "location": "hallway"}, + {"time": "06:33", "event": "no_movement", "location": "hallway"}, + {"time": "06:34", "event": "no_movement", "location": "hallway"}, + {"time": "06:35", "event": "no_movement", "location": "hallway"}, + {"time": "06:40", "event": "movement_detected", "location": "hallway"}, + {"time": "06:55", "event": "front_door_opened", "location": "entrance"}, + {"time": "06:57", "event": "front_door_closed", "location": "entrance"}, + {"time": "07:00", "event": "movement_detected", "location": "living_room"}, + {"time": "07:40", "event": "sat_down", "location": "living_room"}, + {"time": "08:00", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "08:02", "event": "coffee_machine_on", "location": "kitchen"}, + {"time": "08:05", "event": "coffee_ready", "location": "kitchen"}, + {"time": "08:07", "event": "toaster_on", "location": "kitchen"}, + {"time": "08:10", "event": "toast_ready", "location": "kitchen"}, + {"time": "08:12", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "08:13", "event": "medication_taken", "location": "kitchen"}, + {"time": "08:15", "event": "eating", "location": "kitchen"}, + {"time": "08:35", "event": "finished_eating", "location": "kitchen"}, + {"time": "08:40", "event": "entered_living_room", "location": "living_room"}, + {"time": "08:42", "event": "sat_down", "location": "living_room"}, + {"time": "09:25", "event": "front_door_opened", "location": "entrance"}, + {"time": "09:27", "event": "front_door_closed", "location": "entrance"}, + {"time": "09:30", "event": "sat_down", "location": "living_room"}, + {"time": "09:50", "event": "stood_up", "location": "living_room"}, + {"time": "09:52", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "09:53", "event": "urination", "location": "bathroom"}, + {"time": "09:55", "event": "toilet_flush", "location": "bathroom"}, + {"time": "09:56", "event": "hand_wash", "location": "bathroom"}, + {"time": "10:00", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "10:03", "event": "diaper_check", "location": "bedroom"}, + {"time": "10:06", "event": "diaper_change", "location": "bedroom"}, + {"time": "10:10", "event": "lay_down", "location": "bedroom"}, + {"time": "10:30", "event": "fell_asleep", "location": "bedroom"}, + {"time": "11:15", "event": "position_change", "location": "bedroom"}, + {"time": "12:00", "event": "woke_up", "location": "bedroom"}, + {"time": "12:08", "event": "eating", "location": "bedroom"}, + {"time": "12:30", "event": "finished_eating", "location": "bedroom"}, + {"time": "12:50", "event": "stood_up", "location": "bedroom"}, + {"time": "12:52", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "12:53", "event": "urination", "location": "bathroom"}, + {"time": "12:55", "event": "toilet_flush", "location": "bathroom"}, + {"time": "12:56", "event": "hand_wash", "location": "bathroom"}, + {"time": "13:00", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "13:03", "event": "lay_down", "location": "bedroom"}, + {"time": "13:05", "event": "medication_taken", "location": "bedroom"}, + {"time": "13:15", "event": "fell_asleep", "location": "bedroom"}, + {"time": "14:00", "event": "position_change", "location": "bedroom"}, + {"time": "14:45", "event": "position_change", "location": "bedroom"}, + {"time": "15:30", "event": "woke_up", "location": "bedroom"}, + {"time": "15:35", "event": "stood_up", "location": "bedroom"}, + {"time": "15:37", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "15:38", "event": "urination", "location": "bathroom"}, + {"time": "15:40", "event": "toilet_flush", "location": "bathroom"}, + {"time": "15:41", "event": "hand_wash", "location": "bathroom"}, + {"time": "15:45", "event": "entered_living_room", "location": "living_room"}, + {"time": "15:47", "event": "sat_down", "location": "living_room"}, + {"time": "16:00", "event": "kettle_on", "location": "kitchen"}, + {"time": "16:04", "event": "kettle_off", "location": "kitchen"}, + {"time": "16:15", "event": "eating", "location": "living_room"}, + {"time": "17:00", "event": "stood_up", "location": "living_room"}, + {"time": "17:02", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "17:03", "event": "urination", "location": "bathroom"}, + {"time": "17:05", "event": "toilet_flush", "location": "bathroom"}, + {"time": "17:06", "event": "hand_wash", "location": "bathroom"}, + {"time": "17:10", "event": "entered_living_room", "location": "living_room"}, + {"time": "17:12", "event": "sat_down", "location": "living_room"}, + {"time": "17:35", "event": "stove_on", "location": "kitchen"}, + {"time": "17:55", "event": "stove_off", "location": "kitchen"}, + {"time": "18:00", "event": "entered_kitchen", "location": "kitchen"}, + {"time": "18:02", "event": "sat_down", "location": "kitchen"}, + {"time": "18:05", "event": "eating", "location": "kitchen"}, + {"time": "18:30", "event": "finished_eating", "location": "kitchen"}, + {"time": "18:32", "event": "medication_box_opened", "location": "kitchen"}, + {"time": "18:33", "event": "medication_taken", "location": "kitchen"}, + {"time": "18:40", "event": "entered_living_room", "location": "living_room"}, + {"time": "18:42", "event": "sat_down", "location": "living_room"}, + {"time": "19:30", "event": "stood_up", "location": "living_room"}, + {"time": "19:32", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "19:33", "event": "urination", "location": "bathroom"}, + {"time": "19:35", "event": "toilet_flush", "location": "bathroom"}, + {"time": "19:36", "event": "hand_wash", "location": "bathroom"}, + {"time": "19:40", "event": "entered_living_room", "location": "living_room"}, + {"time": "20:30", "event": "stood_up", "location": "living_room"}, + {"time": "20:33", "event": "entered_bathroom", "location": "bathroom"}, + {"time": "20:35", "event": "teeth_brushing", "location": "bathroom"}, + {"time": "20:39", "event": "face_wash", "location": "bathroom"}, + {"time": "20:42", "event": "urination", "location": "bathroom"}, + {"time": "20:44", "event": "toilet_flush", "location": "bathroom"}, + {"time": "20:45", "event": "hand_wash", "location": "bathroom"}, + {"time": "20:48", "event": "entered_bedroom", "location": "bedroom"}, + {"time": "20:51", "event": "diaper_put_on", "location": "bedroom"}, + {"time": "20:55", "event": "lay_down", "location": "bedroom"}, + {"time": "21:15", "event": "fell_asleep", "location": "bedroom"}, + {"time": "22:00", "event": "position_change", "location": "bedroom"}, + {"time": "22:45", "event": "position_change", "location": "bedroom"}, + {"time": "23:30", "event": "position_change", "location": "bedroom"} + ] + } + ], + + "summary": { + "total_days": 7, + "total_alerts": 14, + "alerts_by_severity": { + "critical": 1, + "high": 2, + "medium": 5, + "low": 6 + }, + "alerts_summary": [ + {"date": "6_days_ago", "type": "high_bathroom_frequency", "severity": "medium"}, + {"date": "6_days_ago", "type": "late_to_bed", "severity": "low"}, + {"date": "5_days_ago", "type": "stove_left_on", "severity": "high"}, + {"date": "5_days_ago", "type": "low_activity", "severity": "medium"}, + {"date": "4_days_ago", "type": "unusual_wake_time", "severity": "low"}, + {"date": "4_days_ago", "type": "fridge_not_opened", "severity": "medium"}, + {"date": "3_days_ago", "type": "no_shower", "severity": "low"}, + {"date": "3_days_ago", "type": "prolonged_nap", "severity": "medium"}, + {"date": "2_days_ago", "type": "diaper_not_changed", "severity": "medium"}, + {"date": "2_days_ago", "type": "no_breakfast", "severity": "medium"}, + {"date": "yesterday", "type": "missed_medication", "severity": "high"}, + {"date": "yesterday", "type": "low_water_intake", "severity": "low"}, + {"date": "today", "type": "fall_detected", "severity": "critical"}, + {"date": "today", "type": "no_shower", "severity": "low"} + ] + } +} diff --git a/services/ultravoxService.ts b/services/ultravoxService.ts new file mode 100644 index 0000000..5971f52 --- /dev/null +++ b/services/ultravoxService.ts @@ -0,0 +1,272 @@ +/** + * Ultravox Voice AI Service + * Creates calls via Ultravox API and manages voice configuration + */ + +// Import Ferdinand data +import ferdinandData from '@/assets/data/ferdinand_7days_events.json'; + +// API Configuration +const ULTRAVOX_API_URL = 'https://api.ultravox.ai/api'; +const ULTRAVOX_API_KEY = '4miSVLym.HF3lV9y4euiuzcEbPPTLHEugrOu4jpNU'; + +// Fixed voice - Sarah only +export const VOICE_ID = 'Sarah'; +export const VOICE_NAME = 'Sarah'; + +// Tool definitions for function calling +export interface UltravoxTool { + temporaryTool: { + modelToolName: string; + description: string; + dynamicParameters?: Array<{ + name: string; + location: string; + schema: { + type: string; + description: string; + }; + required: boolean; + }>; + client?: Record; + }; +} + +export const ULTRAVOX_TOOLS: UltravoxTool[] = [ + { + temporaryTool: { + modelToolName: 'navigateToDashboard', + description: 'Navigate to the Dashboard screen to show wellness overview, charts, and real-time status. Use when user asks to see the dashboard, overview, charts, or wants to check the current status visually.', + client: {}, + }, + }, + { + temporaryTool: { + modelToolName: 'navigateToBeneficiaries', + description: 'Navigate to the beneficiaries list screen when user wants to see or manage their loved ones', + client: {}, + }, + }, + { + temporaryTool: { + modelToolName: 'navigateToProfile', + description: 'Navigate to the user profile settings screen', + client: {}, + }, + }, +]; + +// Build context from Ferdinand data +function buildFerdinandContext(): string { + const client = ferdinandData.client; + const summary = ferdinandData.summary; + + // Get today's alerts + const todayData = ferdinandData.days.find(d => d.date === 'today'); + const yesterdayData = ferdinandData.days.find(d => d.date === 'yesterday'); + + let context = ` +BENEFICIARY INFORMATION: +- Name: ${client.name} +- Address: ${client.address} +- Monitoring Period: Last 7 days + +CURRENT STATUS (Today - ${todayData?.day || 'Wednesday'}): +`; + + // Add today's alerts with severity + if (todayData?.alerts && todayData.alerts.length > 0) { + context += `⚠️ ALERTS TODAY:\n`; + todayData.alerts.forEach(alert => { + const emoji = alert.severity === 'critical' ? '🔴' : alert.severity === 'high' ? '🟠' : alert.severity === 'medium' ? '🟡' : '🟢'; + context += ` ${emoji} ${alert.type.replace(/_/g, ' ').toUpperCase()} at ${alert.time}`; + if (alert.note) context += ` - ${alert.note}`; + if (alert.location) context += ` (${alert.location})`; + context += '\n'; + }); + } + + // Add yesterday's alerts + if (yesterdayData?.alerts && yesterdayData.alerts.length > 0) { + context += `\nYESTERDAY'S ALERTS:\n`; + yesterdayData.alerts.forEach(alert => { + const emoji = alert.severity === 'critical' ? '🔴' : alert.severity === 'high' ? '🟠' : alert.severity === 'medium' ? '🟡' : '🟢'; + context += ` ${emoji} ${alert.type.replace(/_/g, ' ')} at ${alert.time}`; + if (alert.note) context += ` - ${alert.note}`; + context += '\n'; + }); + } + + // 7-day summary + context += ` +7-DAY SUMMARY: +- Total alerts: ${summary.total_alerts} +- Critical: ${summary.alerts_by_severity.critical} +- High: ${summary.alerts_by_severity.high} +- Medium: ${summary.alerts_by_severity.medium} +- Low: ${summary.alerts_by_severity.low} + +KEY CONCERNS THIS WEEK: +`; + + // Add all alerts summary + summary.alerts_summary.forEach(alert => { + const emoji = alert.severity === 'critical' ? '🔴' : alert.severity === 'high' ? '🟠' : alert.severity === 'medium' ? '🟡' : '🟢'; + context += ` ${emoji} ${alert.date}: ${alert.type.replace(/_/g, ' ')}\n`; + }); + + // Add typical daily pattern from today's events + if (todayData?.events) { + const wakeUp = todayData.events.find(e => e.event === 'woke_up'); + const sleep = todayData.events.find(e => e.event === 'fell_asleep'); + const meals = todayData.events.filter(e => e.event === 'eating' || e.event === 'finished_eating'); + const medications = todayData.events.filter(e => e.event === 'medication_taken'); + const bathroom = todayData.events.filter(e => e.event === 'urination'); + + context += ` +TODAY'S ACTIVITY PATTERN: +- Wake up: ${wakeUp?.time || 'N/A'} +- Meals: ${meals.length / 2} meals detected +- Medications: ${medications.length} doses taken +- Bathroom visits: ${bathroom.length} times +- Current location: ${todayData.events[todayData.events.length - 1]?.location || 'bedroom'} +`; + } + + return context; +} + +// System prompt generator with Ferdinand context +export function getSystemPrompt(): string { + const context = buildFerdinandContext(); + + return `You are Julia, a compassionate and knowledgeable AI wellness assistant for WellNuo app. +Your role is to help caregivers monitor and understand the wellbeing of their loved ones. + +${context} + +IMPORTANT GUIDELINES: +- Be warm, empathetic, and supportive in your responses +- You have FULL access to ${ferdinandData.client.name}'s wellness data from the last 7 days +- When asked about status, alerts, or concerns - refer to the actual data above +- Prioritize critical and high severity alerts when discussing concerns +- The FALL DETECTED today at 06:32 is the most urgent concern - acknowledge it if user asks about current status +- You can navigate the app using available tools when the user requests it +- If user asks to "show dashboard", "open dashboard", "see the overview" - use the navigateToDashboard tool +- Keep responses conversational and natural for voice interaction +- Speak in a calm, reassuring tone +- Be specific with times and details from the data +- If asked about something not in the data, say you don't have that information + +Remember: You're speaking with a caregiver who wants the best for their loved one. +Be supportive and helpful while maintaining appropriate boundaries about medical advice.`; +} + +// API Response types +export interface CreateCallResponse { + callId: string; + joinUrl: string; + created: string; + ended?: string; + model: string; + voice: string; + firstSpeaker: string; + transcriptOptional: boolean; + recordingEnabled: boolean; +} + +export interface UltravoxError { + error: string; + message: string; +} + +/** + * Create a new Ultravox call + */ +export async function createCall(options: { + systemPrompt: string; + voice?: string; + tools?: UltravoxTool[]; + firstSpeaker?: 'FIRST_SPEAKER_AGENT' | 'FIRST_SPEAKER_USER'; +}): Promise<{ success: true; data: CreateCallResponse } | { success: false; error: string }> { + try { + const response = await fetch(`${ULTRAVOX_API_URL}/calls`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-API-Key': ULTRAVOX_API_KEY, + }, + body: JSON.stringify({ + systemPrompt: options.systemPrompt, + model: 'fixie-ai/ultravox', + voice: options.voice || VOICE_ID, + firstSpeaker: options.firstSpeaker || 'FIRST_SPEAKER_AGENT', + selectedTools: options.tools || ULTRAVOX_TOOLS, + medium: { webRtc: {} }, + recordingEnabled: false, + maxDuration: '1800s', // 30 minutes max + }), + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + console.error('[Ultravox] API error:', response.status, errorData); + return { + success: false, + error: errorData.message || `API error: ${response.status}`, + }; + } + + const data: CreateCallResponse = await response.json(); + console.log('[Ultravox] Call created:', data.callId); + return { success: true, data }; + } catch (error) { + console.error('[Ultravox] Create call error:', error); + return { + success: false, + error: error instanceof Error ? error.message : 'Failed to create call', + }; + } +} + +/** + * Get call details + */ +export async function getCall(callId: string): Promise { + try { + const response = await fetch(`${ULTRAVOX_API_URL}/calls/${callId}`, { + method: 'GET', + headers: { + 'X-API-Key': ULTRAVOX_API_KEY, + }, + }); + + if (!response.ok) { + return null; + } + + return await response.json(); + } catch (error) { + console.error('[Ultravox] Get call error:', error); + return null; + } +} + +/** + * End a call + */ +export async function endCall(callId: string): Promise { + try { + const response = await fetch(`${ULTRAVOX_API_URL}/calls/${callId}`, { + method: 'DELETE', + headers: { + 'X-API-Key': ULTRAVOX_API_KEY, + }, + }); + + return response.ok; + } catch (error) { + console.error('[Ultravox] End call error:', error); + return false; + } +} diff --git a/types/index.ts b/types/index.ts index 193d8b9..f4e54d6 100644 --- a/types/index.ts +++ b/types/index.ts @@ -46,6 +46,8 @@ export interface Message { role: 'user' | 'assistant'; content: string; timestamp: Date; + isVoice?: boolean; + isSystem?: boolean; } export interface ChatResponse {