/** * Ultravox Voice AI Service * Creates calls via Ultravox API and manages voice configuration */ // Import Ferdinand data import ferdinandData from '@/assets/data/ferdinand_7days_events.json'; // API Configuration const ULTRAVOX_API_URL = 'https://api.ultravox.ai/api'; const ULTRAVOX_API_KEY = '4miSVLym.HF3lV9y4euiuzcEbPPTLHEugrOu4jpNU'; // Fixed voice - Sarah only export const VOICE_ID = 'Sarah'; export const VOICE_NAME = 'Sarah'; // Tool definitions for function calling export interface UltravoxTool { temporaryTool: { modelToolName: string; description: string; dynamicParameters?: Array<{ name: string; location: string; schema: { type: string; description: string; }; required: boolean; }>; client?: Record; }; } export const ULTRAVOX_TOOLS: UltravoxTool[] = [ { temporaryTool: { modelToolName: 'navigateToDashboard', description: 'Navigate to the Dashboard screen to show wellness overview, charts, and real-time status. Use when user asks to see the dashboard, overview, charts, or wants to check the current status visually.', client: {}, }, }, { temporaryTool: { modelToolName: 'navigateToBeneficiaries', description: 'Navigate to the beneficiaries list screen when user wants to see or manage their loved ones', client: {}, }, }, { temporaryTool: { modelToolName: 'navigateToProfile', description: 'Navigate to the user profile settings screen', client: {}, }, }, { temporaryTool: { modelToolName: 'navigateToChat', description: 'Navigate back to the Julia AI chat screen. Use when user asks to go to chat, messages, Julia, or wants to return to the conversation screen.', client: {}, }, }, ]; // Build context from Ferdinand data function buildFerdinandContext(): string { const client = ferdinandData.client; const summary = ferdinandData.summary; // Get today's alerts const todayData = ferdinandData.days.find(d => d.date === 'today'); const yesterdayData = ferdinandData.days.find(d => d.date === 'yesterday'); let context = ` BENEFICIARY INFORMATION: - Name: ${client.name} - Address: ${client.address} - Monitoring Period: Last 7 days CURRENT STATUS (Today - ${todayData?.day || 'Wednesday'}): `; // Add today's alerts with severity if (todayData?.alerts && todayData.alerts.length > 0) { context += `⚠️ ALERTS TODAY:\n`; todayData.alerts.forEach(alert => { const emoji = alert.severity === 'critical' ? '🔴' : alert.severity === 'high' ? '🟠' : alert.severity === 'medium' ? '🟡' : '🟢'; context += ` ${emoji} ${alert.type.replace(/_/g, ' ').toUpperCase()} at ${alert.time}`; if (alert.note) context += ` - ${alert.note}`; if ('location' in alert && alert.location) context += ` (${alert.location})`; context += '\n'; }); } // Add yesterday's alerts if (yesterdayData?.alerts && yesterdayData.alerts.length > 0) { context += `\nYESTERDAY'S ALERTS:\n`; yesterdayData.alerts.forEach(alert => { const emoji = alert.severity === 'critical' ? '🔴' : alert.severity === 'high' ? '🟠' : alert.severity === 'medium' ? '🟡' : '🟢'; context += ` ${emoji} ${alert.type.replace(/_/g, ' ')} at ${alert.time}`; if (alert.note) context += ` - ${alert.note}`; context += '\n'; }); } // 7-day summary context += ` 7-DAY SUMMARY: - Total alerts: ${summary.total_alerts} - Critical: ${summary.alerts_by_severity.critical} - High: ${summary.alerts_by_severity.high} - Medium: ${summary.alerts_by_severity.medium} - Low: ${summary.alerts_by_severity.low} KEY CONCERNS THIS WEEK: `; // Add all alerts summary summary.alerts_summary.forEach(alert => { const emoji = alert.severity === 'critical' ? '🔴' : alert.severity === 'high' ? '🟠' : alert.severity === 'medium' ? '🟡' : '🟢'; context += ` ${emoji} ${alert.date}: ${alert.type.replace(/_/g, ' ')}\n`; }); // Add typical daily pattern from today's events if (todayData?.events) { const wakeUp = todayData.events.find(e => e.event === 'woke_up'); const sleep = todayData.events.find(e => e.event === 'fell_asleep'); const meals = todayData.events.filter(e => e.event === 'eating' || e.event === 'finished_eating'); const medications = todayData.events.filter(e => e.event === 'medication_taken'); const bathroom = todayData.events.filter(e => e.event === 'urination'); context += ` TODAY'S ACTIVITY PATTERN: - Wake up: ${wakeUp?.time || 'N/A'} - Meals: ${meals.length / 2} meals detected - Medications: ${medications.length} doses taken - Bathroom visits: ${bathroom.length} times - Current location: ${todayData.events[todayData.events.length - 1]?.location || 'bedroom'} `; } return context; } // System prompt generator with Ferdinand context export function getSystemPrompt(): string { const context = buildFerdinandContext(); // Check if there are critical/high alerts const todayData = ferdinandData.days.find(d => d.date === 'today'); const hasCriticalAlerts = todayData?.alerts?.some(a => a.severity === 'critical' || a.severity === 'high'); return `You are Julia, a compassionate AI wellness assistant for WellNuo app. You help caregivers monitor their loved ones' wellbeing. CRITICAL: You are ALWAYS talking about ${ferdinandData.client.name} (the beneficiary), NOT about yourself! ${context} CONVERSATION RULES: 1. When user asks "how are you?" or "how's it going?" - ALWAYS respond about ${ferdinandData.client.name}'s status, NOT about yourself as AI - NEVER say "I'm doing well as an AI" - the user wants to know about their loved one! 2. When user asks "what's happening?" or "any updates?" - report ${ferdinandData.client.name}'s current status and alerts 3. ALWAYS assume questions are about ${ferdinandData.client.name} unless explicitly about app features RESPONSE STYLE - BE CONCISE, NOT PUSHY: - DON'T overwhelm with information immediately - First give a SHORT summary, then ASK if they want details - Example opening: "Hi! ${hasCriticalAlerts ? `I have some important updates about ${ferdinandData.client.name}. Would you like to hear them?` : `${ferdinandData.client.name} is doing well today. Anything specific you'd like to know?`}" - Wait for user to ask before giving long explanations - Keep initial responses to 1-2 sentences max - Only elaborate when user asks "tell me more", "what happened?", etc. BAD (too pushy): "Hi! Ferdinand had a fall at 6:32 AM in the bathroom, his sleep was only 5 hours, he missed his morning medication..." GOOD (concise): "Hi! I have some concerns about ${ferdinandData.client.name} today - there was an incident this morning. Want me to tell you more?" You're speaking with a caregiver who cares deeply about ${ferdinandData.client.name}.`; } // API Response types export interface CreateCallResponse { callId: string; joinUrl: string; created: string; ended?: string; model: string; voice: string; firstSpeaker: string; transcriptOptional: boolean; recordingEnabled: boolean; } export interface UltravoxError { error: string; message: string; } /** * Create a new Ultravox call */ export async function createCall(options: { systemPrompt: string; voice?: string; tools?: UltravoxTool[]; firstSpeaker?: 'FIRST_SPEAKER_AGENT' | 'FIRST_SPEAKER_USER'; }): Promise<{ success: true; data: CreateCallResponse } | { success: false; error: string }> { try { const response = await fetch(`${ULTRAVOX_API_URL}/calls`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-API-Key': ULTRAVOX_API_KEY, }, body: JSON.stringify({ systemPrompt: options.systemPrompt, model: 'fixie-ai/ultravox', voice: options.voice || VOICE_ID, firstSpeaker: options.firstSpeaker || 'FIRST_SPEAKER_AGENT', selectedTools: options.tools || ULTRAVOX_TOOLS, medium: { webRtc: {} }, recordingEnabled: false, maxDuration: '1800s', // 30 minutes max }), }); if (!response.ok) { const errorData = await response.json().catch(() => ({})); console.error('[Ultravox] API error:', response.status, errorData); return { success: false, error: errorData.message || `API error: ${response.status}`, }; } const data: CreateCallResponse = await response.json(); console.log('[Ultravox] Call created:', data.callId); return { success: true, data }; } catch (error) { console.error('[Ultravox] Create call error:', error); return { success: false, error: error instanceof Error ? error.message : 'Failed to create call', }; } } /** * Get call details */ export async function getCall(callId: string): Promise { try { const response = await fetch(`${ULTRAVOX_API_URL}/calls/${callId}`, { method: 'GET', headers: { 'X-API-Key': ULTRAVOX_API_KEY, }, }); if (!response.ok) { return null; } return await response.json(); } catch (error) { console.error('[Ultravox] Get call error:', error); return null; } } /** * End a call */ export async function endCall(callId: string): Promise { try { const response = await fetch(`${ULTRAVOX_API_URL}/calls/${callId}`, { method: 'DELETE', headers: { 'X-API-Key': ULTRAVOX_API_KEY, }, }); return response.ok; } catch (error) { console.error('[Ultravox] End call error:', error); return false; } }