diff --git a/julia-agent/julia-ai/src/agent.py b/julia-agent/julia-ai/src/agent.py index f39d8b8..17d3700 100644 --- a/julia-agent/julia-ai/src/agent.py +++ b/julia-agent/julia-ai/src/agent.py @@ -7,6 +7,7 @@ Uses WellNuo voice_ask API for LLM responses, Deepgram for STT/TTS import logging import os import random +import re import aiohttp from livekit.agents import ( @@ -31,6 +32,110 @@ WELLNUO_PASSWORD = os.getenv("WELLNUO_PASSWORD", "anandk_8") # Hardcoded Ferdinand's deployment_id for testing DEPLOYMENT_ID = os.getenv("DEPLOYMENT_ID", "21") +# Keywords that indicate user is asking about the care recipient's status +STATUS_KEYWORDS = [ + r"\bhow\s+is\b", + r"\bhow\'?s\b", + r"\bhow\s+are\b", + r"\btell\s+me\s+about\b", + r"\bwhat\'?s\s+up\s+with\b", + r"\bupdate\s+on\b", + r"\bstatus\b", + r"\bdoing\b", + r"\bfeeling\b", + r"\bcheck\s+on\b", + r"\bis\s+\w+\s+okay\b", + r"\bis\s+\w+\s+alright\b", + r"\bis\s+\w+\s+fine\b", + r"\bokay\?\b", + r"\balright\?\b", +] + +# Keywords that indicate the subject is the care recipient +SUBJECT_KEYWORDS = [ + r"\bdad\b", + r"\bfather\b", + r"\bferdinand\b", + r"\bhim\b", + r"\bhe\b", + r"\bmy\s+dad\b", + r"\bmy\s+father\b", + r"\bthe\s+patient\b", + r"\bloved\s+one\b", + r"\bparent\b", + r"\bgrandpa\b", + r"\bgrandfather\b", +] + +# Sleep-related keywords +SLEEP_KEYWORDS = [ + r"\bsleep\b", + r"\bslept\b", + r"\brest\b", + r"\bnight\b", + r"\bwake\b", + r"\bwoke\b", + r"\bbedroom\b", + r"\bbathroom\b", +] + +# Activity-related keywords +ACTIVITY_KEYWORDS = [ + r"\bactivity\b", + r"\bactive\b", + r"\bmoving\b", + r"\bwhere\s+is\b", + r"\blocation\b", + r"\broom\b", + r"\bkitchen\b", + r"\bliving\b", +] + + +def normalize_question(user_message: str) -> str: + """ + Transform user questions into format WellNuo API understands. + + WellNuo API only responds with real sensor data for very specific phrases. + This function maps common user questions to those phrases. + """ + msg_lower = user_message.lower().strip() + + # Check if asking about status/wellbeing + is_status_query = any(re.search(p, msg_lower) for p in STATUS_KEYWORDS) + is_about_recipient = any(re.search(p, msg_lower) for p in SUBJECT_KEYWORDS) + is_sleep_query = any(re.search(p, msg_lower) for p in SLEEP_KEYWORDS) + is_activity_query = any(re.search(p, msg_lower) for p in ACTIVITY_KEYWORDS) + + # If asking about the care recipient's general status + if is_status_query and is_about_recipient: + logger.info(f"Normalized '{user_message}' -> 'how is dad doing'") + return "how is dad doing" + + # If asking about sleep specifically + if is_sleep_query and (is_about_recipient or is_status_query): + logger.info(f"Normalized '{user_message}' -> 'how did dad sleep'") + # Note: This might still not work with WellNuo API, but we try + return "how is dad doing" # Fall back to general status which includes sleep + + # If asking about activity/location + if is_activity_query and (is_about_recipient or is_status_query): + logger.info(f"Normalized '{user_message}' -> 'how is dad doing'") + return "how is dad doing" # General status includes location + + # Generic status questions without clear subject + if is_status_query and not is_about_recipient: + # User might say "how is he" or "how are things" + # Assume they mean the care recipient + logger.info( + f"Normalized '{user_message}' -> 'how is dad doing' (assumed recipient)" + ) + return "how is dad doing" + + # If no transformation needed, return original + logger.info(f"No normalization applied to: '{user_message}'") + return user_message + class WellNuoLLM(llm.LLM): """Custom LLM that uses WellNuo voice_ask API.""" @@ -77,7 +182,10 @@ class WellNuoLLM(llm.LLM): if not user_message: return "I'm here to help. What would you like to know?" - logger.info(f"User question: {user_message}") + logger.info(f"User question (original): {user_message}") + + # Normalize question to format WellNuo API understands + normalized_question = normalize_question(user_message) try: token = await self._ensure_token() @@ -88,7 +196,7 @@ class WellNuoLLM(llm.LLM): "clientId": "MA_001", "user_name": WELLNUO_USER, "token": token, - "question": user_message, + "question": normalized_question, "deployment_id": DEPLOYMENT_ID, } async with session.post(WELLNUO_API_URL, data=data) as resp: