Improve STT quality and add session/chat management

- Switch Android STT from on-device to cloud recognition for better accuracy
- Add lastMessageWasVoiceRef to prevent TTS for text-typed messages
- Stop voice session and clear chat when changing Deployment or Voice API
- Ensures clean state when switching between beneficiaries/models
This commit is contained in:
Sergei 2026-01-29 18:29:00 -08:00
parent 5174366384
commit 9f12830850
7 changed files with 567 additions and 432 deletions

View File

@ -27,7 +27,7 @@
"bitcode": false "bitcode": false
}, },
"android": { "android": {
"package": "com.wellnuo.app", "package": "com.wellnuo.BluetoothScanner",
"softwareKeyboardLayoutMode": "resize", "softwareKeyboardLayoutMode": "resize",
"adaptiveIcon": { "adaptiveIcon": {
"backgroundColor": "#E6F4FE", "backgroundColor": "#E6F4FE",
@ -56,7 +56,7 @@
}, },
"plugins": [ "plugins": [
[ [
"@jamsch/expo-speech-recognition", "expo-speech-recognition",
{ {
"microphonePermission": "WellNuo needs access to your microphone to listen to your voice commands.", "microphonePermission": "WellNuo needs access to your microphone to listen to your voice commands.",
"speechRecognitionPermission": "WellNuo uses speech recognition to convert your voice to text for Julia AI." "speechRecognitionPermission": "WellNuo uses speech recognition to convert your voice to text for Julia AI."

View File

@ -261,8 +261,8 @@ export default function TabLayout() {
// Delay to let TTS fully release audio focus, then restart STT // Delay to let TTS fully release audio focus, then restart STT
// iOS: 300ms for smooth audio fade // iOS: 300ms for smooth audio fade
// Android: 50ms (Audio Focus releases immediately) // Android: 0ms - start immediately to catch first words (Audio Focus releases instantly)
const delay = Platform.OS === 'android' ? 50 : 300; const delay = Platform.OS === 'android' ? 0 : 300;
console.log(`${platformPrefix} [TabLayout] ⏱️ Waiting ${delay}ms before restarting STT (audio focus release)`); console.log(`${platformPrefix} [TabLayout] ⏱️ Waiting ${delay}ms before restarting STT (audio focus release)`);
const timer = setTimeout(() => { const timer = setTimeout(() => {

View File

@ -157,6 +157,9 @@ export default function ChatScreen() {
} }
}, [voicePartial, voiceIsListening]); }, [voicePartial, voiceIsListening]);
// Track if current message was sent via voice (to decide whether to speak response)
const lastMessageWasVoiceRef = useRef(false);
// Clear input when voice switches to processing (transcript was sent) // Clear input when voice switches to processing (transcript was sent)
const prevVoiceStatusRef = useRef(voiceStatus); const prevVoiceStatusRef = useRef(voiceStatus);
useEffect(() => { useEffect(() => {
@ -164,6 +167,8 @@ export default function ChatScreen() {
prevVoiceStatusRef.current = voiceStatus; prevVoiceStatusRef.current = voiceStatus;
if (prev === 'listening' && voiceStatus === 'processing') { if (prev === 'listening' && voiceStatus === 'processing') {
setInput(''); setInput('');
// Mark that this message was sent via voice
lastMessageWasVoiceRef.current = true;
} }
}, [voiceStatus]); }, [voiceStatus]);
@ -187,34 +192,43 @@ export default function ChatScreen() {
}, []) }, [])
); );
// When deployment ID changes, end call and clear chat // When deployment ID changes BY USER ACTION, clear chat
// Track previous value to detect actual changes (not just re-renders) // Track previous value to detect actual changes (not just initial load)
const previousDeploymentIdRef = useRef<string | null | undefined>(undefined); const previousDeploymentIdRef = useRef<string | null | undefined>(undefined);
// Track if we've done initial load (to distinguish from user changing deployment)
const initialLoadDoneRef = useRef(false);
useEffect(() => { useEffect(() => {
// undefined means "not yet initialized" - store current value and skip // First render - just store the value, don't clear anything
if (previousDeploymentIdRef.current === undefined) { if (previousDeploymentIdRef.current === undefined) {
console.log('[Chat] Initializing deployment tracking:', customDeploymentId, 'name:', deploymentName); console.log('[Chat] First render, storing deployment:', customDeploymentId);
previousDeploymentIdRef.current = customDeploymentId; previousDeploymentIdRef.current = customDeploymentId;
// Update initial message with deployment name if we have one
if (customDeploymentId || deploymentName) {
setMessages([createInitialMessage(deploymentName)]);
}
return; return;
} }
// Check if deployment actually changed // Initial async load completed (null → actual value) - don't clear chat!
if (previousDeploymentIdRef.current !== customDeploymentId) { if (!initialLoadDoneRef.current && previousDeploymentIdRef.current === null && customDeploymentId) {
console.log('[Chat] Deployment changed!', { console.log('[Chat] Initial deployment load complete:', customDeploymentId, '- NOT clearing chat');
previousDeploymentIdRef.current = customDeploymentId;
initialLoadDoneRef.current = true;
return;
}
// Mark initial load as done
if (!initialLoadDoneRef.current && customDeploymentId) {
initialLoadDoneRef.current = true;
}
// Only clear chat if deployment ACTUALLY changed by user (after initial load)
if (initialLoadDoneRef.current && previousDeploymentIdRef.current !== customDeploymentId) {
console.log('[Chat] Deployment CHANGED by user!', {
old: previousDeploymentIdRef.current, old: previousDeploymentIdRef.current,
new: customDeploymentId, new: customDeploymentId,
name: deploymentName, name: deploymentName,
}); });
// Clear chat with new initial message (use name instead of ID) // Clear chat with new initial message
setMessages([createInitialMessage(deploymentName)]); setMessages([createInitialMessage(deploymentName)]);
// Update ref
previousDeploymentIdRef.current = customDeploymentId; previousDeploymentIdRef.current = customDeploymentId;
} }
}, [customDeploymentId, deploymentName, createInitialMessage]); }, [customDeploymentId, deploymentName, createInitialMessage]);
@ -358,6 +372,10 @@ export default function ChatScreen() {
setInput(''); setInput('');
inputRef.current = ''; inputRef.current = '';
// This message was sent via text input (keyboard), not voice
// Reset the flag so response won't be spoken
lastMessageWasVoiceRef.current = false;
setMessages(prev => [...prev, userMessage]); setMessages(prev => [...prev, userMessage]);
setIsSending(true); setIsSending(true);
Keyboard.dismiss(); Keyboard.dismiss();
@ -414,9 +432,11 @@ export default function ChatScreen() {
}; };
setMessages(prev => [...prev, assistantMessage]); setMessages(prev => [...prev, assistantMessage]);
// Only speak the response if voice session is active (FAB pressed) // Only speak the response if:
// Don't auto-speak for text-only chat messages // 1. Voice session is active (FAB pressed) AND
if (voiceIsActive) { // 2. The user's message was sent via voice (not typed)
// This way, typing a message while voice is active won't trigger TTS
if (voiceIsActive && lastMessageWasVoiceRef.current) {
speak(responseText); speak(responseText);
} }
} else { } else {

View File

@ -16,6 +16,7 @@ import { Ionicons } from '@expo/vector-icons';
import { SafeAreaView } from 'react-native-safe-area-context'; import { SafeAreaView } from 'react-native-safe-area-context';
import { useAuth } from '@/contexts/AuthContext'; import { useAuth } from '@/contexts/AuthContext';
import { useVoice } from '@/contexts/VoiceContext'; import { useVoice } from '@/contexts/VoiceContext';
import { useChat } from '@/contexts/ChatContext';
import { api } from '@/services/api'; import { api } from '@/services/api';
import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme'; import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme';
@ -56,7 +57,8 @@ function MenuItem({
export default function ProfileScreen() { export default function ProfileScreen() {
const { user, logout } = useAuth(); const { user, logout } = useAuth();
const { updateVoiceApiType, stopSession, isActive } = useVoice(); const { updateVoiceApiType, stopSession } = useVoice();
const { clearMessages } = useChat();
const [deploymentId, setDeploymentId] = useState<string>(''); const [deploymentId, setDeploymentId] = useState<string>('');
const [deploymentName, setDeploymentName] = useState<string>(''); const [deploymentName, setDeploymentName] = useState<string>('');
const [showDeploymentModal, setShowDeploymentModal] = useState(false); const [showDeploymentModal, setShowDeploymentModal] = useState(false);
@ -124,6 +126,18 @@ export default function ProfileScreen() {
try { try {
const result = await api.validateDeploymentId(trimmed); const result = await api.validateDeploymentId(trimmed);
if (result.ok && result.data?.valid) { if (result.ok && result.data?.valid) {
// ALWAYS stop voice session when deployment changes
console.log('[Profile] Stopping voice session and clearing chat before deployment change');
stopSession();
// Clear chat history when deployment changes
clearMessages({
id: '1',
role: 'assistant',
content: `Hello! I'm Julia, your AI wellness companion.${result.data.name ? `\n\nI'm here to help you monitor ${result.data.name}.` : ''}\n\nType a message below to chat with me.`,
timestamp: new Date(),
});
await api.setDeploymentId(trimmed); await api.setDeploymentId(trimmed);
if (result.data.name) { if (result.data.name) {
await api.setDeploymentName(result.data.name); await api.setDeploymentName(result.data.name);
@ -142,25 +156,43 @@ export default function ProfileScreen() {
setIsValidating(false); setIsValidating(false);
} }
} else { } else {
// ALWAYS stop voice session when deployment is cleared
console.log('[Profile] Stopping voice session and clearing chat before clearing deployment');
stopSession();
// Clear chat history when deployment is cleared
clearMessages({
id: '1',
role: 'assistant',
content: "Hello! I'm Julia, your AI wellness companion.\n\nType a message below to chat with me.",
timestamp: new Date(),
});
await api.clearDeploymentId(); await api.clearDeploymentId();
setDeploymentId(''); setDeploymentId('');
setDeploymentName(''); setDeploymentName('');
setShowDeploymentModal(false); setShowDeploymentModal(false);
} }
}, [tempDeploymentId]); }, [tempDeploymentId, stopSession, clearMessages]);
const saveVoiceApiType = useCallback(async () => { const saveVoiceApiType = useCallback(async () => {
// Stop active voice session if any before changing API type // ALWAYS stop voice session when API type changes
if (isActive) { console.log('[Profile] Stopping voice session and clearing chat before API type change');
console.log('[Profile] Stopping active voice session before API type change');
stopSession(); stopSession();
}
// Clear chat history when Voice API changes
clearMessages({
id: '1',
role: 'assistant',
content: "Hello! I'm Julia, your AI wellness companion.\n\nType a message below to chat with me.",
timestamp: new Date(),
});
await api.setVoiceApiType(tempVoiceApiType); await api.setVoiceApiType(tempVoiceApiType);
setVoiceApiType(tempVoiceApiType); setVoiceApiType(tempVoiceApiType);
updateVoiceApiType(tempVoiceApiType); updateVoiceApiType(tempVoiceApiType);
setShowVoiceApiModal(false); setShowVoiceApiModal(false);
}, [tempVoiceApiType, updateVoiceApiType, isActive, stopSession]); }, [tempVoiceApiType, updateVoiceApiType, stopSession, clearMessages]);
const openTerms = () => { const openTerms = () => {
router.push('/terms'); router.push('/terms');

View File

@ -30,7 +30,7 @@ let ExpoSpeechRecognitionModule: any = null;
let useSpeechRecognitionEvent: any = () => {}; // no-op by default let useSpeechRecognitionEvent: any = () => {}; // no-op by default
try { try {
const speechRecognition = require('@jamsch/expo-speech-recognition'); const speechRecognition = require('expo-speech-recognition');
ExpoSpeechRecognitionModule = speechRecognition.ExpoSpeechRecognitionModule; ExpoSpeechRecognitionModule = speechRecognition.ExpoSpeechRecognitionModule;
useSpeechRecognitionEvent = speechRecognition.useSpeechRecognitionEvent; useSpeechRecognitionEvent = speechRecognition.useSpeechRecognitionEvent;
} catch (e) { } catch (e) {
@ -258,6 +258,10 @@ export function useSpeechRecognition(
interimResults, interimResults,
continuous, continuous,
addsPunctuation: Platform.OS === 'ios' ? addsPunctuation : undefined, addsPunctuation: Platform.OS === 'ios' ? addsPunctuation : undefined,
// Android: use CLOUD recognition for better quality
// On-device models often have worse accuracy
// Setting to false allows the system to use Google's cloud ASR
requiresOnDeviceRecognition: false,
// Android-specific: longer silence timeout for more natural pauses // Android-specific: longer silence timeout for more natural pauses
// CRITICAL FIX: Increased from 2000ms to 4000ms to prevent premature speech cutoff // CRITICAL FIX: Increased from 2000ms to 4000ms to prevent premature speech cutoff
// This allows users to pause between sentences without being cut off // This allows users to pause between sentences without being cut off

877
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -16,25 +16,29 @@
"dependencies": { "dependencies": {
"@dr.pogodin/react-native-fs": "^2.36.2", "@dr.pogodin/react-native-fs": "^2.36.2",
"@expo/vector-icons": "^15.0.3", "@expo/vector-icons": "^15.0.3",
"@jamsch/expo-speech-recognition": "^0.2.15",
"@notifee/react-native": "^9.1.8", "@notifee/react-native": "^9.1.8",
"@react-native-async-storage/async-storage": "2.2.0", "@react-native-async-storage/async-storage": "2.2.0",
"@react-navigation/bottom-tabs": "^7.4.0", "@react-navigation/bottom-tabs": "^7.4.0",
"@react-navigation/elements": "^2.6.3", "@react-navigation/elements": "^2.6.3",
"@react-navigation/native": "^7.1.8", "@react-navigation/native": "^7.1.8",
"@stripe/stripe-react-native": "^0.58.0",
"expo": "~54.0.29", "expo": "~54.0.29",
"expo-clipboard": "~8.0.8", "expo-clipboard": "~8.0.8",
"expo-constants": "~18.0.12", "expo-constants": "~18.0.12",
"expo-crypto": "^15.0.8",
"expo-device": "~8.0.10", "expo-device": "~8.0.10",
"expo-file-system": "~19.0.21", "expo-file-system": "~19.0.21",
"expo-font": "~14.0.10", "expo-font": "~14.0.10",
"expo-haptics": "~15.0.8", "expo-haptics": "~15.0.8",
"expo-image": "~3.0.11", "expo-image": "~3.0.11",
"expo-image-manipulator": "^14.0.8",
"expo-image-picker": "^17.0.10",
"expo-keep-awake": "^15.0.8", "expo-keep-awake": "^15.0.8",
"expo-linking": "~8.0.10", "expo-linking": "~8.0.10",
"expo-router": "~6.0.19", "expo-router": "~6.0.19",
"expo-secure-store": "^15.0.8", "expo-secure-store": "^15.0.8",
"expo-speech": "~14.0.6", "expo-speech": "~14.0.6",
"expo-speech-recognition": "^3.1.0",
"expo-splash-screen": "~31.0.12", "expo-splash-screen": "~31.0.12",
"expo-status-bar": "~3.0.9", "expo-status-bar": "~3.0.9",
"expo-symbols": "~1.0.8", "expo-symbols": "~1.0.8",