Improve STT quality and add session/chat management
- Switch Android STT from on-device to cloud recognition for better accuracy - Add lastMessageWasVoiceRef to prevent TTS for text-typed messages - Stop voice session and clear chat when changing Deployment or Voice API - Ensures clean state when switching between beneficiaries/models
This commit is contained in:
parent
5174366384
commit
9f12830850
4
app.json
4
app.json
@ -27,7 +27,7 @@
|
|||||||
"bitcode": false
|
"bitcode": false
|
||||||
},
|
},
|
||||||
"android": {
|
"android": {
|
||||||
"package": "com.wellnuo.app",
|
"package": "com.wellnuo.BluetoothScanner",
|
||||||
"softwareKeyboardLayoutMode": "resize",
|
"softwareKeyboardLayoutMode": "resize",
|
||||||
"adaptiveIcon": {
|
"adaptiveIcon": {
|
||||||
"backgroundColor": "#E6F4FE",
|
"backgroundColor": "#E6F4FE",
|
||||||
@ -56,7 +56,7 @@
|
|||||||
},
|
},
|
||||||
"plugins": [
|
"plugins": [
|
||||||
[
|
[
|
||||||
"@jamsch/expo-speech-recognition",
|
"expo-speech-recognition",
|
||||||
{
|
{
|
||||||
"microphonePermission": "WellNuo needs access to your microphone to listen to your voice commands.",
|
"microphonePermission": "WellNuo needs access to your microphone to listen to your voice commands.",
|
||||||
"speechRecognitionPermission": "WellNuo uses speech recognition to convert your voice to text for Julia AI."
|
"speechRecognitionPermission": "WellNuo uses speech recognition to convert your voice to text for Julia AI."
|
||||||
|
|||||||
@ -261,8 +261,8 @@ export default function TabLayout() {
|
|||||||
|
|
||||||
// Delay to let TTS fully release audio focus, then restart STT
|
// Delay to let TTS fully release audio focus, then restart STT
|
||||||
// iOS: 300ms for smooth audio fade
|
// iOS: 300ms for smooth audio fade
|
||||||
// Android: 50ms (Audio Focus releases immediately)
|
// Android: 0ms - start immediately to catch first words (Audio Focus releases instantly)
|
||||||
const delay = Platform.OS === 'android' ? 50 : 300;
|
const delay = Platform.OS === 'android' ? 0 : 300;
|
||||||
console.log(`${platformPrefix} [TabLayout] ⏱️ Waiting ${delay}ms before restarting STT (audio focus release)`);
|
console.log(`${platformPrefix} [TabLayout] ⏱️ Waiting ${delay}ms before restarting STT (audio focus release)`);
|
||||||
|
|
||||||
const timer = setTimeout(() => {
|
const timer = setTimeout(() => {
|
||||||
|
|||||||
@ -157,6 +157,9 @@ export default function ChatScreen() {
|
|||||||
}
|
}
|
||||||
}, [voicePartial, voiceIsListening]);
|
}, [voicePartial, voiceIsListening]);
|
||||||
|
|
||||||
|
// Track if current message was sent via voice (to decide whether to speak response)
|
||||||
|
const lastMessageWasVoiceRef = useRef(false);
|
||||||
|
|
||||||
// Clear input when voice switches to processing (transcript was sent)
|
// Clear input when voice switches to processing (transcript was sent)
|
||||||
const prevVoiceStatusRef = useRef(voiceStatus);
|
const prevVoiceStatusRef = useRef(voiceStatus);
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@ -164,6 +167,8 @@ export default function ChatScreen() {
|
|||||||
prevVoiceStatusRef.current = voiceStatus;
|
prevVoiceStatusRef.current = voiceStatus;
|
||||||
if (prev === 'listening' && voiceStatus === 'processing') {
|
if (prev === 'listening' && voiceStatus === 'processing') {
|
||||||
setInput('');
|
setInput('');
|
||||||
|
// Mark that this message was sent via voice
|
||||||
|
lastMessageWasVoiceRef.current = true;
|
||||||
}
|
}
|
||||||
}, [voiceStatus]);
|
}, [voiceStatus]);
|
||||||
|
|
||||||
@ -187,34 +192,43 @@ export default function ChatScreen() {
|
|||||||
}, [])
|
}, [])
|
||||||
);
|
);
|
||||||
|
|
||||||
// When deployment ID changes, end call and clear chat
|
// When deployment ID changes BY USER ACTION, clear chat
|
||||||
// Track previous value to detect actual changes (not just re-renders)
|
// Track previous value to detect actual changes (not just initial load)
|
||||||
const previousDeploymentIdRef = useRef<string | null | undefined>(undefined);
|
const previousDeploymentIdRef = useRef<string | null | undefined>(undefined);
|
||||||
|
// Track if we've done initial load (to distinguish from user changing deployment)
|
||||||
|
const initialLoadDoneRef = useRef(false);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
// undefined means "not yet initialized" - store current value and skip
|
// First render - just store the value, don't clear anything
|
||||||
if (previousDeploymentIdRef.current === undefined) {
|
if (previousDeploymentIdRef.current === undefined) {
|
||||||
console.log('[Chat] Initializing deployment tracking:', customDeploymentId, 'name:', deploymentName);
|
console.log('[Chat] First render, storing deployment:', customDeploymentId);
|
||||||
previousDeploymentIdRef.current = customDeploymentId;
|
previousDeploymentIdRef.current = customDeploymentId;
|
||||||
// Update initial message with deployment name if we have one
|
|
||||||
if (customDeploymentId || deploymentName) {
|
|
||||||
setMessages([createInitialMessage(deploymentName)]);
|
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if deployment actually changed
|
// Initial async load completed (null → actual value) - don't clear chat!
|
||||||
if (previousDeploymentIdRef.current !== customDeploymentId) {
|
if (!initialLoadDoneRef.current && previousDeploymentIdRef.current === null && customDeploymentId) {
|
||||||
console.log('[Chat] Deployment changed!', {
|
console.log('[Chat] Initial deployment load complete:', customDeploymentId, '- NOT clearing chat');
|
||||||
|
previousDeploymentIdRef.current = customDeploymentId;
|
||||||
|
initialLoadDoneRef.current = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mark initial load as done
|
||||||
|
if (!initialLoadDoneRef.current && customDeploymentId) {
|
||||||
|
initialLoadDoneRef.current = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only clear chat if deployment ACTUALLY changed by user (after initial load)
|
||||||
|
if (initialLoadDoneRef.current && previousDeploymentIdRef.current !== customDeploymentId) {
|
||||||
|
console.log('[Chat] Deployment CHANGED by user!', {
|
||||||
old: previousDeploymentIdRef.current,
|
old: previousDeploymentIdRef.current,
|
||||||
new: customDeploymentId,
|
new: customDeploymentId,
|
||||||
name: deploymentName,
|
name: deploymentName,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Clear chat with new initial message (use name instead of ID)
|
// Clear chat with new initial message
|
||||||
setMessages([createInitialMessage(deploymentName)]);
|
setMessages([createInitialMessage(deploymentName)]);
|
||||||
|
|
||||||
// Update ref
|
|
||||||
previousDeploymentIdRef.current = customDeploymentId;
|
previousDeploymentIdRef.current = customDeploymentId;
|
||||||
}
|
}
|
||||||
}, [customDeploymentId, deploymentName, createInitialMessage]);
|
}, [customDeploymentId, deploymentName, createInitialMessage]);
|
||||||
@ -358,6 +372,10 @@ export default function ChatScreen() {
|
|||||||
setInput('');
|
setInput('');
|
||||||
inputRef.current = '';
|
inputRef.current = '';
|
||||||
|
|
||||||
|
// This message was sent via text input (keyboard), not voice
|
||||||
|
// Reset the flag so response won't be spoken
|
||||||
|
lastMessageWasVoiceRef.current = false;
|
||||||
|
|
||||||
setMessages(prev => [...prev, userMessage]);
|
setMessages(prev => [...prev, userMessage]);
|
||||||
setIsSending(true);
|
setIsSending(true);
|
||||||
Keyboard.dismiss();
|
Keyboard.dismiss();
|
||||||
@ -414,9 +432,11 @@ export default function ChatScreen() {
|
|||||||
};
|
};
|
||||||
setMessages(prev => [...prev, assistantMessage]);
|
setMessages(prev => [...prev, assistantMessage]);
|
||||||
|
|
||||||
// Only speak the response if voice session is active (FAB pressed)
|
// Only speak the response if:
|
||||||
// Don't auto-speak for text-only chat messages
|
// 1. Voice session is active (FAB pressed) AND
|
||||||
if (voiceIsActive) {
|
// 2. The user's message was sent via voice (not typed)
|
||||||
|
// This way, typing a message while voice is active won't trigger TTS
|
||||||
|
if (voiceIsActive && lastMessageWasVoiceRef.current) {
|
||||||
speak(responseText);
|
speak(responseText);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@ -16,6 +16,7 @@ import { Ionicons } from '@expo/vector-icons';
|
|||||||
import { SafeAreaView } from 'react-native-safe-area-context';
|
import { SafeAreaView } from 'react-native-safe-area-context';
|
||||||
import { useAuth } from '@/contexts/AuthContext';
|
import { useAuth } from '@/contexts/AuthContext';
|
||||||
import { useVoice } from '@/contexts/VoiceContext';
|
import { useVoice } from '@/contexts/VoiceContext';
|
||||||
|
import { useChat } from '@/contexts/ChatContext';
|
||||||
import { api } from '@/services/api';
|
import { api } from '@/services/api';
|
||||||
import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme';
|
import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme';
|
||||||
|
|
||||||
@ -56,7 +57,8 @@ function MenuItem({
|
|||||||
|
|
||||||
export default function ProfileScreen() {
|
export default function ProfileScreen() {
|
||||||
const { user, logout } = useAuth();
|
const { user, logout } = useAuth();
|
||||||
const { updateVoiceApiType, stopSession, isActive } = useVoice();
|
const { updateVoiceApiType, stopSession } = useVoice();
|
||||||
|
const { clearMessages } = useChat();
|
||||||
const [deploymentId, setDeploymentId] = useState<string>('');
|
const [deploymentId, setDeploymentId] = useState<string>('');
|
||||||
const [deploymentName, setDeploymentName] = useState<string>('');
|
const [deploymentName, setDeploymentName] = useState<string>('');
|
||||||
const [showDeploymentModal, setShowDeploymentModal] = useState(false);
|
const [showDeploymentModal, setShowDeploymentModal] = useState(false);
|
||||||
@ -124,6 +126,18 @@ export default function ProfileScreen() {
|
|||||||
try {
|
try {
|
||||||
const result = await api.validateDeploymentId(trimmed);
|
const result = await api.validateDeploymentId(trimmed);
|
||||||
if (result.ok && result.data?.valid) {
|
if (result.ok && result.data?.valid) {
|
||||||
|
// ALWAYS stop voice session when deployment changes
|
||||||
|
console.log('[Profile] Stopping voice session and clearing chat before deployment change');
|
||||||
|
stopSession();
|
||||||
|
|
||||||
|
// Clear chat history when deployment changes
|
||||||
|
clearMessages({
|
||||||
|
id: '1',
|
||||||
|
role: 'assistant',
|
||||||
|
content: `Hello! I'm Julia, your AI wellness companion.${result.data.name ? `\n\nI'm here to help you monitor ${result.data.name}.` : ''}\n\nType a message below to chat with me.`,
|
||||||
|
timestamp: new Date(),
|
||||||
|
});
|
||||||
|
|
||||||
await api.setDeploymentId(trimmed);
|
await api.setDeploymentId(trimmed);
|
||||||
if (result.data.name) {
|
if (result.data.name) {
|
||||||
await api.setDeploymentName(result.data.name);
|
await api.setDeploymentName(result.data.name);
|
||||||
@ -142,25 +156,43 @@ export default function ProfileScreen() {
|
|||||||
setIsValidating(false);
|
setIsValidating(false);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
// ALWAYS stop voice session when deployment is cleared
|
||||||
|
console.log('[Profile] Stopping voice session and clearing chat before clearing deployment');
|
||||||
|
stopSession();
|
||||||
|
|
||||||
|
// Clear chat history when deployment is cleared
|
||||||
|
clearMessages({
|
||||||
|
id: '1',
|
||||||
|
role: 'assistant',
|
||||||
|
content: "Hello! I'm Julia, your AI wellness companion.\n\nType a message below to chat with me.",
|
||||||
|
timestamp: new Date(),
|
||||||
|
});
|
||||||
|
|
||||||
await api.clearDeploymentId();
|
await api.clearDeploymentId();
|
||||||
setDeploymentId('');
|
setDeploymentId('');
|
||||||
setDeploymentName('');
|
setDeploymentName('');
|
||||||
setShowDeploymentModal(false);
|
setShowDeploymentModal(false);
|
||||||
}
|
}
|
||||||
}, [tempDeploymentId]);
|
}, [tempDeploymentId, stopSession, clearMessages]);
|
||||||
|
|
||||||
const saveVoiceApiType = useCallback(async () => {
|
const saveVoiceApiType = useCallback(async () => {
|
||||||
// Stop active voice session if any before changing API type
|
// ALWAYS stop voice session when API type changes
|
||||||
if (isActive) {
|
console.log('[Profile] Stopping voice session and clearing chat before API type change');
|
||||||
console.log('[Profile] Stopping active voice session before API type change');
|
|
||||||
stopSession();
|
stopSession();
|
||||||
}
|
|
||||||
|
// Clear chat history when Voice API changes
|
||||||
|
clearMessages({
|
||||||
|
id: '1',
|
||||||
|
role: 'assistant',
|
||||||
|
content: "Hello! I'm Julia, your AI wellness companion.\n\nType a message below to chat with me.",
|
||||||
|
timestamp: new Date(),
|
||||||
|
});
|
||||||
|
|
||||||
await api.setVoiceApiType(tempVoiceApiType);
|
await api.setVoiceApiType(tempVoiceApiType);
|
||||||
setVoiceApiType(tempVoiceApiType);
|
setVoiceApiType(tempVoiceApiType);
|
||||||
updateVoiceApiType(tempVoiceApiType);
|
updateVoiceApiType(tempVoiceApiType);
|
||||||
setShowVoiceApiModal(false);
|
setShowVoiceApiModal(false);
|
||||||
}, [tempVoiceApiType, updateVoiceApiType, isActive, stopSession]);
|
}, [tempVoiceApiType, updateVoiceApiType, stopSession, clearMessages]);
|
||||||
|
|
||||||
const openTerms = () => {
|
const openTerms = () => {
|
||||||
router.push('/terms');
|
router.push('/terms');
|
||||||
|
|||||||
@ -30,7 +30,7 @@ let ExpoSpeechRecognitionModule: any = null;
|
|||||||
let useSpeechRecognitionEvent: any = () => {}; // no-op by default
|
let useSpeechRecognitionEvent: any = () => {}; // no-op by default
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const speechRecognition = require('@jamsch/expo-speech-recognition');
|
const speechRecognition = require('expo-speech-recognition');
|
||||||
ExpoSpeechRecognitionModule = speechRecognition.ExpoSpeechRecognitionModule;
|
ExpoSpeechRecognitionModule = speechRecognition.ExpoSpeechRecognitionModule;
|
||||||
useSpeechRecognitionEvent = speechRecognition.useSpeechRecognitionEvent;
|
useSpeechRecognitionEvent = speechRecognition.useSpeechRecognitionEvent;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@ -258,6 +258,10 @@ export function useSpeechRecognition(
|
|||||||
interimResults,
|
interimResults,
|
||||||
continuous,
|
continuous,
|
||||||
addsPunctuation: Platform.OS === 'ios' ? addsPunctuation : undefined,
|
addsPunctuation: Platform.OS === 'ios' ? addsPunctuation : undefined,
|
||||||
|
// Android: use CLOUD recognition for better quality
|
||||||
|
// On-device models often have worse accuracy
|
||||||
|
// Setting to false allows the system to use Google's cloud ASR
|
||||||
|
requiresOnDeviceRecognition: false,
|
||||||
// Android-specific: longer silence timeout for more natural pauses
|
// Android-specific: longer silence timeout for more natural pauses
|
||||||
// CRITICAL FIX: Increased from 2000ms to 4000ms to prevent premature speech cutoff
|
// CRITICAL FIX: Increased from 2000ms to 4000ms to prevent premature speech cutoff
|
||||||
// This allows users to pause between sentences without being cut off
|
// This allows users to pause between sentences without being cut off
|
||||||
|
|||||||
877
package-lock.json
generated
877
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@ -16,25 +16,29 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@dr.pogodin/react-native-fs": "^2.36.2",
|
"@dr.pogodin/react-native-fs": "^2.36.2",
|
||||||
"@expo/vector-icons": "^15.0.3",
|
"@expo/vector-icons": "^15.0.3",
|
||||||
"@jamsch/expo-speech-recognition": "^0.2.15",
|
|
||||||
"@notifee/react-native": "^9.1.8",
|
"@notifee/react-native": "^9.1.8",
|
||||||
"@react-native-async-storage/async-storage": "2.2.0",
|
"@react-native-async-storage/async-storage": "2.2.0",
|
||||||
"@react-navigation/bottom-tabs": "^7.4.0",
|
"@react-navigation/bottom-tabs": "^7.4.0",
|
||||||
"@react-navigation/elements": "^2.6.3",
|
"@react-navigation/elements": "^2.6.3",
|
||||||
"@react-navigation/native": "^7.1.8",
|
"@react-navigation/native": "^7.1.8",
|
||||||
|
"@stripe/stripe-react-native": "^0.58.0",
|
||||||
"expo": "~54.0.29",
|
"expo": "~54.0.29",
|
||||||
"expo-clipboard": "~8.0.8",
|
"expo-clipboard": "~8.0.8",
|
||||||
"expo-constants": "~18.0.12",
|
"expo-constants": "~18.0.12",
|
||||||
|
"expo-crypto": "^15.0.8",
|
||||||
"expo-device": "~8.0.10",
|
"expo-device": "~8.0.10",
|
||||||
"expo-file-system": "~19.0.21",
|
"expo-file-system": "~19.0.21",
|
||||||
"expo-font": "~14.0.10",
|
"expo-font": "~14.0.10",
|
||||||
"expo-haptics": "~15.0.8",
|
"expo-haptics": "~15.0.8",
|
||||||
"expo-image": "~3.0.11",
|
"expo-image": "~3.0.11",
|
||||||
|
"expo-image-manipulator": "^14.0.8",
|
||||||
|
"expo-image-picker": "^17.0.10",
|
||||||
"expo-keep-awake": "^15.0.8",
|
"expo-keep-awake": "^15.0.8",
|
||||||
"expo-linking": "~8.0.10",
|
"expo-linking": "~8.0.10",
|
||||||
"expo-router": "~6.0.19",
|
"expo-router": "~6.0.19",
|
||||||
"expo-secure-store": "^15.0.8",
|
"expo-secure-store": "^15.0.8",
|
||||||
"expo-speech": "~14.0.6",
|
"expo-speech": "~14.0.6",
|
||||||
|
"expo-speech-recognition": "^3.1.0",
|
||||||
"expo-splash-screen": "~31.0.12",
|
"expo-splash-screen": "~31.0.12",
|
||||||
"expo-status-bar": "~3.0.9",
|
"expo-status-bar": "~3.0.9",
|
||||||
"expo-symbols": "~1.0.8",
|
"expo-symbols": "~1.0.8",
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user