Improve STT quality and add session/chat management
- Switch Android STT from on-device to cloud recognition for better accuracy - Add lastMessageWasVoiceRef to prevent TTS for text-typed messages - Stop voice session and clear chat when changing Deployment or Voice API - Ensures clean state when switching between beneficiaries/models
This commit is contained in:
parent
5174366384
commit
9f12830850
4
app.json
4
app.json
@ -27,7 +27,7 @@
|
||||
"bitcode": false
|
||||
},
|
||||
"android": {
|
||||
"package": "com.wellnuo.app",
|
||||
"package": "com.wellnuo.BluetoothScanner",
|
||||
"softwareKeyboardLayoutMode": "resize",
|
||||
"adaptiveIcon": {
|
||||
"backgroundColor": "#E6F4FE",
|
||||
@ -56,7 +56,7 @@
|
||||
},
|
||||
"plugins": [
|
||||
[
|
||||
"@jamsch/expo-speech-recognition",
|
||||
"expo-speech-recognition",
|
||||
{
|
||||
"microphonePermission": "WellNuo needs access to your microphone to listen to your voice commands.",
|
||||
"speechRecognitionPermission": "WellNuo uses speech recognition to convert your voice to text for Julia AI."
|
||||
|
||||
@ -261,8 +261,8 @@ export default function TabLayout() {
|
||||
|
||||
// Delay to let TTS fully release audio focus, then restart STT
|
||||
// iOS: 300ms for smooth audio fade
|
||||
// Android: 50ms (Audio Focus releases immediately)
|
||||
const delay = Platform.OS === 'android' ? 50 : 300;
|
||||
// Android: 0ms - start immediately to catch first words (Audio Focus releases instantly)
|
||||
const delay = Platform.OS === 'android' ? 0 : 300;
|
||||
console.log(`${platformPrefix} [TabLayout] ⏱️ Waiting ${delay}ms before restarting STT (audio focus release)`);
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
|
||||
@ -157,6 +157,9 @@ export default function ChatScreen() {
|
||||
}
|
||||
}, [voicePartial, voiceIsListening]);
|
||||
|
||||
// Track if current message was sent via voice (to decide whether to speak response)
|
||||
const lastMessageWasVoiceRef = useRef(false);
|
||||
|
||||
// Clear input when voice switches to processing (transcript was sent)
|
||||
const prevVoiceStatusRef = useRef(voiceStatus);
|
||||
useEffect(() => {
|
||||
@ -164,6 +167,8 @@ export default function ChatScreen() {
|
||||
prevVoiceStatusRef.current = voiceStatus;
|
||||
if (prev === 'listening' && voiceStatus === 'processing') {
|
||||
setInput('');
|
||||
// Mark that this message was sent via voice
|
||||
lastMessageWasVoiceRef.current = true;
|
||||
}
|
||||
}, [voiceStatus]);
|
||||
|
||||
@ -187,34 +192,43 @@ export default function ChatScreen() {
|
||||
}, [])
|
||||
);
|
||||
|
||||
// When deployment ID changes, end call and clear chat
|
||||
// Track previous value to detect actual changes (not just re-renders)
|
||||
// When deployment ID changes BY USER ACTION, clear chat
|
||||
// Track previous value to detect actual changes (not just initial load)
|
||||
const previousDeploymentIdRef = useRef<string | null | undefined>(undefined);
|
||||
// Track if we've done initial load (to distinguish from user changing deployment)
|
||||
const initialLoadDoneRef = useRef(false);
|
||||
|
||||
useEffect(() => {
|
||||
// undefined means "not yet initialized" - store current value and skip
|
||||
// First render - just store the value, don't clear anything
|
||||
if (previousDeploymentIdRef.current === undefined) {
|
||||
console.log('[Chat] Initializing deployment tracking:', customDeploymentId, 'name:', deploymentName);
|
||||
console.log('[Chat] First render, storing deployment:', customDeploymentId);
|
||||
previousDeploymentIdRef.current = customDeploymentId;
|
||||
// Update initial message with deployment name if we have one
|
||||
if (customDeploymentId || deploymentName) {
|
||||
setMessages([createInitialMessage(deploymentName)]);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if deployment actually changed
|
||||
if (previousDeploymentIdRef.current !== customDeploymentId) {
|
||||
console.log('[Chat] Deployment changed!', {
|
||||
// Initial async load completed (null → actual value) - don't clear chat!
|
||||
if (!initialLoadDoneRef.current && previousDeploymentIdRef.current === null && customDeploymentId) {
|
||||
console.log('[Chat] Initial deployment load complete:', customDeploymentId, '- NOT clearing chat');
|
||||
previousDeploymentIdRef.current = customDeploymentId;
|
||||
initialLoadDoneRef.current = true;
|
||||
return;
|
||||
}
|
||||
|
||||
// Mark initial load as done
|
||||
if (!initialLoadDoneRef.current && customDeploymentId) {
|
||||
initialLoadDoneRef.current = true;
|
||||
}
|
||||
|
||||
// Only clear chat if deployment ACTUALLY changed by user (after initial load)
|
||||
if (initialLoadDoneRef.current && previousDeploymentIdRef.current !== customDeploymentId) {
|
||||
console.log('[Chat] Deployment CHANGED by user!', {
|
||||
old: previousDeploymentIdRef.current,
|
||||
new: customDeploymentId,
|
||||
name: deploymentName,
|
||||
});
|
||||
|
||||
// Clear chat with new initial message (use name instead of ID)
|
||||
// Clear chat with new initial message
|
||||
setMessages([createInitialMessage(deploymentName)]);
|
||||
|
||||
// Update ref
|
||||
previousDeploymentIdRef.current = customDeploymentId;
|
||||
}
|
||||
}, [customDeploymentId, deploymentName, createInitialMessage]);
|
||||
@ -358,6 +372,10 @@ export default function ChatScreen() {
|
||||
setInput('');
|
||||
inputRef.current = '';
|
||||
|
||||
// This message was sent via text input (keyboard), not voice
|
||||
// Reset the flag so response won't be spoken
|
||||
lastMessageWasVoiceRef.current = false;
|
||||
|
||||
setMessages(prev => [...prev, userMessage]);
|
||||
setIsSending(true);
|
||||
Keyboard.dismiss();
|
||||
@ -414,9 +432,11 @@ export default function ChatScreen() {
|
||||
};
|
||||
setMessages(prev => [...prev, assistantMessage]);
|
||||
|
||||
// Only speak the response if voice session is active (FAB pressed)
|
||||
// Don't auto-speak for text-only chat messages
|
||||
if (voiceIsActive) {
|
||||
// Only speak the response if:
|
||||
// 1. Voice session is active (FAB pressed) AND
|
||||
// 2. The user's message was sent via voice (not typed)
|
||||
// This way, typing a message while voice is active won't trigger TTS
|
||||
if (voiceIsActive && lastMessageWasVoiceRef.current) {
|
||||
speak(responseText);
|
||||
}
|
||||
} else {
|
||||
|
||||
@ -16,6 +16,7 @@ import { Ionicons } from '@expo/vector-icons';
|
||||
import { SafeAreaView } from 'react-native-safe-area-context';
|
||||
import { useAuth } from '@/contexts/AuthContext';
|
||||
import { useVoice } from '@/contexts/VoiceContext';
|
||||
import { useChat } from '@/contexts/ChatContext';
|
||||
import { api } from '@/services/api';
|
||||
import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme';
|
||||
|
||||
@ -56,7 +57,8 @@ function MenuItem({
|
||||
|
||||
export default function ProfileScreen() {
|
||||
const { user, logout } = useAuth();
|
||||
const { updateVoiceApiType, stopSession, isActive } = useVoice();
|
||||
const { updateVoiceApiType, stopSession } = useVoice();
|
||||
const { clearMessages } = useChat();
|
||||
const [deploymentId, setDeploymentId] = useState<string>('');
|
||||
const [deploymentName, setDeploymentName] = useState<string>('');
|
||||
const [showDeploymentModal, setShowDeploymentModal] = useState(false);
|
||||
@ -124,6 +126,18 @@ export default function ProfileScreen() {
|
||||
try {
|
||||
const result = await api.validateDeploymentId(trimmed);
|
||||
if (result.ok && result.data?.valid) {
|
||||
// ALWAYS stop voice session when deployment changes
|
||||
console.log('[Profile] Stopping voice session and clearing chat before deployment change');
|
||||
stopSession();
|
||||
|
||||
// Clear chat history when deployment changes
|
||||
clearMessages({
|
||||
id: '1',
|
||||
role: 'assistant',
|
||||
content: `Hello! I'm Julia, your AI wellness companion.${result.data.name ? `\n\nI'm here to help you monitor ${result.data.name}.` : ''}\n\nType a message below to chat with me.`,
|
||||
timestamp: new Date(),
|
||||
});
|
||||
|
||||
await api.setDeploymentId(trimmed);
|
||||
if (result.data.name) {
|
||||
await api.setDeploymentName(result.data.name);
|
||||
@ -142,25 +156,43 @@ export default function ProfileScreen() {
|
||||
setIsValidating(false);
|
||||
}
|
||||
} else {
|
||||
// ALWAYS stop voice session when deployment is cleared
|
||||
console.log('[Profile] Stopping voice session and clearing chat before clearing deployment');
|
||||
stopSession();
|
||||
|
||||
// Clear chat history when deployment is cleared
|
||||
clearMessages({
|
||||
id: '1',
|
||||
role: 'assistant',
|
||||
content: "Hello! I'm Julia, your AI wellness companion.\n\nType a message below to chat with me.",
|
||||
timestamp: new Date(),
|
||||
});
|
||||
|
||||
await api.clearDeploymentId();
|
||||
setDeploymentId('');
|
||||
setDeploymentName('');
|
||||
setShowDeploymentModal(false);
|
||||
}
|
||||
}, [tempDeploymentId]);
|
||||
}, [tempDeploymentId, stopSession, clearMessages]);
|
||||
|
||||
const saveVoiceApiType = useCallback(async () => {
|
||||
// Stop active voice session if any before changing API type
|
||||
if (isActive) {
|
||||
console.log('[Profile] Stopping active voice session before API type change');
|
||||
stopSession();
|
||||
}
|
||||
// ALWAYS stop voice session when API type changes
|
||||
console.log('[Profile] Stopping voice session and clearing chat before API type change');
|
||||
stopSession();
|
||||
|
||||
// Clear chat history when Voice API changes
|
||||
clearMessages({
|
||||
id: '1',
|
||||
role: 'assistant',
|
||||
content: "Hello! I'm Julia, your AI wellness companion.\n\nType a message below to chat with me.",
|
||||
timestamp: new Date(),
|
||||
});
|
||||
|
||||
await api.setVoiceApiType(tempVoiceApiType);
|
||||
setVoiceApiType(tempVoiceApiType);
|
||||
updateVoiceApiType(tempVoiceApiType);
|
||||
setShowVoiceApiModal(false);
|
||||
}, [tempVoiceApiType, updateVoiceApiType, isActive, stopSession]);
|
||||
}, [tempVoiceApiType, updateVoiceApiType, stopSession, clearMessages]);
|
||||
|
||||
const openTerms = () => {
|
||||
router.push('/terms');
|
||||
|
||||
@ -30,7 +30,7 @@ let ExpoSpeechRecognitionModule: any = null;
|
||||
let useSpeechRecognitionEvent: any = () => {}; // no-op by default
|
||||
|
||||
try {
|
||||
const speechRecognition = require('@jamsch/expo-speech-recognition');
|
||||
const speechRecognition = require('expo-speech-recognition');
|
||||
ExpoSpeechRecognitionModule = speechRecognition.ExpoSpeechRecognitionModule;
|
||||
useSpeechRecognitionEvent = speechRecognition.useSpeechRecognitionEvent;
|
||||
} catch (e) {
|
||||
@ -258,6 +258,10 @@ export function useSpeechRecognition(
|
||||
interimResults,
|
||||
continuous,
|
||||
addsPunctuation: Platform.OS === 'ios' ? addsPunctuation : undefined,
|
||||
// Android: use CLOUD recognition for better quality
|
||||
// On-device models often have worse accuracy
|
||||
// Setting to false allows the system to use Google's cloud ASR
|
||||
requiresOnDeviceRecognition: false,
|
||||
// Android-specific: longer silence timeout for more natural pauses
|
||||
// CRITICAL FIX: Increased from 2000ms to 4000ms to prevent premature speech cutoff
|
||||
// This allows users to pause between sentences without being cut off
|
||||
|
||||
877
package-lock.json
generated
877
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@ -16,25 +16,29 @@
|
||||
"dependencies": {
|
||||
"@dr.pogodin/react-native-fs": "^2.36.2",
|
||||
"@expo/vector-icons": "^15.0.3",
|
||||
"@jamsch/expo-speech-recognition": "^0.2.15",
|
||||
"@notifee/react-native": "^9.1.8",
|
||||
"@react-native-async-storage/async-storage": "2.2.0",
|
||||
"@react-navigation/bottom-tabs": "^7.4.0",
|
||||
"@react-navigation/elements": "^2.6.3",
|
||||
"@react-navigation/native": "^7.1.8",
|
||||
"@stripe/stripe-react-native": "^0.58.0",
|
||||
"expo": "~54.0.29",
|
||||
"expo-clipboard": "~8.0.8",
|
||||
"expo-constants": "~18.0.12",
|
||||
"expo-crypto": "^15.0.8",
|
||||
"expo-device": "~8.0.10",
|
||||
"expo-file-system": "~19.0.21",
|
||||
"expo-font": "~14.0.10",
|
||||
"expo-haptics": "~15.0.8",
|
||||
"expo-image": "~3.0.11",
|
||||
"expo-image-manipulator": "^14.0.8",
|
||||
"expo-image-picker": "^17.0.10",
|
||||
"expo-keep-awake": "^15.0.8",
|
||||
"expo-linking": "~8.0.10",
|
||||
"expo-router": "~6.0.19",
|
||||
"expo-secure-store": "^15.0.8",
|
||||
"expo-speech": "~14.0.6",
|
||||
"expo-speech-recognition": "^3.1.0",
|
||||
"expo-splash-screen": "~31.0.12",
|
||||
"expo-status-bar": "~3.0.9",
|
||||
"expo-symbols": "~1.0.8",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user