Sergei
81a0c59060
PROBLEM: startListening() triggers spurious AppState change to "background" on Android, causing voice session to stop immediately after Julia responds. ROOT CAUSE: React Native AppState bug - requesting audio focus triggers false background event. SOLUTION: - Added sttStartingIgnoreAppStateRef flag - Ignore AppState changes for 200ms after startListening() call - Protects against false session termination during STT initialization CHANGES: - app/(tabs)/_layout.tsx: Added Android workaround with 200ms protection window - VOICE_DEBUG_GUIDE.md: Documented bug, workaround, and expected logs RESULT: Voice session now continues correctly after Julia's response on Android. STT successfully restarts and user can speak again without manual restart.
482 lines
19 KiB
TypeScript
482 lines
19 KiB
TypeScript
import { Tabs } from 'expo-router';
|
|
import React, { useCallback, useEffect, useRef } from 'react';
|
|
import { Platform, View, AppState, AppStateStatus, TouchableOpacity, StyleSheet } from 'react-native';
|
|
import { Feather } from '@expo/vector-icons';
|
|
import { useSafeAreaInsets } from 'react-native-safe-area-context';
|
|
|
|
import { HapticTab } from '@/components/haptic-tab';
|
|
import { VoiceFAB } from '@/components/VoiceFAB';
|
|
import { AppColors } from '@/constants/theme';
|
|
import { useColorScheme } from '@/hooks/use-color-scheme';
|
|
import { useVoiceCall } from '@/contexts/VoiceCallContext';
|
|
import { useVoice } from '@/contexts/VoiceContext';
|
|
import { useSpeechRecognition } from '@/hooks/useSpeechRecognition';
|
|
|
|
export default function TabLayout() {
|
|
const colorScheme = useColorScheme();
|
|
const isDark = colorScheme === 'dark';
|
|
const insets = useSafeAreaInsets();
|
|
// VoiceFAB uses VoiceCallContext internally to hide when call is active
|
|
useVoiceCall(); // Ensure context is available
|
|
|
|
// Voice context for listening mode toggle and TTS interruption
|
|
const {
|
|
isListening,
|
|
isSpeaking,
|
|
status,
|
|
startSession,
|
|
stopSession,
|
|
interruptIfSpeaking,
|
|
setTranscript,
|
|
setPartialTranscript,
|
|
partialTranscript, // for iOS auto-stop timer
|
|
sendTranscript,
|
|
} = useVoice();
|
|
|
|
// Track whether session is active (listening mode on, even during TTS)
|
|
const sessionActiveRef = useRef(false);
|
|
// Track if we need to restart STT after it ends during active session
|
|
const shouldRestartSTTRef = useRef(false);
|
|
// Track pending transcript from interruption (to send after TTS stops)
|
|
const pendingInterruptTranscriptRef = useRef<string | null>(null);
|
|
|
|
// Callback for voice detection - interrupt TTS when user speaks
|
|
// NOTE: On Android, STT doesn't run during TTS (shared audio focus),
|
|
// so interruption on Android happens via FAB press instead.
|
|
// On iOS, STT can run alongside TTS, so voice detection works.
|
|
const handleVoiceDetected = useCallback(() => {
|
|
const platformPrefix = Platform.OS === 'ios' ? '[iOS]' : '[Android]';
|
|
console.log(`${platformPrefix} [TabLayout] handleVoiceDetected called - status: ${status}, isSpeaking: ${isSpeaking}`);
|
|
|
|
if (Platform.OS === 'ios' && (status === 'speaking' || isSpeaking)) {
|
|
console.log('[iOS] [TabLayout] Voice detected during TTS - INTERRUPTING Julia');
|
|
interruptIfSpeaking();
|
|
} else if (Platform.OS === 'android') {
|
|
console.log('[Android] [TabLayout] Voice detected but ignoring (STT disabled during TTS on Android)');
|
|
}
|
|
}, [status, isSpeaking, interruptIfSpeaking]);
|
|
|
|
// Callback when STT ends - may need to restart if session is still active
|
|
const handleSTTEnd = useCallback(() => {
|
|
const platformPrefix = Platform.OS === 'ios' ? '[iOS]' : '[Android]';
|
|
console.log(`${platformPrefix} [TabLayout] handleSTTEnd - sessionActive: ${sessionActiveRef.current}, status: ${status}`);
|
|
|
|
// If session is still active (user didn't stop it), we should restart STT
|
|
// This ensures STT continues during and after TTS playback
|
|
if (sessionActiveRef.current) {
|
|
shouldRestartSTTRef.current = true;
|
|
console.log(`${platformPrefix} [TabLayout] → shouldRestartSTT set to TRUE`);
|
|
} else {
|
|
console.log(`${platformPrefix} [TabLayout] → Session not active, will NOT restart STT`);
|
|
}
|
|
}, [status]);
|
|
|
|
// Callback for STT results
|
|
const handleSpeechResult = useCallback((transcript: string, isFinal: boolean) => {
|
|
const platformPrefix = Platform.OS === 'ios' ? '[iOS]' : '[Android]';
|
|
console.log(`${platformPrefix} [TabLayout] handleSpeechResult - isFinal: ${isFinal}, status: ${status}, transcript: "${transcript.slice(0, 40)}..."`);
|
|
|
|
// Ignore any STT results during TTS playback or processing (echo prevention)
|
|
if (status === 'speaking' || status === 'processing') {
|
|
if (isFinal) {
|
|
// User interrupted Julia with speech — store to send after TTS stops
|
|
console.log(`${platformPrefix} [TabLayout] Got FINAL result during ${status} - storing for after interruption: "${transcript}"`);
|
|
pendingInterruptTranscriptRef.current = transcript;
|
|
} else {
|
|
console.log(`${platformPrefix} [TabLayout] Ignoring PARTIAL transcript during ${status} (likely echo)`);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (isFinal) {
|
|
console.log(`${platformPrefix} [TabLayout] → Processing FINAL transcript, sending to API`);
|
|
setTranscript(transcript);
|
|
sendTranscript(transcript);
|
|
} else {
|
|
console.log(`${platformPrefix} [TabLayout] → Updating PARTIAL transcript`);
|
|
setPartialTranscript(transcript);
|
|
}
|
|
}, [setTranscript, setPartialTranscript, sendTranscript, status]);
|
|
|
|
// Speech recognition with voice detection callback
|
|
const {
|
|
startListening,
|
|
stopListening,
|
|
isListening: sttIsListening,
|
|
} = useSpeechRecognition({
|
|
lang: 'en-US',
|
|
continuous: true,
|
|
interimResults: true,
|
|
onVoiceDetected: handleVoiceDetected,
|
|
onResult: handleSpeechResult,
|
|
onEnd: handleSTTEnd,
|
|
});
|
|
|
|
// Ref to prevent concurrent startListening calls
|
|
const sttStartingRef = useRef(false);
|
|
// Ref to ignore AppState changes during STT start (Android bug workaround)
|
|
const sttStartingIgnoreAppStateRef = useRef(false);
|
|
// Ref to track last partial transcript for iOS auto-stop
|
|
const lastPartialTextRef = useRef('');
|
|
const silenceTimerRef = useRef<NodeJS.Timeout | null>(null);
|
|
|
|
// iOS AUTO-STOP: Stop STT after 2 seconds of silence (no new partial transcripts)
|
|
// This triggers onEnd → iOS fix sends lastPartial as final
|
|
useEffect(() => {
|
|
const platformPrefix = Platform.OS === 'ios' ? '[iOS]' : '[Android]';
|
|
|
|
// Clear existing timer
|
|
if (silenceTimerRef.current) {
|
|
clearTimeout(silenceTimerRef.current);
|
|
silenceTimerRef.current = null;
|
|
}
|
|
|
|
// Only track silence when STT is listening (not during processing/speaking)
|
|
if (sttIsListening && status !== 'processing' && status !== 'speaking') {
|
|
// Get current partial from VoiceContext (set by handleSpeechResult)
|
|
const currentPartial = partialTranscript;
|
|
|
|
// If partial changed, update ref and set new 2s timer
|
|
if (currentPartial !== lastPartialTextRef.current) {
|
|
console.log(`${platformPrefix} [TabLayout] Partial changed: "${lastPartialTextRef.current}" → "${currentPartial}"`);
|
|
lastPartialTextRef.current = currentPartial;
|
|
|
|
// Start 2-second silence timer
|
|
silenceTimerRef.current = setTimeout(() => {
|
|
if (sttIsListening && sessionActiveRef.current) {
|
|
if (Platform.OS === 'ios') {
|
|
console.log('[iOS] [TabLayout] 🍎 AUTO-STOP: 2s silence - stopping STT to trigger onEnd → iOS fix');
|
|
} else {
|
|
console.log('[Android] [TabLayout] 🤖 AUTO-STOP: 2s silence - stopping STT');
|
|
}
|
|
stopListening();
|
|
}
|
|
}, 2000);
|
|
|
|
console.log(`${platformPrefix} [TabLayout] → Started 2s silence timer`);
|
|
}
|
|
}
|
|
|
|
return () => {
|
|
if (silenceTimerRef.current) {
|
|
clearTimeout(silenceTimerRef.current);
|
|
silenceTimerRef.current = null;
|
|
}
|
|
};
|
|
}, [sttIsListening, status, partialTranscript, stopListening]);
|
|
|
|
// Safe wrapper to start STT with debounce protection
|
|
const safeStartSTT = useCallback(() => {
|
|
const platformPrefix = Platform.OS === 'ios' ? '[iOS]' : '[Android]';
|
|
|
|
if (sttIsListening || sttStartingRef.current) {
|
|
console.log(`${platformPrefix} [TabLayout] safeStartSTT - already listening or starting, skipping`);
|
|
return; // Already listening or starting
|
|
}
|
|
|
|
// Don't start STT during TTS on Android - they share audio focus
|
|
if (Platform.OS === 'android' && (status === 'speaking' || isSpeaking)) {
|
|
console.log('[Android] [TabLayout] ⚠️ SKIPPING STT start - TTS is playing (audio focus conflict)');
|
|
return;
|
|
}
|
|
|
|
sttStartingRef.current = true;
|
|
|
|
// ANDROID BUG WORKAROUND: startListening() triggers AppState change to background
|
|
// Ignore AppState changes for 200ms after starting STT
|
|
if (Platform.OS === 'android') {
|
|
sttStartingIgnoreAppStateRef.current = true;
|
|
console.log('[Android] [TabLayout] 🛡️ Ignoring AppState changes for 200ms (STT start workaround)');
|
|
setTimeout(() => {
|
|
sttStartingIgnoreAppStateRef.current = false;
|
|
console.log('[Android] [TabLayout] ✅ AppState monitoring resumed');
|
|
}, 200);
|
|
}
|
|
|
|
console.log(`${platformPrefix} [TabLayout] ▶️ STARTING STT... (status: ${status})`);
|
|
|
|
startListening()
|
|
.then(() => {
|
|
console.log(`${platformPrefix} [TabLayout] ✅ STT started successfully`);
|
|
})
|
|
.catch((err) => {
|
|
console.error(`${platformPrefix} [TabLayout] ❌ STT start failed:`, err);
|
|
})
|
|
.finally(() => {
|
|
sttStartingRef.current = false;
|
|
});
|
|
}, [sttIsListening, status, isSpeaking, startListening]);
|
|
|
|
// Update session active ref when isListening changes
|
|
useEffect(() => {
|
|
sessionActiveRef.current = isListening;
|
|
if (!isListening) {
|
|
shouldRestartSTTRef.current = false;
|
|
}
|
|
}, [isListening]);
|
|
|
|
// Start/stop STT when voice session starts/stops
|
|
useEffect(() => {
|
|
const platformPrefix = Platform.OS === 'ios' ? '[iOS]' : '[Android]';
|
|
|
|
if (isListening) {
|
|
console.log(`${platformPrefix} [TabLayout] 🎤 Voice session STARTED - starting STT`);
|
|
safeStartSTT();
|
|
} else {
|
|
console.log(`${platformPrefix} [TabLayout] 🛑 Voice session ENDED - stopping STT`);
|
|
stopListening();
|
|
}
|
|
}, [isListening]); // eslint-disable-line react-hooks/exhaustive-deps
|
|
|
|
// Track previous status to detect transition from speaking to listening
|
|
const prevStatusRef = useRef<typeof status>('idle');
|
|
|
|
// Stop STT when entering processing or speaking state (prevent echo)
|
|
// Restart STT when TTS finishes (speaking → listening)
|
|
useEffect(() => {
|
|
const platformPrefix = Platform.OS === 'ios' ? '[iOS]' : '[Android]';
|
|
const prevStatus = prevStatusRef.current;
|
|
prevStatusRef.current = status;
|
|
|
|
console.log(`${platformPrefix} [TabLayout] Status transition: ${prevStatus} → ${status}, sttIsListening: ${sttIsListening}`);
|
|
|
|
// Stop STT when processing starts or TTS starts (prevent Julia hearing herself)
|
|
if ((status === 'processing' || status === 'speaking') && sttIsListening) {
|
|
console.log(`${platformPrefix} [TabLayout] ⏸️ Stopping STT during ${status} (echo prevention)`);
|
|
stopListening();
|
|
}
|
|
|
|
// When TTS finishes (speaking → listening), restart STT
|
|
if (prevStatus === 'speaking' && status === 'listening' && sessionActiveRef.current) {
|
|
console.log(`${platformPrefix} [TabLayout] 🔄 TTS FINISHED - preparing to restart STT`);
|
|
|
|
// Process pending transcript from interruption if any
|
|
const pendingTranscript = pendingInterruptTranscriptRef.current;
|
|
if (pendingTranscript) {
|
|
console.log(`${platformPrefix} [TabLayout] 📝 Processing pending interrupt transcript: "${pendingTranscript}"`);
|
|
pendingInterruptTranscriptRef.current = null;
|
|
setTranscript(pendingTranscript);
|
|
sendTranscript(pendingTranscript);
|
|
}
|
|
|
|
// Delay to let TTS fully release audio focus, then restart STT
|
|
// iOS: 300ms for smooth audio fade
|
|
// Android: 50ms (Audio Focus releases immediately)
|
|
const delay = Platform.OS === 'android' ? 50 : 300;
|
|
console.log(`${platformPrefix} [TabLayout] ⏱️ Waiting ${delay}ms before restarting STT (audio focus release)`);
|
|
|
|
const timer = setTimeout(() => {
|
|
if (sessionActiveRef.current) {
|
|
console.log(`${platformPrefix} [TabLayout] ⏰ Delay complete - restarting STT now`);
|
|
safeStartSTT();
|
|
} else {
|
|
console.log(`${platformPrefix} [TabLayout] ⚠️ Session stopped during delay, NOT restarting STT`);
|
|
}
|
|
}, delay);
|
|
return () => clearTimeout(timer);
|
|
}
|
|
|
|
// When processing finishes and goes to speaking, STT is already stopped (above)
|
|
// When speaking finishes and goes to listening, STT restarts (above)
|
|
}, [status]); // eslint-disable-line react-hooks/exhaustive-deps
|
|
|
|
// When STT ends unexpectedly during active session, restart it (but not during TTS)
|
|
useEffect(() => {
|
|
const platformPrefix = Platform.OS === 'ios' ? '[iOS]' : '[Android]';
|
|
|
|
if (
|
|
shouldRestartSTTRef.current &&
|
|
sessionActiveRef.current &&
|
|
!sttIsListening &&
|
|
status !== 'processing' &&
|
|
status !== 'speaking'
|
|
) {
|
|
shouldRestartSTTRef.current = false;
|
|
console.log(`${platformPrefix} [TabLayout] 🔄 STT ended UNEXPECTEDLY - will restart in 300ms`);
|
|
console.log(`${platformPrefix} [TabLayout] → Conditions: sessionActive=${sessionActiveRef.current}, status=${status}`);
|
|
|
|
const timer = setTimeout(() => {
|
|
if (sessionActiveRef.current) {
|
|
console.log(`${platformPrefix} [TabLayout] ⏰ Restarting STT after unexpected end`);
|
|
safeStartSTT();
|
|
} else {
|
|
console.log(`${platformPrefix} [TabLayout] ⚠️ Session stopped during delay, NOT restarting`);
|
|
}
|
|
}, 300);
|
|
return () => clearTimeout(timer);
|
|
}
|
|
}, [sttIsListening]); // eslint-disable-line react-hooks/exhaustive-deps
|
|
|
|
// Handle app state changes (background/foreground)
|
|
useEffect(() => {
|
|
const handleAppStateChange = (nextAppState: AppStateStatus) => {
|
|
const platformPrefix = Platform.OS === 'ios' ? '[iOS]' : '[Android]';
|
|
console.log(`${platformPrefix} [TabLayout] 📱 AppState changed to: "${nextAppState}"`);
|
|
|
|
// ANDROID BUG WORKAROUND: Ignore AppState changes during STT start
|
|
// startListening() triggers spurious background transition on Android
|
|
if (Platform.OS === 'android' && sttStartingIgnoreAppStateRef.current) {
|
|
console.log(`[Android] [TabLayout] 🛡️ IGNORING AppState change (STT start protection active)`);
|
|
return;
|
|
}
|
|
|
|
// When app goes to background/inactive - stop voice session
|
|
// STT/TTS cannot work in background, so it's pointless to keep session active
|
|
if ((nextAppState === 'background' || nextAppState === 'inactive') && sessionActiveRef.current) {
|
|
console.log(`${platformPrefix} [TabLayout] App going to ${nextAppState} - stopping voice session`);
|
|
stopListening();
|
|
stopSession();
|
|
sessionActiveRef.current = false;
|
|
shouldRestartSTTRef.current = false;
|
|
pendingInterruptTranscriptRef.current = null;
|
|
}
|
|
|
|
// When app comes back to foreground - do NOT auto-restart session
|
|
// User must manually press FAB to start new session
|
|
if (nextAppState === 'active') {
|
|
console.log(`${platformPrefix} [TabLayout] App foregrounded - session remains stopped (user must restart via FAB)`);
|
|
}
|
|
};
|
|
|
|
const subscription = AppState.addEventListener('change', handleAppStateChange);
|
|
return () => subscription.remove();
|
|
}, [stopListening, stopSession]);
|
|
|
|
// Handle voice FAB press - toggle listening mode
|
|
// Must check ALL active states (listening, processing, speaking), not just isListening
|
|
const handleVoiceFABPress = useCallback(() => {
|
|
const platformPrefix = Platform.OS === 'ios' ? '[iOS]' : '[Android]';
|
|
const isSessionActive = isListening || status === 'speaking' || status === 'processing';
|
|
console.log(`${platformPrefix} [TabLayout] 🎯 FAB PRESSED - isSessionActive: ${isSessionActive}, status: ${status}, isListening: ${isListening}`);
|
|
|
|
if (isSessionActive) {
|
|
// Force-stop everything: STT, TTS, and session state
|
|
console.log(`${platformPrefix} [TabLayout] 🛑 FORCE-STOPPING everything (FAB stop)`);
|
|
stopListening();
|
|
stopSession();
|
|
sessionActiveRef.current = false;
|
|
shouldRestartSTTRef.current = false;
|
|
pendingInterruptTranscriptRef.current = null;
|
|
console.log(`${platformPrefix} [TabLayout] → All flags cleared`);
|
|
} else {
|
|
console.log(`${platformPrefix} [TabLayout] ▶️ STARTING session (FAB start)`);
|
|
startSession();
|
|
}
|
|
}, [isListening, status, startSession, stopSession, stopListening]);
|
|
|
|
// Calculate tab bar height based on safe area
|
|
// On iOS with home indicator, insets.bottom is ~34px
|
|
// On Android with gesture navigation or software buttons (Samsung/Pixel):
|
|
// - insets.bottom should reflect the navigation bar height
|
|
// - But some devices/modes may return 0, so we add a minimum for Android
|
|
// Android minimum: 16px to ensure content doesn't touch system buttons
|
|
const androidMinPadding = Platform.OS === 'android' ? 16 : 0;
|
|
const bottomPadding = Math.max(insets.bottom, androidMinPadding, 10);
|
|
const tabBarHeight = 60 + bottomPadding; // 60px for content + safe area padding
|
|
|
|
return (
|
|
<View style={{ flex: 1 }}>
|
|
<Tabs
|
|
screenOptions={{
|
|
tabBarActiveTintColor: AppColors.primary,
|
|
tabBarInactiveTintColor: isDark ? '#9BA1A6' : '#687076',
|
|
tabBarStyle: {
|
|
backgroundColor: isDark ? '#151718' : AppColors.background,
|
|
borderTopColor: isDark ? '#2D3135' : AppColors.border,
|
|
height: tabBarHeight,
|
|
paddingBottom: bottomPadding,
|
|
paddingTop: 10,
|
|
},
|
|
tabBarLabelStyle: {
|
|
fontSize: 11,
|
|
fontWeight: '500',
|
|
},
|
|
headerShown: false,
|
|
tabBarButton: HapticTab,
|
|
}}
|
|
>
|
|
<Tabs.Screen
|
|
name="index"
|
|
options={{
|
|
title: 'Dashboard',
|
|
tabBarIcon: ({ color, size }) => (
|
|
<Feather name="grid" size={22} color={color} />
|
|
),
|
|
}}
|
|
/>
|
|
{/* Hide old dashboard - now index shows WebView dashboard */}
|
|
<Tabs.Screen
|
|
name="dashboard"
|
|
options={{
|
|
href: null,
|
|
}}
|
|
/>
|
|
{/* Chat with Julia AI */}
|
|
<Tabs.Screen
|
|
name="chat"
|
|
options={{
|
|
title: 'Julia',
|
|
tabBarIcon: ({ color, size }) => (
|
|
<Feather name="message-circle" size={22} color={color} />
|
|
),
|
|
}}
|
|
/>
|
|
{/* Voice FAB - center tab button */}
|
|
<Tabs.Screen
|
|
name="explore"
|
|
options={{
|
|
title: '',
|
|
tabBarButton: () => (
|
|
<View style={tabFABStyles.fabWrapper}>
|
|
<VoiceFAB onPress={handleVoiceFABPress} isListening={isListening || status === 'speaking' || status === 'processing'} />
|
|
</View>
|
|
),
|
|
}}
|
|
/>
|
|
{/* Voice Debug - VISIBLE for debugging */}
|
|
<Tabs.Screen
|
|
name="voice-debug"
|
|
options={{
|
|
title: 'Debug',
|
|
tabBarIcon: ({ color, size }) => (
|
|
<Feather name="activity" size={22} color={color} />
|
|
),
|
|
}}
|
|
/>
|
|
<Tabs.Screen
|
|
name="profile"
|
|
options={{
|
|
title: 'Profile',
|
|
tabBarIcon: ({ color, size }) => (
|
|
<Feather name="user" size={22} color={color} />
|
|
),
|
|
}}
|
|
/>
|
|
{/* Audio Debug - hidden */}
|
|
<Tabs.Screen
|
|
name="audio-debug"
|
|
options={{
|
|
href: null,
|
|
}}
|
|
/>
|
|
{/* Beneficiaries - hidden from tab bar but keeps tab bar visible */}
|
|
<Tabs.Screen
|
|
name="beneficiaries"
|
|
options={{
|
|
href: null,
|
|
}}
|
|
/>
|
|
</Tabs>
|
|
</View>
|
|
);
|
|
}
|
|
|
|
const tabFABStyles = StyleSheet.create({
|
|
fabWrapper: {
|
|
flex: 1,
|
|
alignItems: 'center',
|
|
justifyContent: 'center',
|
|
top: -20,
|
|
},
|
|
});
|