Keep STT listening during TTS playback for interruption detection

- Add sessionActiveRef to track when voice session is active
- Add shouldRestartSTTRef to auto-restart STT after it ends
- STT now continues listening during TTS playback
- Voice detection callback checks both status and isSpeaking
- Final results during TTS are ignored (user interrupted)
- STT automatically restarts after ending if session is still active

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Sergei 2026-01-27 16:36:08 -08:00
parent 3c7a48df5b
commit 59f1f088ed

View File

@ -1,5 +1,5 @@
import { Tabs } from 'expo-router';
import React, { useCallback, useEffect } from 'react';
import React, { useCallback, useEffect, useRef } from 'react';
import { Platform, View } from 'react-native';
import { Feather } from '@expo/vector-icons';
import { useSafeAreaInsets } from 'react-native-safe-area-context';
@ -22,6 +22,7 @@ export default function TabLayout() {
// Voice context for listening mode toggle and TTS interruption
const {
isListening,
isSpeaking,
status,
startSession,
stopSession,
@ -31,46 +32,95 @@ export default function TabLayout() {
sendTranscript,
} = useVoice();
// Track whether session is active (listening mode on, even during TTS)
const sessionActiveRef = useRef(false);
// Track if we need to restart STT after it ends during active session
const shouldRestartSTTRef = useRef(false);
// Callback for voice detection - interrupt TTS when user speaks
const handleVoiceDetected = useCallback(() => {
// Interrupt TTS when user starts speaking during 'speaking' state
if (status === 'speaking') {
console.log('[TabLayout] Voice detected during speaking - interrupting TTS');
if (status === 'speaking' || isSpeaking) {
console.log('[TabLayout] Voice detected during TTS playback - interrupting');
interruptIfSpeaking();
}
}, [status, interruptIfSpeaking]);
}, [status, isSpeaking, interruptIfSpeaking]);
// Callback when STT ends - may need to restart if session is still active
const handleSTTEnd = useCallback(() => {
console.log('[TabLayout] STT ended, sessionActive:', sessionActiveRef.current);
// If session is still active (user didn't stop it), we should restart STT
// This ensures STT continues during and after TTS playback
if (sessionActiveRef.current) {
shouldRestartSTTRef.current = true;
}
}, []);
// Callback for STT results
const handleSpeechResult = useCallback((transcript: string, isFinal: boolean) => {
if (isFinal) {
// Only process final results when NOT speaking (avoid processing interrupted speech)
if (!isSpeaking && status !== 'speaking') {
setTranscript(transcript);
// Send to API when final result is received
sendTranscript(transcript);
} else {
// Got final result while speaking - this is the interruption
console.log('[TabLayout] Got final result while TTS playing - user interrupted');
}
} else {
setPartialTranscript(transcript);
}
}, [setTranscript, setPartialTranscript, sendTranscript]);
}, [setTranscript, setPartialTranscript, sendTranscript, isSpeaking, status]);
// Speech recognition with voice detection callback
const {
startListening,
stopListening,
isListening: sttIsListening,
} = useSpeechRecognition({
continuous: true,
interimResults: true,
onVoiceDetected: handleVoiceDetected,
onResult: handleSpeechResult,
onEnd: handleSTTEnd,
});
// Update session active ref when isListening changes
useEffect(() => {
sessionActiveRef.current = isListening;
if (!isListening) {
shouldRestartSTTRef.current = false;
}
}, [isListening]);
// Start/stop STT when voice session starts/stops
useEffect(() => {
if (isListening) {
console.log('[TabLayout] Starting STT for voice session');
startListening();
} else {
console.log('[TabLayout] Stopping STT - session ended');
stopListening();
}
}, [isListening, startListening, stopListening]);
// Restart STT if it ended while session is still active
// This ensures continuous listening even during/after TTS playback
useEffect(() => {
if (shouldRestartSTTRef.current && sessionActiveRef.current && !sttIsListening) {
console.log('[TabLayout] Restarting STT - session still active');
shouldRestartSTTRef.current = false;
// Small delay to ensure clean restart
const timer = setTimeout(() => {
if (sessionActiveRef.current) {
startListening();
}
}, 100);
return () => clearTimeout(timer);
}
}, [sttIsListening, startListening]);
// Handle voice FAB press - toggle listening mode
const handleVoiceFABPress = useCallback(() => {
if (isListening) {